OXYGENHUB

Coverage / crates/oxy-core/src/repo/walk.rs 87.6%

211 of 241 lines run, 38 of 61 functions · run #423 on main · run missed

1
//! Listing the working copy (spec §12.3): every file and symlink under the
2
//! root, found in parallel. `.oxy` is skipped, and so is a nested
3
//! repository (a directory holding `.oxy` or `.git`), which is reported
4
//! instead. `.oxyignore` files exclude paths with gitignore syntax, the
5
//! nearest file deciding; `.gitignore` and `.ignore` files count too, below
6
//! `.oxyignore` in the same directory (oxygen#44), so a project coming from
7
//! Git does not commit what Git ignored, and `.oxyignore` can still say
8
//! otherwise with `!`.
9
//!
10
//! Each directory is read once; a file's stat comes from its directory
11
//! entry, and results are gathered a directory at a time, so a tree of
12
//! 100 000 files costs about what its system calls cost.
13
14
use super::*;
15
use ignore::gitignore::{Gitignore, GitignoreBuilder};
16
use ignore::Match;
17
use std::sync::{Arc, Mutex};
18
19
/// What a stat says about a file that the working copy needs: its size,
20
/// modification time and executable bit.
21
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
22
pub(crate) struct Stat {
23
    pub size: u64,
24
    pub mtime_ns: u128,
25
    pub exec: bool,
26
}
27
28
impl Stat {
2917162×
    pub(crate) fn of(m: &fs::Metadata) -> Stat {
3017162×
        Stat {
3117162×
            size: m.len(),
3217162×
            mtime_ns: super::prompt::mtime_ns(m),
3317162×
            exec: super::is_executable(m),
3417162×
        }
3517162×
    }
36
}
37
38
/// What a listing found, each list in path order.
39
#[derive(Default)]
40
pub(crate) struct Listing {
41
    /// Regular files: path relative to the root (with `/`), full path, stat.
42
    pub files: Vec<(String, PathBuf, Stat)>,
43
    /// Symbolic links and their targets.
44
    pub symlinks: Vec<(String, Vec<u8>)>,
45
    /// Directories holding a repository of their own.
46
    pub nested: Vec<PathBuf>,
47
    /// Every directory listed (the root as ""), with its modification time
48
    /// read before its entries were: a later change to its entries moves
49
    /// the time, so a manifest that keeps these can skip reading a
50
    /// directory whose time has not moved.
51
    pub dirs: Vec<(String, u128)>,
52
}
53
54
/// The ignore files in force for a directory, innermost first.
55
#[derive(Clone, Default)]
56
struct Rules(Option<Arc<(Gitignore, Rules)>>);
57
58
impl Rules {
59
    /// The rules for `dir`, adding its own ignore files if it has any.
603043×
    fn enter(&self, dir: &Path, names: &[String]) -> Rules {
61
        // Later files win: .oxyignore over .ignore over .gitignore.
623043×
        let files: Vec<&str> = [".gitignore", ".ignore", ".oxyignore"]
633043×
            .into_iter()
6460977×
            .filter(|f| names.iter().any(|n| n == f))
653043×
            .collect();
663043×
        if files.is_empty() {
673004×
            return self.clone();
6839×
        }
6939×
        let mut b = GitignoreBuilder::new(dir);
7041×
        for f in files {
7141×
            // An unreadable or malformed ignore file excludes nothing.
7241×
            let _ = b.add(dir.join(f));
7341×
        }
7439×
        match b.build() {
7539×
            Ok(g) if !g.is_empty() => Rules(Some(Arc::new((g, self.clone())))),
760×
            _ => self.clone(),
77
        }
783043×
    }
79
804272×
    fn ignored(&self, path: &Path, is_dir: bool) -> bool {
814272×
        let mut at = &self.0;
824370×
        while let Some(node) = at {
83164×
            match node.0.matched(path, is_dir) {
8460×
                Match::Ignore(_) => return true,
856×
                Match::Whitelist(_) => return false,
8698×
                Match::None => {}
87
            }
8898×
            at = &(node.1).0;
89
        }
904206×
        false
914272×
    }
92
}
93
94
struct Ctx<'a> {
95
    in_scope: &'a (dyn Fn(&str) -> bool + Sync),
96
    known: &'a (dyn Fn(&str) -> bool + Sync),
97
    out: Mutex<Listing>,
98
    failed: Mutex<Option<String>>,
99
}
100
101
/// List the working copy under `root`; files outside `in_scope` (a sparse
102
/// checkout's paths) are left out, though every directory is visited.
1034×
pub(crate) fn list(root: &Path, in_scope: &(dyn Fn(&str) -> bool + Sync)) -> Result<Listing> {
1044×
    list_knowing(root, in_scope, &|_| false)
1054×
}
106
107
/// [`list`], taking files `known` says are tracked without asking the
108
/// ignore rules, which apply to untracked paths only: on the Linux tree,
109
/// matching every file against its 400 `.gitignore` files cost more than
110
/// a manifest lookup.
1112176×
pub(crate) fn list_knowing(
1122176×
    root: &Path,
1132176×
    in_scope: &(dyn Fn(&str) -> bool + Sync),
1142176×
    known: &(dyn Fn(&str) -> bool + Sync),
1152176×
) -> Result<Listing> {
1162176×
    let ctx = Ctx {
1172176×
        in_scope,
1182176×
        known,
1192176×
        out: Mutex::new(Listing::default()),
1202176×
        failed: Mutex::new(None),
1212176×
    };
1222176×
    let root_time = fs::metadata(root)
1232176×
        .map(|m| super::prompt::mtime_ns(&m))
1242176×
        .unwrap_or(0);
1252176×
    rayon::scope(|s| {
1262176×
        visit(
1272176×
            s,
1282176×
            root.to_path_buf(),
1292176×
            String::new(),
1302176×
            root_time,
131
            true,
1322176×
            Rules::default(),
1332176×
            &ctx,
134
        )
1352176×
    });
1362176×
    if let Some(e) = ctx.failed.into_inner().unwrap_or_else(|e| e.into_inner()) {
1370×
        return fail(e);
1382176×
    }
1392176×
    let mut out = ctx.out.into_inner().unwrap_or_else(|e| e.into_inner());
140
    // In path order, as the maps built from it want.
141
    {
142
        use rayon::prelude::*;
14358939×
        out.files.par_sort_unstable_by(|a, b| a.0.cmp(&b.0));
1442176×
        out.symlinks.sort_unstable_by(|a, b| a.0.cmp(&b.0));
1452176×
        out.dirs.sort_unstable_by(|a, b| a.0.cmp(&b.0));
1462176×
        out.nested.sort_unstable();
147
    }
1482176×
    Ok(out)
1492176×
}
150
151
#[allow(clippy::too_many_arguments)]
1523110×
fn visit<'s>(
1533110×
    s: &rayon::Scope<'s>,
1543110×
    dir: PathBuf,
1553110×
    rel: String,
1563110×
    mtime: u128,
1573110×
    top: bool,
1583110×
    rules: Rules,
1593110×
    ctx: &'s Ctx<'s>,
1603110×
) {
1613110×
    let read = || -> std::io::Result<Vec<fs::DirEntry>> { fs::read_dir(&dir)?.collect() };
1623110×
    let entries = match read() {
1633110×
        Ok(e) => e,
1640×
        Err(e) => {
1650×
            ctx.failed
1660×
                .lock()
1670×
                .unwrap_or_else(|e| e.into_inner())
1680×
                .get_or_insert_with(|| format!("{}: {e}", dir.display()));
1690×
            return;
170
        }
171
    };
1723110×
    let names: Vec<String> = entries
1733110×
        .iter()
17420495×
        .map(|e| e.file_name().to_string_lossy().into_owned())
1753110×
        .collect();
17613574×
    if !top && names.iter().any(|n| n == OXY_DIR || n == ".git") {
17767×
        ctx.out
17867×
            .lock()
17967×
            .unwrap_or_else(|e| e.into_inner())
180
            .nested
18167×
            .push(dir);
18267×
        return;
1833043×
    }
1843043×
    let rules = rules.enter(&dir, &names);
1853043×
    let mut mine = Listing::default();
1863043×
    mine.dirs.push((rel.clone(), mtime));
18720357×
    for (e, name) in entries.into_iter().zip(names) {
188
        // `.git` is never content: a side-by-side Git repository
189
        // (`git.colocate`), or Git's own beside an import.
19020357×
        if name == OXY_DIR || name.eq_ignore_ascii_case(".git") {
1912179×
            continue;
19218178×
        }
19318178×
        let Ok(ft) = e.file_type() else { continue };
19418178×
        let path = e.path();
19518178×
        let child = if rel.is_empty() {
1964739×
            name
197
        } else {
19813439×
            format!("{rel}/{name}")
199
        };
20018178×
        if (ft.is_dir() || !(ctx.known)(&child)) && rules.ignored(&path, ft.is_dir()) {
20160×
            continue;
20218118×
        }
20318118×
        if ft.is_dir() {
204934×
            let rules = rules.clone();
205
            // The directory's time now, before it is read.
206934×
            let time = e
207934×
                .metadata()
208934×
                .map(|m| super::prompt::mtime_ns(&m))
209934×
                .unwrap_or(0);
210934×
            s.spawn(move |s| visit(s, path, child, time, false, rules, ctx));
21117184×
        } else if !(ctx.in_scope)(&child) {
21217165×
        } else if ft.is_symlink() {
2133×
            match fs::read_link(&path) {
2143×
                Ok(t) => mine
2153×
                    .symlinks
2163×
                    .push((child, t.to_string_lossy().as_bytes().to_vec())),
2170×
                Err(e) => {
2180×
                    ctx.failed
2190×
                        .lock()
2200×
                        .unwrap_or_else(|e| e.into_inner())
2210×
                        .get_or_insert_with(|| format!("{}: {e}", path.display()));
222
                }
223
            }
22417162×
        } else if ft.is_file() {
22517162×
            match e.metadata() {
22617162×
                Ok(m) => mine.files.push((child, path, Stat::of(&m))),
2270×
                Err(e) => {
2280×
                    ctx.failed
2290×
                        .lock()
2300×
                        .unwrap_or_else(|e| e.into_inner())
2310×
                        .get_or_insert_with(|| format!("{}: {e}", path.display()));
232
                }
233
            }
2340×
        }
235
    }
2363043×
    let mut out = ctx.out.lock().unwrap_or_else(|e| e.into_inner());
2373043×
    out.files.append(&mut mine.files);
2383043×
    out.symlinks.append(&mut mine.symlinks);
2393043×
    out.dirs.append(&mut mine.dirs);
2403110×
}
241
242
#[cfg(test)]
243
mod tests {
244
    use super::*;
245
24622×
    fn write(root: &Path, rel: &str, text: &str) {
24722×
        let p = root.join(rel);
24822×
        fs::create_dir_all(p.parent().unwrap()).unwrap();
24922×
        fs::write(p, text).unwrap();
25022×
    }
251
2522×
    fn listed(root: &Path, in_scope: &(dyn Fn(&str) -> bool + Sync)) -> (Vec<String>, Vec<String>) {
2532×
        let l = list(root, in_scope).unwrap();
2542×
        let mut files: Vec<String> = l.files.into_iter().map(|(r, _, _)| r).collect();
2552×
        files.extend(l.symlinks.into_iter().map(|(r, _)| r));
2562×
        files.sort();
2572×
        let mut nested: Vec<String> = l
2582×
            .nested
2592×
            .iter()
2604×
            .map(|d| {
2614×
                d.strip_prefix(root)
2624×
                    .unwrap()
2634×
                    .to_string_lossy()
2644×
                    .replace('\\', "/")
2654×
            })
2662×
            .collect();
2672×
        nested.sort();
2682×
        (files, nested)
2692×
    }
270
271
    #[test]
2721×
    fn ignore_files_nested_repositories_and_scope() {
2731×
        let dir = tempfile::tempdir().unwrap();
2741×
        let root = dir.path();
2751×
        write(root, ".oxy/config", "");
2761×
        write(root, ".oxyignore", "*.log\nbuild/\n");
2771×
        write(root, ".hidden", "");
2781×
        write(root, "a.txt", "");
2791×
        write(root, "a.log", "");
2801×
        write(root, "build/out.o", "");
2811×
        write(root, "src/main.c", "");
2821×
        write(root, "src/deep/x.log", "");
283
        // A nearer file overrides an outer rule, and `.ignore` counts too.
2841×
        write(root, "keep/.oxyignore", "!*.log\n");
2851×
        write(root, "keep/k.log", "");
2861×
        write(root, "other/.ignore", "secret\n");
2871×
        write(root, "other/secret", "");
2881×
        write(root, "other/public", "");
289
        // .gitignore counts, below .oxyignore in the same directory.
2901×
        write(root, "gitproj/.gitignore", "build/\n*.tmp\nkeep.tmp\n");
2911×
        write(root, "gitproj/.oxyignore", "!keep.tmp\n");
2921×
        write(root, "gitproj/build/out.bin", "");
2931×
        write(root, "gitproj/x.tmp", "");
2941×
        write(root, "gitproj/keep.tmp", "");
2951×
        write(root, "gitproj/src.c", "");
296
        // Nested repositories are reported, not entered.
2971×
        write(root, "sub/.oxy/config", "");
2981×
        write(root, "sub/inner.txt", "");
2991×
        write(root, "vendor/g/.git/HEAD", "");
300
        #[cfg(unix)]
3011×
        std::os::unix::fs::symlink("a.txt", root.join("link")).unwrap();
302
3031×
        let (files, nested) = listed(root, &|_| true);
3041×
        let mut want = vec![
305
            ".hidden",
3061×
            ".oxyignore",
3071×
            "a.txt",
3081×
            "gitproj/.gitignore",
3091×
            "gitproj/.oxyignore",
3101×
            "gitproj/keep.tmp",
3111×
            "gitproj/src.c",
3121×
            "keep/.oxyignore",
3131×
            "keep/k.log",
3141×
            "other/.ignore",
3151×
            "other/public",
3161×
            "src/main.c",
317
        ];
3181×
        if cfg!(unix) {
3191×
            want.push("link");
3201×
        }
3211×
        want.sort();
3221×
        assert_eq!(files, want);
3231×
        assert_eq!(nested, vec!["sub", "vendor/g"]);
324
325
        // A sparse scope leaves files out but still visits directories.
32613×
        let (files, _) = listed(root, &|p: &str| p.starts_with("src/"));
3271×
        assert_eq!(files, vec!["src/main.c"]);
3281×
    }
329
}