OXYGENHUB
# c3d2ef6104c2 oxygen / crates / oxy-core / src / repo / walk.rs
9.5 KiB · Rust · last changed Projects coming from Git: .gitignore honoured, nested Git repositories named (closes #43, closes #44) 1 month ago source blame history raw
76740263 Andy McCall ↶ 1
//! Listing the working copy (spec §12.3): every file and symlink under the
2
//! root, found in parallel. `.oxy` is skipped, and so is a nested
3
//! repository (a directory holding `.oxy` or `.git`), which is reported
d4dbab97 Andy McCall ↶ 4
//! instead. `.oxyignore` files exclude paths with gitignore syntax, the
5
//! nearest file deciding; `.gitignore` and `.ignore` files count too, below
6
//! `.oxyignore` in the same directory (oxygen#44), so a project coming from
7
//! Git does not commit what Git ignored, and `.oxyignore` can still say
8
//! otherwise with `!`.
76740263 Andy McCall ↶ 9
//!
10
//! Each directory is read once; a file's stat comes from its directory
11
//! entry, and results are gathered a directory at a time, so a tree of
12
//! 100 000 files costs about what its system calls cost.
13
14
use super::*;
15
use ignore::gitignore::{Gitignore, GitignoreBuilder};
16
use ignore::Match;
17
use std::sync::{Arc, Mutex};
18
19
/// What a listing found.
20
#[derive(Default)]
21
pub(crate) struct Listing {
22
    /// Regular files: path relative to the root (with `/`), full path, stat.
23
    pub files: Vec<(String, PathBuf, fs::Metadata)>,
24
    /// Symbolic links and their targets.
25
    pub symlinks: Vec<(String, Vec<u8>)>,
26
    /// Directories holding a repository of their own.
27
    pub nested: Vec<PathBuf>,
28
}
29
30
/// The ignore files in force for a directory, innermost first.
31
#[derive(Clone, Default)]
32
struct Rules(Option<Arc<(Gitignore, Rules)>>);
33
34
impl Rules {
35
    /// The rules for `dir`, adding its own ignore files if it has any.
36
    fn enter(&self, dir: &Path, names: &[String]) -> Rules {
d4dbab97 Andy McCall ↶ 37
        // Later files win: .oxyignore over .ignore over .gitignore.
38
        let files: Vec<&str> = [".gitignore", ".ignore", ".oxyignore"]
76740263 Andy McCall ↶ 39
            .into_iter()
40
            .filter(|f| names.iter().any(|n| n == f))
41
            .collect();
42
        if files.is_empty() {
43
            return self.clone();
44
        }
45
        let mut b = GitignoreBuilder::new(dir);
46
        for f in files {
47
            // An unreadable or malformed ignore file excludes nothing.
48
            let _ = b.add(dir.join(f));
49
        }
50
        match b.build() {
51
            Ok(g) if !g.is_empty() => Rules(Some(Arc::new((g, self.clone())))),
52
            _ => self.clone(),
53
        }
54
    }
55
56
    fn ignored(&self, path: &Path, is_dir: bool) -> bool {
57
        let mut at = &self.0;
58
        while let Some(node) = at {
59
            match node.0.matched(path, is_dir) {
60
                Match::Ignore(_) => return true,
61
                Match::Whitelist(_) => return false,
62
                Match::None => {}
63
            }
64
            at = &(node.1).0;
65
        }
66
        false
67
    }
68
}
69
70
struct Ctx<'a> {
71
    in_scope: &'a (dyn Fn(&str) -> bool + Sync),
d4dbab97 Andy McCall ↶ 72
    known: &'a (dyn Fn(&str) -> bool + Sync),
76740263 Andy McCall ↶ 73
    out: Mutex<Listing>,
74
    failed: Mutex<Option<String>>,
75
}
76
77
/// List the working copy under `root`; files outside `in_scope` (a sparse
78
/// checkout's paths) are left out, though every directory is visited.
79
pub(crate) fn list(root: &Path, in_scope: &(dyn Fn(&str) -> bool + Sync)) -> Result<Listing> {
d4dbab97 Andy McCall ↶ 80
    list_knowing(root, in_scope, &|_| false)
81
}
82
83
/// [`list`], taking files `known` says are tracked without asking the
84
/// ignore rules, which apply to untracked paths only: on the Linux tree,
85
/// matching every file against its 400 `.gitignore` files cost more than
86
/// a manifest lookup.
87
pub(crate) fn list_knowing(
88
    root: &Path,
89
    in_scope: &(dyn Fn(&str) -> bool + Sync),
90
    known: &(dyn Fn(&str) -> bool + Sync),
91
) -> Result<Listing> {
76740263 Andy McCall ↶ 92
    let ctx = Ctx {
93
        in_scope,
d4dbab97 Andy McCall ↶ 94
        known,
76740263 Andy McCall ↶ 95
        out: Mutex::new(Listing::default()),
96
        failed: Mutex::new(None),
97
    };
98
    rayon::scope(|s| {
99
        visit(
100
            s,
101
            root.to_path_buf(),
102
            String::new(),
103
            true,
104
            Rules::default(),
105
            &ctx,
106
        )
107
    });
108
    if let Some(e) = ctx.failed.into_inner().unwrap_or_else(|e| e.into_inner()) {
109
        return fail(e);
110
    }
111
    Ok(ctx.out.into_inner().unwrap_or_else(|e| e.into_inner()))
112
}
113
114
fn visit<'s>(
115
    s: &rayon::Scope<'s>,
116
    dir: PathBuf,
117
    rel: String,
118
    top: bool,
119
    rules: Rules,
120
    ctx: &'s Ctx<'s>,
121
) {
122
    let read = || -> std::io::Result<Vec<fs::DirEntry>> { fs::read_dir(&dir)?.collect() };
123
    let entries = match read() {
124
        Ok(e) => e,
125
        Err(e) => {
126
            ctx.failed
127
                .lock()
128
                .unwrap_or_else(|e| e.into_inner())
129
                .get_or_insert_with(|| format!("{}: {e}", dir.display()));
130
            return;
131
        }
132
    };
133
    let names: Vec<String> = entries
134
        .iter()
135
        .map(|e| e.file_name().to_string_lossy().into_owned())
136
        .collect();
137
    if !top && names.iter().any(|n| n == OXY_DIR || n == ".git") {
138
        ctx.out
139
            .lock()
140
            .unwrap_or_else(|e| e.into_inner())
141
            .nested
142
            .push(dir);
143
        return;
144
    }
145
    let rules = rules.enter(&dir, &names);
146
    let mut mine = Listing::default();
147
    for (e, name) in entries.into_iter().zip(names) {
148
        if name == OXY_DIR {
149
            continue;
150
        }
151
        let Ok(ft) = e.file_type() else { continue };
152
        let path = e.path();
153
        let child = if rel.is_empty() {
154
            name
155
        } else {
156
            format!("{rel}/{name}")
157
        };
d4dbab97 Andy McCall ↶ 158
        if (ft.is_dir() || !(ctx.known)(&child)) && rules.ignored(&path, ft.is_dir()) {
159
            continue;
160
        }
76740263 Andy McCall ↶ 161
        if ft.is_dir() {
162
            let rules = rules.clone();
163
            s.spawn(move |s| visit(s, path, child, false, rules, ctx));
164
        } else if !(ctx.in_scope)(&child) {
165
        } else if ft.is_symlink() {
166
            match fs::read_link(&path) {
167
                Ok(t) => mine
168
                    .symlinks
169
                    .push((child, t.to_string_lossy().as_bytes().to_vec())),
170
                Err(e) => {
171
                    ctx.failed
172
                        .lock()
173
                        .unwrap_or_else(|e| e.into_inner())
174
                        .get_or_insert_with(|| format!("{}: {e}", path.display()));
175
                }
176
            }
177
        } else if ft.is_file() {
178
            match e.metadata() {
179
                Ok(m) => mine.files.push((child, path, m)),
180
                Err(e) => {
181
                    ctx.failed
182
                        .lock()
183
                        .unwrap_or_else(|e| e.into_inner())
184
                        .get_or_insert_with(|| format!("{}: {e}", path.display()));
185
                }
186
            }
187
        }
188
    }
189
    if !mine.files.is_empty() || !mine.symlinks.is_empty() {
190
        let mut out = ctx.out.lock().unwrap_or_else(|e| e.into_inner());
191
        out.files.append(&mut mine.files);
192
        out.symlinks.append(&mut mine.symlinks);
193
    }
194
}
195
196
#[cfg(test)]
197
mod tests {
198
    use super::*;
199
200
    fn write(root: &Path, rel: &str, text: &str) {
201
        let p = root.join(rel);
202
        fs::create_dir_all(p.parent().unwrap()).unwrap();
203
        fs::write(p, text).unwrap();
204
    }
205
206
    fn listed(root: &Path, in_scope: &(dyn Fn(&str) -> bool + Sync)) -> (Vec<String>, Vec<String>) {
207
        let l = list(root, in_scope).unwrap();
208
        let mut files: Vec<String> = l.files.into_iter().map(|(r, _, _)| r).collect();
209
        files.extend(l.symlinks.into_iter().map(|(r, _)| r));
210
        files.sort();
211
        let mut nested: Vec<String> = l
212
            .nested
213
            .iter()
214
            .map(|d| {
215
                d.strip_prefix(root)
216
                    .unwrap()
217
                    .to_string_lossy()
218
                    .replace('\\', "/")
219
            })
220
            .collect();
221
        nested.sort();
222
        (files, nested)
223
    }
224
225
    #[test]
226
    fn ignore_files_nested_repositories_and_scope() {
227
        let dir = tempfile::tempdir().unwrap();
228
        let root = dir.path();
229
        write(root, ".oxy/config", "");
230
        write(root, ".oxyignore", "*.log\nbuild/\n");
231
        write(root, ".hidden", "");
232
        write(root, "a.txt", "");
233
        write(root, "a.log", "");
234
        write(root, "build/out.o", "");
235
        write(root, "src/main.c", "");
236
        write(root, "src/deep/x.log", "");
237
        // A nearer file overrides an outer rule, and `.ignore` counts too.
238
        write(root, "keep/.oxyignore", "!*.log\n");
239
        write(root, "keep/k.log", "");
240
        write(root, "other/.ignore", "secret\n");
241
        write(root, "other/secret", "");
242
        write(root, "other/public", "");
d4dbab97 Andy McCall ↶ 243
        // .gitignore counts, below .oxyignore in the same directory.
244
        write(root, "gitproj/.gitignore", "build/\n*.tmp\nkeep.tmp\n");
245
        write(root, "gitproj/.oxyignore", "!keep.tmp\n");
246
        write(root, "gitproj/build/out.bin", "");
247
        write(root, "gitproj/x.tmp", "");
248
        write(root, "gitproj/keep.tmp", "");
249
        write(root, "gitproj/src.c", "");
76740263 Andy McCall ↶ 250
        // Nested repositories are reported, not entered.
251
        write(root, "sub/.oxy/config", "");
252
        write(root, "sub/inner.txt", "");
253
        write(root, "vendor/g/.git/HEAD", "");
254
        #[cfg(unix)]
255
        std::os::unix::fs::symlink("a.txt", root.join("link")).unwrap();
256
257
        let (files, nested) = listed(root, &|_| true);
258
        let mut want = vec![
259
            ".hidden",
260
            ".oxyignore",
261
            "a.txt",
d4dbab97 Andy McCall ↶ 262
            "gitproj/.gitignore",
263
            "gitproj/.oxyignore",
264
            "gitproj/keep.tmp",
265
            "gitproj/src.c",
76740263 Andy McCall ↶ 266
            "keep/.oxyignore",
267
            "keep/k.log",
268
            "other/.ignore",
269
            "other/public",
270
            "src/main.c",
271
        ];
272
        if cfg!(unix) {
273
            want.push("link");
274
        }
275
        want.sort();
276
        assert_eq!(files, want);
277
        assert_eq!(nested, vec!["sub", "vendor/g"]);
278
279
        // A sparse scope leaves files out but still visits directories.
280
        let (files, _) = listed(root, &|p: &str| p.starts_with("src/"));
281
        assert_eq!(files, vec!["src/main.c"]);
282
    }
283
}