OXYGENHUB
# 1ec3f008e96a oxygen / crates / oxy-core / src / repo / walk.rs
10.6 KiB · Rust · last changed The manifest keeps directory times: status lists the tree only when a directory changed (refs #41) 1 month ago source blame history raw
1
//! Listing the working copy (spec §12.3): every file and symlink under the
2
//! root, found in parallel. `.oxy` is skipped, and so is a nested
3
//! repository (a directory holding `.oxy` or `.git`), which is reported
4
//! instead. `.oxyignore` files exclude paths with gitignore syntax, the
5
//! nearest file deciding; `.gitignore` and `.ignore` files count too, below
6
//! `.oxyignore` in the same directory (oxygen#44), so a project coming from
7
//! Git does not commit what Git ignored, and `.oxyignore` can still say
8
//! otherwise with `!`.
9
//!
10
//! Each directory is read once; a file's stat comes from its directory
11
//! entry, and results are gathered a directory at a time, so a tree of
12
//! 100 000 files costs about what its system calls cost.
13
14
use super::*;
15
use ignore::gitignore::{Gitignore, GitignoreBuilder};
16
use ignore::Match;
17
use std::sync::{Arc, Mutex};
18
19
/// What a listing found, each list in path order.
20
#[derive(Default)]
21
pub(crate) struct Listing {
22
    /// Regular files: path relative to the root (with `/`), full path, stat.
23
    pub files: Vec<(String, PathBuf, fs::Metadata)>,
24
    /// Symbolic links and their targets.
25
    pub symlinks: Vec<(String, Vec<u8>)>,
26
    /// Directories holding a repository of their own.
27
    pub nested: Vec<PathBuf>,
28
    /// Every directory listed (the root as ""), with its modification time
29
    /// read before its entries were: a later change to its entries moves
30
    /// the time, so a manifest that keeps these can skip reading a
31
    /// directory whose time has not moved.
32
    pub dirs: Vec<(String, u128)>,
33
}
34
35
/// The ignore files in force for a directory, innermost first.
36
#[derive(Clone, Default)]
37
struct Rules(Option<Arc<(Gitignore, Rules)>>);
38
39
impl Rules {
40
    /// The rules for `dir`, adding its own ignore files if it has any.
41
    fn enter(&self, dir: &Path, names: &[String]) -> Rules {
42
        // Later files win: .oxyignore over .ignore over .gitignore.
43
        let files: Vec<&str> = [".gitignore", ".ignore", ".oxyignore"]
44
            .into_iter()
45
            .filter(|f| names.iter().any(|n| n == f))
46
            .collect();
47
        if files.is_empty() {
48
            return self.clone();
49
        }
50
        let mut b = GitignoreBuilder::new(dir);
51
        for f in files {
52
            // An unreadable or malformed ignore file excludes nothing.
53
            let _ = b.add(dir.join(f));
54
        }
55
        match b.build() {
56
            Ok(g) if !g.is_empty() => Rules(Some(Arc::new((g, self.clone())))),
57
            _ => self.clone(),
58
        }
59
    }
60
61
    fn ignored(&self, path: &Path, is_dir: bool) -> bool {
62
        let mut at = &self.0;
63
        while let Some(node) = at {
64
            match node.0.matched(path, is_dir) {
65
                Match::Ignore(_) => return true,
66
                Match::Whitelist(_) => return false,
67
                Match::None => {}
68
            }
69
            at = &(node.1).0;
70
        }
71
        false
72
    }
73
}
74
75
struct Ctx<'a> {
76
    in_scope: &'a (dyn Fn(&str) -> bool + Sync),
77
    known: &'a (dyn Fn(&str) -> bool + Sync),
78
    out: Mutex<Listing>,
79
    failed: Mutex<Option<String>>,
80
}
81
82
/// List the working copy under `root`; files outside `in_scope` (a sparse
83
/// checkout's paths) are left out, though every directory is visited.
84
pub(crate) fn list(root: &Path, in_scope: &(dyn Fn(&str) -> bool + Sync)) -> Result<Listing> {
85
    list_knowing(root, in_scope, &|_| false)
86
}
87
88
/// [`list`], taking files `known` says are tracked without asking the
89
/// ignore rules, which apply to untracked paths only: on the Linux tree,
90
/// matching every file against its 400 `.gitignore` files cost more than
91
/// a manifest lookup.
92
pub(crate) fn list_knowing(
93
    root: &Path,
94
    in_scope: &(dyn Fn(&str) -> bool + Sync),
95
    known: &(dyn Fn(&str) -> bool + Sync),
96
) -> Result<Listing> {
97
    let ctx = Ctx {
98
        in_scope,
99
        known,
100
        out: Mutex::new(Listing::default()),
101
        failed: Mutex::new(None),
102
    };
103
    let root_time = fs::metadata(root)
104
        .map(|m| super::prompt::mtime_ns(&m))
105
        .unwrap_or(0);
106
    rayon::scope(|s| {
107
        visit(
108
            s,
109
            root.to_path_buf(),
110
            String::new(),
111
            root_time,
112
            true,
113
            Rules::default(),
114
            &ctx,
115
        )
116
    });
117
    if let Some(e) = ctx.failed.into_inner().unwrap_or_else(|e| e.into_inner()) {
118
        return fail(e);
119
    }
120
    let mut out = ctx.out.into_inner().unwrap_or_else(|e| e.into_inner());
121
    // In path order, as the maps built from it want.
122
    {
123
        use rayon::prelude::*;
124
        out.files.par_sort_unstable_by(|a, b| a.0.cmp(&b.0));
125
        out.symlinks.sort_unstable_by(|a, b| a.0.cmp(&b.0));
126
        out.dirs.sort_unstable_by(|a, b| a.0.cmp(&b.0));
127
        out.nested.sort_unstable();
128
    }
129
    Ok(out)
130
}
131
132
#[allow(clippy::too_many_arguments)]
133
fn visit<'s>(
134
    s: &rayon::Scope<'s>,
135
    dir: PathBuf,
136
    rel: String,
137
    mtime: u128,
138
    top: bool,
139
    rules: Rules,
140
    ctx: &'s Ctx<'s>,
141
) {
142
    let read = || -> std::io::Result<Vec<fs::DirEntry>> { fs::read_dir(&dir)?.collect() };
143
    let entries = match read() {
144
        Ok(e) => e,
145
        Err(e) => {
146
            ctx.failed
147
                .lock()
148
                .unwrap_or_else(|e| e.into_inner())
149
                .get_or_insert_with(|| format!("{}: {e}", dir.display()));
150
            return;
151
        }
152
    };
153
    let names: Vec<String> = entries
154
        .iter()
155
        .map(|e| e.file_name().to_string_lossy().into_owned())
156
        .collect();
157
    if !top && names.iter().any(|n| n == OXY_DIR || n == ".git") {
158
        ctx.out
159
            .lock()
160
            .unwrap_or_else(|e| e.into_inner())
161
            .nested
162
            .push(dir);
163
        return;
164
    }
165
    let rules = rules.enter(&dir, &names);
166
    let mut mine = Listing::default();
167
    mine.dirs.push((rel.clone(), mtime));
168
    for (e, name) in entries.into_iter().zip(names) {
169
        if name == OXY_DIR {
170
            continue;
171
        }
172
        let Ok(ft) = e.file_type() else { continue };
173
        let path = e.path();
174
        let child = if rel.is_empty() {
175
            name
176
        } else {
177
            format!("{rel}/{name}")
178
        };
179
        if (ft.is_dir() || !(ctx.known)(&child)) && rules.ignored(&path, ft.is_dir()) {
180
            continue;
181
        }
182
        if ft.is_dir() {
183
            let rules = rules.clone();
184
            // The directory's time now, before it is read.
185
            let time = e
186
                .metadata()
187
                .map(|m| super::prompt::mtime_ns(&m))
188
                .unwrap_or(0);
189
            s.spawn(move |s| visit(s, path, child, time, false, rules, ctx));
190
        } else if !(ctx.in_scope)(&child) {
191
        } else if ft.is_symlink() {
192
            match fs::read_link(&path) {
193
                Ok(t) => mine
194
                    .symlinks
195
                    .push((child, t.to_string_lossy().as_bytes().to_vec())),
196
                Err(e) => {
197
                    ctx.failed
198
                        .lock()
199
                        .unwrap_or_else(|e| e.into_inner())
200
                        .get_or_insert_with(|| format!("{}: {e}", path.display()));
201
                }
202
            }
203
        } else if ft.is_file() {
204
            match e.metadata() {
205
                Ok(m) => mine.files.push((child, path, m)),
206
                Err(e) => {
207
                    ctx.failed
208
                        .lock()
209
                        .unwrap_or_else(|e| e.into_inner())
210
                        .get_or_insert_with(|| format!("{}: {e}", path.display()));
211
                }
212
            }
213
        }
214
    }
215
    let mut out = ctx.out.lock().unwrap_or_else(|e| e.into_inner());
216
    out.files.append(&mut mine.files);
217
    out.symlinks.append(&mut mine.symlinks);
218
    out.dirs.append(&mut mine.dirs);
219
}
220
221
#[cfg(test)]
222
mod tests {
223
    use super::*;
224
225
    fn write(root: &Path, rel: &str, text: &str) {
226
        let p = root.join(rel);
227
        fs::create_dir_all(p.parent().unwrap()).unwrap();
228
        fs::write(p, text).unwrap();
229
    }
230
231
    fn listed(root: &Path, in_scope: &(dyn Fn(&str) -> bool + Sync)) -> (Vec<String>, Vec<String>) {
232
        let l = list(root, in_scope).unwrap();
233
        let mut files: Vec<String> = l.files.into_iter().map(|(r, _, _)| r).collect();
234
        files.extend(l.symlinks.into_iter().map(|(r, _)| r));
235
        files.sort();
236
        let mut nested: Vec<String> = l
237
            .nested
238
            .iter()
239
            .map(|d| {
240
                d.strip_prefix(root)
241
                    .unwrap()
242
                    .to_string_lossy()
243
                    .replace('\\', "/")
244
            })
245
            .collect();
246
        nested.sort();
247
        (files, nested)
248
    }
249
250
    #[test]
251
    fn ignore_files_nested_repositories_and_scope() {
252
        let dir = tempfile::tempdir().unwrap();
253
        let root = dir.path();
254
        write(root, ".oxy/config", "");
255
        write(root, ".oxyignore", "*.log\nbuild/\n");
256
        write(root, ".hidden", "");
257
        write(root, "a.txt", "");
258
        write(root, "a.log", "");
259
        write(root, "build/out.o", "");
260
        write(root, "src/main.c", "");
261
        write(root, "src/deep/x.log", "");
262
        // A nearer file overrides an outer rule, and `.ignore` counts too.
263
        write(root, "keep/.oxyignore", "!*.log\n");
264
        write(root, "keep/k.log", "");
265
        write(root, "other/.ignore", "secret\n");
266
        write(root, "other/secret", "");
267
        write(root, "other/public", "");
268
        // .gitignore counts, below .oxyignore in the same directory.
269
        write(root, "gitproj/.gitignore", "build/\n*.tmp\nkeep.tmp\n");
270
        write(root, "gitproj/.oxyignore", "!keep.tmp\n");
271
        write(root, "gitproj/build/out.bin", "");
272
        write(root, "gitproj/x.tmp", "");
273
        write(root, "gitproj/keep.tmp", "");
274
        write(root, "gitproj/src.c", "");
275
        // Nested repositories are reported, not entered.
276
        write(root, "sub/.oxy/config", "");
277
        write(root, "sub/inner.txt", "");
278
        write(root, "vendor/g/.git/HEAD", "");
279
        #[cfg(unix)]
280
        std::os::unix::fs::symlink("a.txt", root.join("link")).unwrap();
281
282
        let (files, nested) = listed(root, &|_| true);
283
        let mut want = vec![
284
            ".hidden",
285
            ".oxyignore",
286
            "a.txt",
287
            "gitproj/.gitignore",
288
            "gitproj/.oxyignore",
289
            "gitproj/keep.tmp",
290
            "gitproj/src.c",
291
            "keep/.oxyignore",
292
            "keep/k.log",
293
            "other/.ignore",
294
            "other/public",
295
            "src/main.c",
296
        ];
297
        if cfg!(unix) {
298
            want.push("link");
299
        }
300
        want.sort();
301
        assert_eq!(files, want);
302
        assert_eq!(nested, vec!["sub", "vendor/g"]);
303
304
        // A sparse scope leaves files out but still visits directories.
305
        let (files, _) = listed(root, &|p: &str| p.starts_with("src/"));
306
        assert_eq!(files, vec!["src/main.c"]);
307
    }
308
}