OXYGENHUB
# 36e51aa806c6 oxygen / crates / oxy-core / src / repo / walk.rs
8.5 KiB · Rust · last changed Status: maps built in path order, fast maps for node lookups (refs #41) 1 month ago source blame history raw
1
//! Listing the working copy (spec §12.3): every file and symlink under the
2
//! root, found in parallel. `.oxy` is skipped, and so is a nested
3
//! repository (a directory holding `.oxy` or `.git`), which is reported
4
//! instead. `.oxyignore` files (and `.ignore` files, as before) exclude
5
//! paths with gitignore syntax, the nearest file deciding.
6
//!
7
//! Each directory is read once; a file's stat comes from its directory
8
//! entry, and results are gathered a directory at a time, so a tree of
9
//! 100 000 files costs about what its system calls cost.
10
11
use super::*;
12
use ignore::gitignore::{Gitignore, GitignoreBuilder};
13
use ignore::Match;
14
use std::sync::{Arc, Mutex};
15
16
/// What a listing found, each list in path order.
17
#[derive(Default)]
18
pub(crate) struct Listing {
19
    /// Regular files: path relative to the root (with `/`), full path, stat.
20
    pub files: Vec<(String, PathBuf, fs::Metadata)>,
21
    /// Symbolic links and their targets.
22
    pub symlinks: Vec<(String, Vec<u8>)>,
23
    /// Directories holding a repository of their own.
24
    pub nested: Vec<PathBuf>,
25
}
26
27
/// The ignore files in force for a directory, innermost first.
28
#[derive(Clone, Default)]
29
struct Rules(Option<Arc<(Gitignore, Rules)>>);
30
31
impl Rules {
32
    /// The rules for `dir`, adding its own ignore files if it has any.
33
    fn enter(&self, dir: &Path, names: &[String]) -> Rules {
34
        let files: Vec<&str> = [".ignore", ".oxyignore"]
35
            .into_iter()
36
            .filter(|f| names.iter().any(|n| n == f))
37
            .collect();
38
        if files.is_empty() {
39
            return self.clone();
40
        }
41
        let mut b = GitignoreBuilder::new(dir);
42
        for f in files {
43
            // An unreadable or malformed ignore file excludes nothing.
44
            let _ = b.add(dir.join(f));
45
        }
46
        match b.build() {
47
            Ok(g) if !g.is_empty() => Rules(Some(Arc::new((g, self.clone())))),
48
            _ => self.clone(),
49
        }
50
    }
51
52
    fn ignored(&self, path: &Path, is_dir: bool) -> bool {
53
        let mut at = &self.0;
54
        while let Some(node) = at {
55
            match node.0.matched(path, is_dir) {
56
                Match::Ignore(_) => return true,
57
                Match::Whitelist(_) => return false,
58
                Match::None => {}
59
            }
60
            at = &(node.1).0;
61
        }
62
        false
63
    }
64
}
65
66
struct Ctx<'a> {
67
    in_scope: &'a (dyn Fn(&str) -> bool + Sync),
68
    out: Mutex<Listing>,
69
    failed: Mutex<Option<String>>,
70
}
71
72
/// List the working copy under `root`; files outside `in_scope` (a sparse
73
/// checkout's paths) are left out, though every directory is visited.
74
pub(crate) fn list(root: &Path, in_scope: &(dyn Fn(&str) -> bool + Sync)) -> Result<Listing> {
75
    let ctx = Ctx {
76
        in_scope,
77
        out: Mutex::new(Listing::default()),
78
        failed: Mutex::new(None),
79
    };
80
    rayon::scope(|s| {
81
        visit(
82
            s,
83
            root.to_path_buf(),
84
            String::new(),
85
            true,
86
            Rules::default(),
87
            &ctx,
88
        )
89
    });
90
    if let Some(e) = ctx.failed.into_inner().unwrap_or_else(|e| e.into_inner()) {
91
        return fail(e);
92
    }
93
    let mut out = ctx.out.into_inner().unwrap_or_else(|e| e.into_inner());
94
    // In path order, as the maps built from it want.
95
    {
96
        use rayon::prelude::*;
97
        out.files.par_sort_unstable_by(|a, b| a.0.cmp(&b.0));
98
        out.symlinks.sort_unstable_by(|a, b| a.0.cmp(&b.0));
99
    }
100
    Ok(out)
101
}
102
103
fn visit<'s>(
104
    s: &rayon::Scope<'s>,
105
    dir: PathBuf,
106
    rel: String,
107
    top: bool,
108
    rules: Rules,
109
    ctx: &'s Ctx<'s>,
110
) {
111
    let read = || -> std::io::Result<Vec<fs::DirEntry>> { fs::read_dir(&dir)?.collect() };
112
    let entries = match read() {
113
        Ok(e) => e,
114
        Err(e) => {
115
            ctx.failed
116
                .lock()
117
                .unwrap_or_else(|e| e.into_inner())
118
                .get_or_insert_with(|| format!("{}: {e}", dir.display()));
119
            return;
120
        }
121
    };
122
    let names: Vec<String> = entries
123
        .iter()
124
        .map(|e| e.file_name().to_string_lossy().into_owned())
125
        .collect();
126
    if !top && names.iter().any(|n| n == OXY_DIR || n == ".git") {
127
        ctx.out
128
            .lock()
129
            .unwrap_or_else(|e| e.into_inner())
130
            .nested
131
            .push(dir);
132
        return;
133
    }
134
    let rules = rules.enter(&dir, &names);
135
    let mut mine = Listing::default();
136
    for (e, name) in entries.into_iter().zip(names) {
137
        if name == OXY_DIR {
138
            continue;
139
        }
140
        let Ok(ft) = e.file_type() else { continue };
141
        let path = e.path();
142
        if rules.ignored(&path, ft.is_dir()) {
143
            continue;
144
        }
145
        let child = if rel.is_empty() {
146
            name
147
        } else {
148
            format!("{rel}/{name}")
149
        };
150
        if ft.is_dir() {
151
            let rules = rules.clone();
152
            s.spawn(move |s| visit(s, path, child, false, rules, ctx));
153
        } else if !(ctx.in_scope)(&child) {
154
        } else if ft.is_symlink() {
155
            match fs::read_link(&path) {
156
                Ok(t) => mine
157
                    .symlinks
158
                    .push((child, t.to_string_lossy().as_bytes().to_vec())),
159
                Err(e) => {
160
                    ctx.failed
161
                        .lock()
162
                        .unwrap_or_else(|e| e.into_inner())
163
                        .get_or_insert_with(|| format!("{}: {e}", path.display()));
164
                }
165
            }
166
        } else if ft.is_file() {
167
            match e.metadata() {
168
                Ok(m) => mine.files.push((child, path, m)),
169
                Err(e) => {
170
                    ctx.failed
171
                        .lock()
172
                        .unwrap_or_else(|e| e.into_inner())
173
                        .get_or_insert_with(|| format!("{}: {e}", path.display()));
174
                }
175
            }
176
        }
177
    }
178
    if !mine.files.is_empty() || !mine.symlinks.is_empty() {
179
        let mut out = ctx.out.lock().unwrap_or_else(|e| e.into_inner());
180
        out.files.append(&mut mine.files);
181
        out.symlinks.append(&mut mine.symlinks);
182
    }
183
}
184
185
#[cfg(test)]
186
mod tests {
187
    use super::*;
188
189
    fn write(root: &Path, rel: &str, text: &str) {
190
        let p = root.join(rel);
191
        fs::create_dir_all(p.parent().unwrap()).unwrap();
192
        fs::write(p, text).unwrap();
193
    }
194
195
    fn listed(root: &Path, in_scope: &(dyn Fn(&str) -> bool + Sync)) -> (Vec<String>, Vec<String>) {
196
        let l = list(root, in_scope).unwrap();
197
        let mut files: Vec<String> = l.files.into_iter().map(|(r, _, _)| r).collect();
198
        files.extend(l.symlinks.into_iter().map(|(r, _)| r));
199
        files.sort();
200
        let mut nested: Vec<String> = l
201
            .nested
202
            .iter()
203
            .map(|d| {
204
                d.strip_prefix(root)
205
                    .unwrap()
206
                    .to_string_lossy()
207
                    .replace('\\', "/")
208
            })
209
            .collect();
210
        nested.sort();
211
        (files, nested)
212
    }
213
214
    #[test]
215
    fn ignore_files_nested_repositories_and_scope() {
216
        let dir = tempfile::tempdir().unwrap();
217
        let root = dir.path();
218
        write(root, ".oxy/config", "");
219
        write(root, ".oxyignore", "*.log\nbuild/\n");
220
        write(root, ".hidden", "");
221
        write(root, "a.txt", "");
222
        write(root, "a.log", "");
223
        write(root, "build/out.o", "");
224
        write(root, "src/main.c", "");
225
        write(root, "src/deep/x.log", "");
226
        // A nearer file overrides an outer rule, and `.ignore` counts too.
227
        write(root, "keep/.oxyignore", "!*.log\n");
228
        write(root, "keep/k.log", "");
229
        write(root, "other/.ignore", "secret\n");
230
        write(root, "other/secret", "");
231
        write(root, "other/public", "");
232
        // Nested repositories are reported, not entered.
233
        write(root, "sub/.oxy/config", "");
234
        write(root, "sub/inner.txt", "");
235
        write(root, "vendor/g/.git/HEAD", "");
236
        #[cfg(unix)]
237
        std::os::unix::fs::symlink("a.txt", root.join("link")).unwrap();
238
239
        let (files, nested) = listed(root, &|_| true);
240
        let mut want = vec![
241
            ".hidden",
242
            ".oxyignore",
243
            "a.txt",
244
            "keep/.oxyignore",
245
            "keep/k.log",
246
            "other/.ignore",
247
            "other/public",
248
            "src/main.c",
249
        ];
250
        if cfg!(unix) {
251
            want.push("link");
252
        }
253
        want.sort();
254
        assert_eq!(files, want);
255
        assert_eq!(nested, vec!["sub", "vendor/g"]);
256
257
        // A sparse scope leaves files out but still visits directories.
258
        let (files, _) = listed(root, &|p: &str| p.starts_with("src/"));
259
        assert_eq!(files, vec!["src/main.c"]);
260
    }
261
}