Status: a lean parallel lister of the working copy, reused by the snapshot after a failed clean check (refs #41) 76740263089c

//! Listing the working copy (spec §12.3): every file and symlink under the

//! root, found in parallel. `.oxy` is skipped, and so is a nested

//! repository (a directory holding `.oxy` or `.git`), which is reported

//! instead. `.oxyignore` files (and `.ignore` files, as before) exclude

//! paths with gitignore syntax, the nearest file deciding.

//!

//! Each directory is read once; a file's stat comes from its directory

//! entry, and results are gathered a directory at a time, so a tree of

//! 100 000 files costs about what its system calls cost.

use super::*;

use ignore::gitignore::{Gitignore, GitignoreBuilder};

use ignore::Match;

use std::sync::{Arc, Mutex};

/// What a listing found.

#[derive(Default)]

pub(crate) struct Listing {

    /// Regular files: path relative to the root (with `/`), full path, stat.

    pub files: Vec<(String, PathBuf, fs::Metadata)>,

    /// Symbolic links and their targets.

    pub symlinks: Vec<(String, Vec<u8>)>,

    /// Directories holding a repository of their own.

    pub nested: Vec<PathBuf>,

}

/// The ignore files in force for a directory, innermost first.

#[derive(Clone, Default)]

struct Rules(Option<Arc<(Gitignore, Rules)>>);

impl Rules {

    /// The rules for `dir`, adding its own ignore files if it has any.

    fn enter(&self, dir: &Path, names: &[String]) -> Rules {

        let files: Vec<&str> = [".ignore", ".oxyignore"]

            .into_iter()

            .filter(|f| names.iter().any(|n| n == f))

            .collect();

        if files.is_empty() {

            return self.clone();

        }

        let mut b = GitignoreBuilder::new(dir);

        for f in files {

            // An unreadable or malformed ignore file excludes nothing.

            let _ = b.add(dir.join(f));

        }

        match b.build() {

            Ok(g) if !g.is_empty() => Rules(Some(Arc::new((g, self.clone())))),

            _ => self.clone(),

        }

    }

    fn ignored(&self, path: &Path, is_dir: bool) -> bool {

        let mut at = &self.0;

        while let Some(node) = at {

            match node.0.matched(path, is_dir) {

                Match::Ignore(_) => return true,

                Match::Whitelist(_) => return false,

                Match::None => {}

            }

            at = &(node.1).0;

        }

        false

    }

}

struct Ctx<'a> {

    in_scope: &'a (dyn Fn(&str) -> bool + Sync),

    out: Mutex<Listing>,

    failed: Mutex<Option<String>>,

}

/// List the working copy under `root`; files outside `in_scope` (a sparse

/// checkout's paths) are left out, though every directory is visited.

pub(crate) fn list(root: &Path, in_scope: &(dyn Fn(&str) -> bool + Sync)) -> Result<Listing> {

    let ctx = Ctx {

        in_scope,

        out: Mutex::new(Listing::default()),

        failed: Mutex::new(None),

    };

    rayon::scope(|s| {

        visit(

            s,

            root.to_path_buf(),

            String::new(),

            true,

            Rules::default(),

            &ctx,

        )

    });

    if let Some(e) = ctx.failed.into_inner().unwrap_or_else(|e| e.into_inner()) {

        return fail(e);

    }

    Ok(ctx.out.into_inner().unwrap_or_else(|e| e.into_inner()))

}

fn visit<'s>(

    s: &rayon::Scope<'s>,

    dir: PathBuf,

    rel: String,

    top: bool,

    rules: Rules,

    ctx: &'s Ctx<'s>,

) {

    let read = || -> std::io::Result<Vec<fs::DirEntry>> { fs::read_dir(&dir)?.collect() };

    let entries = match read() {

        Ok(e) => e,

        Err(e) => {

            ctx.failed

                .lock()

                .unwrap_or_else(|e| e.into_inner())

                .get_or_insert_with(|| format!("{}: {e}", dir.display()));

            return;

        }

    };

    let names: Vec<String> = entries

        .iter()

        .map(|e| e.file_name().to_string_lossy().into_owned())

        .collect();

    if !top && names.iter().any(|n| n == OXY_DIR || n == ".git") {

        ctx.out

            .lock()

            .unwrap_or_else(|e| e.into_inner())

            .nested

            .push(dir);

        return;

    }

    let rules = rules.enter(&dir, &names);

    let mut mine = Listing::default();

    for (e, name) in entries.into_iter().zip(names) {

        if name == OXY_DIR {

            continue;

        }

        let Ok(ft) = e.file_type() else { continue };

        let path = e.path();

        if rules.ignored(&path, ft.is_dir()) {

            continue;

        }

        let child = if rel.is_empty() {

            name

        } else {

            format!("{rel}/{name}")

        };

        if ft.is_dir() {

            let rules = rules.clone();

            s.spawn(move |s| visit(s, path, child, false, rules, ctx));

        } else if !(ctx.in_scope)(&child) {

        } else if ft.is_symlink() {

            match fs::read_link(&path) {

                Ok(t) => mine

                    .symlinks

                    .push((child, t.to_string_lossy().as_bytes().to_vec())),

                Err(e) => {

                    ctx.failed

                        .lock()

                        .unwrap_or_else(|e| e.into_inner())

                        .get_or_insert_with(|| format!("{}: {e}", path.display()));

                }

            }

        } else if ft.is_file() {

            match e.metadata() {

                Ok(m) => mine.files.push((child, path, m)),

                Err(e) => {

                    ctx.failed

                        .lock()

                        .unwrap_or_else(|e| e.into_inner())

                        .get_or_insert_with(|| format!("{}: {e}", path.display()));

                }

            }

        }

    }

    if !mine.files.is_empty() || !mine.symlinks.is_empty() {

        let mut out = ctx.out.lock().unwrap_or_else(|e| e.into_inner());

        out.files.append(&mut mine.files);

        out.symlinks.append(&mut mine.symlinks);

    }

}

#[cfg(test)]

mod tests {

    use super::*;

    fn write(root: &Path, rel: &str, text: &str) {

        let p = root.join(rel);

        fs::create_dir_all(p.parent().unwrap()).unwrap();

        fs::write(p, text).unwrap();

    }

    fn listed(root: &Path, in_scope: &(dyn Fn(&str) -> bool + Sync)) -> (Vec<String>, Vec<String>) {

        let l = list(root, in_scope).unwrap();

        let mut files: Vec<String> = l.files.into_iter().map(|(r, _, _)| r).collect();

        files.extend(l.symlinks.into_iter().map(|(r, _)| r));

        files.sort();

        let mut nested: Vec<String> = l

            .nested

            .iter()

            .map(|d| {

                d.strip_prefix(root)

                    .unwrap()

                    .to_string_lossy()

                    .replace('\\', "/")

            })

            .collect();

        nested.sort();

        (files, nested)

    }

    #[test]

    fn ignore_files_nested_repositories_and_scope() {

        let dir = tempfile::tempdir().unwrap();

        let root = dir.path();

        write(root, ".oxy/config", "");

        write(root, ".oxyignore", "*.log\nbuild/\n");

        write(root, ".hidden", "");

        write(root, "a.txt", "");

        write(root, "a.log", "");

        write(root, "build/out.o", "");

        write(root, "src/main.c", "");

        write(root, "src/deep/x.log", "");

        // A nearer file overrides an outer rule, and `.ignore` counts too.

        write(root, "keep/.oxyignore", "!*.log\n");

        write(root, "keep/k.log", "");

        write(root, "other/.ignore", "secret\n");

        write(root, "other/secret", "");

        write(root, "other/public", "");

        // Nested repositories are reported, not entered.

        write(root, "sub/.oxy/config", "");

        write(root, "sub/inner.txt", "");

        write(root, "vendor/g/.git/HEAD", "");

        #[cfg(unix)]

        std::os::unix::fs::symlink("a.txt", root.join("link")).unwrap();

        let (files, nested) = listed(root, &|_| true);

        let mut want = vec![

            ".hidden",

            ".oxyignore",

            "a.txt",

            "keep/.oxyignore",

            "keep/k.log",

            "other/.ignore",

            "other/public",

            "src/main.c",

        ];

        if cfg!(unix) {

            want.push("link");

        }

        want.sort();

        assert_eq!(files, want);

        assert_eq!(nested, vec!["sub", "vendor/g"]);

        // A sparse scope leaves files out but still visits directories.

        let (files, _) = listed(root, &|p: &str| p.starts_with("src/"));

        assert_eq!(files, vec!["src/main.c"]);

    }

}

mod walk;

    /// A listing of the working copy left by a failed clean check, for the

    /// snapshot that follows it to reuse; taken by the next walk and

    /// dropped whenever the manifest is written.

    pub(crate) listing_cache: std::cell::RefCell<Option<walk::Listing>>,

            listing_cache: Default::default(),

            listing_cache: Default::default(),

                    listing_cache: Default::default(),

                    listing_cache: Default::default(),

        let sparse = &self.config.sparse_paths;

        let walk_phase = crate::timing::phase("walk:   of which listing");

        let walk::Listing {

            files,

            symlinks,

            nested,

        } = match self.listing_cache.take() {

            Some(l) => l,

            None => walk::list(&self.root, &in_scope)?,

        };

        for (rel, target) in symlinks {

            out.insert(rel, FsEntry::new(NodeKind::Symlink, false, target));

        drop(walk_phase);

        let _read_phase = crate::timing::phase("walk:   of which stat and compare");

        drop(_read_phase);

        let _rest = crate::timing::phase("walk:   of which subrepos and hidden");

        for dir in nested {

        self.listing_cache.take();

        self.listing_cache.take();

        self.listing_cache.take();

        Ok(self.quick_dirty_keeping(true)? == Some(false))

        self.quick_dirty_keeping(false)

    }

    /// `quick_dirty`; with `keep`, the listing it made is left for the

    /// next walk of the working copy.

    fn quick_dirty_keeping(&self, keep: bool) -> Result<Option<bool>> {

        use rayon::prelude::*;

        let sparse = &self.config.sparse_paths;

        };

        // A working copy that cannot be listed counts as changed.

        let Ok(listing) = super::walk::list(&self.root, &in_scope) else {

            return Ok(Some(true));

        let differs = || -> Result<bool> {

            let links_differ = listing.symlinks.iter().any(|(rel, target)| {

                manifest

                    .entries

                    .get(rel)

                    .is_none_or(|me| blake3::hash(target).to_hex().as_str() != me.hash)

            });

            if links_differ {

                return Ok(true);

            }

            // Each file is checked on its own, in parallel, stopping at the

            // first difference; an unreadable one counts as different.

            let files_differ = listing.files.par_iter().any(|(rel, path, m)| {

                manifest.entries.get(rel).is_none_or(|me| {

                    m.len() != me.size

                        || super::is_executable(m) != me.exec

                        || (mtime_ns(m) != me.mtime_ns

                            && !fs::read(path)

                                .is_ok_and(|b| blake3::hash(&b).to_hex().as_str() == me.hash))

                })

            });

            if files_differ {

                return Ok(true);

            }

            let seen = listing.files.len() + listing.symlinks.len();

            if seen == manifest.entries.len() {

                return Ok(false);

            }

            // Manifest entries the walk skipped: tracked files that ignore rules

            // hide. They count as present when still on disk and unchanged.

            for (rel, me) in &manifest.entries {

                let path = self.root.join(rel);

                let Ok(m) = fs::symlink_metadata(&path) else {

                    return Ok(true);

                if m.file_type().is_symlink() {

                    continue;

                if m.len() != me.size

                    || super::is_executable(&m) != me.exec

                    || (mtime_ns(&m) != me.mtime_ns

                        && blake3::hash(&fs::read(&path)?).to_hex().as_str() != me.hash)

                {

                    return Ok(true);

            Ok(false)

        };

        let dirty = differs()?;

        if dirty && keep {

            *self.listing_cache.borrow_mut() = Some(listing);

        Ok(Some(dirty))