The manifest keeps directory times: status lists the tree only when a directory changed (refs #41) 1ec3f008e96a

- **Status twice as fast as Git on the Linux tree (#41).** With uncommitted work, `oxy status` answers from a status cache while every file is as the last snapshot left it, and the manifest keeps every directory's time, so the tree is listed only when a directory's entries changed. One edited file is 0.10 to 0.12 s (from 0.44 s; Git 0.22 s) and a clean status 0.09 to 0.12 s (Git 0.20 to 0.26 s), in 80 MB. `oxy diff` renders the working copy from the cache too (0.25 s, from 0.41 s). A file written within the racy window is now hashed by the prompt's check too, as it was by status.

    /// The directories (with their times) and nested repositories of the

    /// last walk, for the manifest written from it.

    pub(crate) walked_dirs: std::cell::RefCell<Option<WalkedDirs>>,

/// A walk's directories with their times, and its nested repositories.

pub(crate) type WalkedDirs = (Vec<(String, u128)>, Vec<String>);

            walked_dirs: Default::default(),

            walked_dirs: Default::default(),

                    walked_dirs: Default::default(),

                    walked_dirs: Default::default(),

            dirs,

        *self.walked_dirs.borrow_mut() =

            Some((dirs, nested.iter().map(|d| self.rel_path(d)).collect()));

    /// Every directory the walk listed, with its time then; empty when the

    /// manifest was not written from a walk (a checkout), and then the

    /// check lists the tree.

    pub dirs: Vec<(String, u128)>,

    /// Directories holding a repository of their own, as the walk found.

    pub nested: Vec<String>,

/// whether it is executable. Format 3 puts before the entries every

/// directory the walk listed with its time, and the nested repositories. A file that does not check, or in any other

const MANIFEST_MAGIC: &[u8; 8] = b"oxyman3\0";

    dirs: &[(String, u128)],

    nested: &[String],

    w.count(dirs.len());

    for (d, t) in dirs {

        w.bytes(d.as_bytes());

        w.fixed(&t.to_le_bytes());

    }

    w.count(nested.len());

    for n in nested {

        w.bytes(n.as_bytes());

    }

    }

    let mut dirs = Vec::new();

    for _ in 0..r.count().ok()? {

        let d = std::str::from_utf8(r.bytes().ok()?).ok()?.to_string();

        dirs.push((d, u128::from_le_bytes(r.fixed().ok()?)));

    }

    let mut nested = Vec::new();

    for _ in 0..r.count().ok()? {

        nested.push(std::str::from_utf8(r.bytes().ok()?).ok()?.to_string());

            dirs,

            nested,

        // The directories of the walk these entries came from, if any.

        let (dirs, nested) = self.walked_dirs.take().unwrap_or_default();

        let new = encode_manifest(conflicted, uncommitted, written, &dirs, &nested, &mut rows);

            &encode_manifest(

                conflicted,

                uncommitted,

                m.written_ns,

                &m.dirs,

                &m.nested,

                &mut rows,

            ),

        if self.disk_differs(&m, true)? {

        // A file written within the racy window can change without its size

        // or time moving (a coarse clock): those entries are hashed.

        if self.racy_change(manifest).is_some() {

            return Ok(true);

        }

        if let Some(differs) = self.disk_differs_by_dirs(manifest) {

            return Ok(differs);

        }

        self.disk_differs_listing(manifest, keep)

    }

    /// [`Repo::disk_differs`] without listing the tree, when the manifest

    /// knows its directories: a directory's time moves when an entry is

    /// added, removed or renamed in it, so with every directory's time as

    /// recorded (and none within the racy window) the files are the

    /// manifest's, and each is checked by its stat. `None` when a

    /// directory changed or the manifest has none: the tree must be listed.

    pub(crate) fn disk_differs_by_dirs(&self, manifest: &Manifest) -> Option<bool> {

        use rayon::prelude::*;

        if manifest.dirs.is_empty() || !self.config.sparse_paths.is_empty() {

            return None;

        }

        let _t = crate::timing::phase("status: directories and files by stat");

        let root = &self.root;

        let dirs_same = manifest.dirs.par_iter().all(|(rel, t)| {

            *t + 2_000_000_000 < manifest.written_ns

                && fs::metadata(root.join(rel)).is_ok_and(|m| m.is_dir() && mtime_ns(&m) == *t)

        });

        if !dirs_same {

            return None;

        }

        let differs = manifest.entries.par_iter().any(|(rel, me)| {

            let path = root.join(rel.as_str());

            let Ok(m) = fs::symlink_metadata(&path) else {

                return true;

            };

            if m.file_type().is_symlink() {

                return !fs::read_link(&path).is_ok_and(|t| {

                    *blake3::hash(t.to_string_lossy().as_bytes()).as_bytes() == me.hash

                });

            }

            !m.is_file()

                || m.len() != me.size

                || super::is_executable(&m) != me.exec

                || (mtime_ns(&m) != me.mtime_ns

                    && !fs::read(&path).is_ok_and(|b| *blake3::hash(&b).as_bytes() == me.hash))

        });

        if !differs {

            // Nested repositories the manifest has no entry for are not

            // subrepos: nothing in them is tracked (oxygen#43).

            *self.untracked_nested.borrow_mut() = manifest

                .nested

                .iter()

                .filter(|rel| !manifest.entries.contains_key(*rel))

                .cloned()

                .collect();

        }

        Some(differs)

    }

    /// [`Repo::disk_differs`] by listing the tree.

    fn disk_differs_listing(&self, manifest: &Manifest, keep: bool) -> Result<bool> {

/// oxygen#41: a manifest written from a walk keeps every directory's time;

/// while none has moved, the check stats the tracked files without listing

/// the tree, and a file added or removed anywhere moves its directory's

/// time, so the tree is listed again.

#[test]

fn the_manifest_check_skips_listing_while_no_directory_changed() {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let repo = Repo::init(root).unwrap();

    write(root, "a.txt", b"a\n");

    write(root, "sub/deep/b.txt", b"b\n");

    repo.commit("base", &[]).unwrap();

    // Out of the racy window: every file and directory a minute old.

    let old = std::time::SystemTime::now() - std::time::Duration::from_secs(60);

    for p in ["a.txt", "sub/deep/b.txt", "sub/deep", "sub", ""] {

        fs::File::open(root.join(p))

            .unwrap()

            .set_modified(old)

            .unwrap();

    }

    repo.snapshot_status("oxy status").unwrap();

    repo.write_manifest(false, false).unwrap();

    let m = repo.read_manifest().unwrap().unwrap().2;

    assert!(m.dirs.iter().any(|(d, _)| d == "sub/deep"), "{:?}", m.dirs);

    assert_eq!(

        repo.disk_differs_by_dirs(&m),

        Some(false),

        "no listing needed"

    );

    assert_eq!(repo.status_by_manifest().unwrap(), Some(vec![]));

    // A same-size edit with a new time: found by its stat.

    write(root, "sub/deep/b.txt", b"c\n");

    assert_eq!(repo.disk_differs_by_dirs(&m), Some(true));

    write(root, "sub/deep/b.txt", b"b\n");

    fs::File::open(root.join("sub/deep/b.txt"))

        .unwrap()

        .set_modified(old)

        .unwrap();

    assert_eq!(repo.disk_differs_by_dirs(&m), Some(false));

    // A new file moves its directory's time: the tree must be listed.

    write(root, "sub/deep/new.txt", b"n\n");

    assert_eq!(repo.disk_differs_by_dirs(&m), None);

    assert_eq!(repo.status_by_manifest().unwrap(), None);

    fs::remove_file(root.join("sub/deep/new.txt")).unwrap();

    // A removed one too.

    fs::remove_file(root.join("a.txt")).unwrap();

    assert_eq!(repo.disk_differs_by_dirs(&m), None);

}

    /// Every directory listed (the root as ""), with its modification time

    /// read before its entries were: a later change to its entries moves

    /// the time, so a manifest that keeps these can skip reading a

    /// directory whose time has not moved.

    pub dirs: Vec<(String, u128)>,

    let root_time = fs::metadata(root)

        .map(|m| super::prompt::mtime_ns(&m))

        .unwrap_or(0);

            root_time,

        out.dirs.sort_unstable_by(|a, b| a.0.cmp(&b.0));

        out.nested.sort_unstable();

#[allow(clippy::too_many_arguments)]

    mtime: u128,

    mine.dirs.push((rel.clone(), mtime));

            // The directory's time now, before it is read.

            let time = e

                .metadata()

                .map(|m| super::prompt::mtime_ns(&m))

                .unwrap_or(0);

            s.spawn(move |s| visit(s, path, child, time, false, rules, ctx));

    let mut out = ctx.out.lock().unwrap_or_else(|e| e.into_inner());

    out.files.append(&mut mine.files);

    out.symlinks.append(&mut mine.symlinks);

    out.dirs.append(&mut mine.dirs);

Then directory times (same evening, load average 3.1): the manifest

(format 3) keeps every directory the walk listed with its time, read

before the directory was. While no directory's time has moved, the check

stats the tracked files and does not list the tree; a directory whose

entries changed sends it back to a listing.

| | before | after | Git |

|---|---:|---:|---:|

| status, one file edited | 0.16 to 0.20 s, 120 to 155 MB | **0.10 to 0.12 s, 80 MB** | 0.22 s |

| status, clean | 0.15 to 0.21 s | **0.09 to 0.12 s, 80 MB** | 0.20 to 0.26 s |

| diff, one file | 0.34 s | 0.24 to 0.25 s | 0.09 to 0.13 s |