Commit: cheaper bookkeeping around a change (refs #41) f9466021286a

/// Replace a cache file: whole or not at all, but without waiting for the

/// disk. A cache checks its own bytes when read, so one a crash leaves

/// short or stale is rebuilt rather than trusted.

/// How many changes a state index may fall behind its working copy before

/// it is written again; until then each load replays them.

const INDEX_REPLAY_LIMIT: usize = 16;

const INDEX_REPLAY_TIME: std::time::Duration = std::time::Duration::from_millis(50);

pub(crate) fn write_cache(path: &Path, data: &[u8]) -> Result<()> {

    let tmp = path.with_extension(format!("tmp-{}", std::process::id()));

    fs::write(&tmp, data)?;

    fs::rename(&tmp, path)?;

    Ok(())

}

                // A change the state holds brings its history with it.

                if !seen.insert(h) || state.contains(&h) {

        // Nested repositories and ignored paths are left out, as in a walk.

        let listing = walk::list(&self.root, &|p: &str| is_attributes(p)).unwrap_or_default();

        let files: Vec<(String, Vec<u8>)> = listing

            .files

            .into_iter()

            .filter_map(|(rel, path, _)| Some((rel, fs::read(path).ok()?)))

            .collect();

        if let Some((st, heads)) = self.read_state_index() {

            let mut have = st.maximal(deps);

            have.sort();

            if deps.iter().all(|d| st.contains(d)) && have == heads {

                #[cfg(test)]

                assert_eq!(

                    st.write_index(),

                    self.load_state_history(deps)?.write_index(),

                    "a state index holds the state history gives"

                );

                return Ok(st);

            }

            // An index a few changes behind (a commit, a pull since): add

            // what it lacks, as long as all it holds is history of `deps`.

            // Anything amiss (a change beyond a shallow boundary) takes the

            // long way instead.

            let (indexed, started) = (st.len(), std::time::Instant::now());

            if let Ok(st) = self.load_state_replayed(deps, st) {

                let mut all = deps.to_vec();

                all.extend(heads.iter().copied());

                let (mut within, mut want) = (st.maximal(&all), st.maximal(deps));

                within.sort();

                want.sort();

                if within == want {

                    #[cfg(test)]

                    self.assert_same_state(&st, &self.load_state_history(deps)?);

                    // Writing the index costs far more than replaying a

                    // few small changes: it is rewritten once it falls

                    // behind, or when replaying took long (large changes).

                    if st.len() - indexed > INDEX_REPLAY_LIMIT

                        || started.elapsed() > INDEX_REPLAY_TIME

                    {

                        self.write_state_index(deps, &st);

                    }

                    return Ok(st);

                }

            }

    }

    /// Two history-only states hold the same changes and render the same

    /// files. Their indexes may differ: one loaded from a checkpoint keeps

    /// the checkpoint's graphs as the bases of its files' histories.

    #[cfg(test)]

    fn assert_same_state(&self, a: &State, b: &State) {

        let ids = |s: &State| s.changes().map(|(h, _)| *h).collect::<BTreeSet<_>>();

        assert_eq!(ids(a), ids(b), "the same changes");

        let files = |s: &State| {

            let mut s = s.clone();

            self.load_all_graphs(&mut s).unwrap();

            s.materialize().files

        };

        assert_eq!(files(a), files(b), "the same files");

    fn read_state_index(&self) -> Option<(State, Vec<ChangeHash>)> {

        Some((State::read_index(body).ok()?, heads))

        let encoding = crate::timing::phase("write:   of which encoding");

        drop(encoding);

        let _ = fs::create_dir_all(self.oxy.join("index"));

        let _ = write_cache(&path, &w.0);

/// recognised; BLAKE3 of everything else; then each entry in path order:

/// path, size, mtime in nanoseconds, BLAKE3 of the content, the stored

/// blob of an opaque file, the graph key it was found equal to, and

/// whether it is executable. A file that does not check, or in any other

/// layout (an older manifest), reads as no manifest, and the next snapshot

/// writes it again: it is a cache, written without waiting for the disk.

const MANIFEST_MAGIC: &[u8; 8] = b"oxyman2\0";

/// Where the checksum sits, after the written time.

const MANIFEST_SUM: std::ops::Range<usize> = 26..58;

fn manifest_sum(buf: &[u8]) -> [u8; 32] {

    let mut h = blake3::Hasher::new();

    h.update(&buf[..MANIFEST_SUM.start]);

    h.update_rayon(&buf[MANIFEST_SUM.end..]);

    *h.finalize().as_bytes()

}

    w.fixed(&[0; 32]);

    let sum = manifest_sum(&w.0);

    w.0[MANIFEST_SUM].copy_from_slice(&sum);

    if r.fixed::<32>().ok()? != manifest_sum(buf) {

        return None;

    }

            let (w, sum) = (MANIFEST_WRITTEN, MANIFEST_SUM);

                && old[sum.end..] == new[sum.end..]

        write_cache(&p, &new)

        write_cache(

        // Why a snapshot must decide, with timings on.

        let not = |why: &str| {

            if crate::timing::enabled() {

                eprintln!("timing status: not clean by the manifest: {why}");

            }

            Ok(false)

        };

            return not("no manifest");

            return not("conflicted, uncommitted or subrepos");

                return not(&format!("{rel} changed within the racy window"));

        }

        if self.quick_dirty_keeping(true)? != Some(false) {

            return not("a file differs");

        Ok(true)

    /// The `.oxyattributes` in force on disk, from the files the walk read.

    pub(crate) attributes: crate::attributes::Attributes,

        self.snapshot_as(command, false, false)

        self.snapshot_as(command, true, false)

    /// `snapshot_as`; `force_manifest` writes the manifest even where a

    /// status would not bother (an edited file is read next time whatever

    /// it says), for a commit, after which that file is unchanged.

    fn snapshot_as(

        &self,

        command: &str,

        partial: bool,

        force_manifest: bool,

    ) -> Result<SnapshotResult> {

        let result = self.snapshot_locked(command, partial, force_manifest);

    fn snapshot_locked(

        &self,

        command: &str,

        partial: bool,

        force_manifest: bool,

    ) -> Result<SnapshotResult> {

            _ if force_manifest => true,

        let attributes = crate::attributes::Attributes::from_files(

            fs_entries

                .iter()

                .filter(|(p, _)| is_attributes(p))

                .map(|(p, e)| (p.as_str(), e.content.as_slice())),

        );

            attributes,

    /// The graph key in `st` of each text file at `paths`, the paths a

    /// commit changed: only their graphs changed, so the keys the manifest

    /// holds for the others still stand (and any that does not is simply

    /// not trusted by the next snapshot, which compares keys). Files with

    /// an `eol` attribute are left out: their bytes on disk are not the

    /// rendering.

    fn committed_graph_keys(

        &self,

        st: &State,

        paths: &[String],

        attrs: &crate::attributes::Attributes,

    ) -> HashMap<String, [u8; 32]> {

        let view = st.tree().view();

        paths

            .filter_map(|p| {

                let e = view.entries.get(p)?;

                (e.kind == NodeKind::File && !e.mode.opaque && attrs.lookup(p).eol.is_none())

                    .then_some(())?;

                Some((p.clone(), st.graph_key(&e.node)?))

        let mut snap = self.snapshot_as("oxy commit", true, true)?;

        let changed_paths: Vec<String> = snap.statuses.iter().map(|s| s.path.clone()).collect();

        let signing = crate::timing::phase("commit: sign and store");

        drop(signing);

        let extending = crate::timing::phase("commit: extend the state");

        drop(extending);

        let committed_keys =

            (!partial).then(|| self.committed_graph_keys(&st, &changed_paths, &snap.attributes));

        let recording = crate::timing::phase("commit: record");

        drop(recording);

        let _rewrite = crate::timing::phase("commit: manifest and hooks");

#[test]

fn a_manifest_that_does_not_check_reads_as_none() {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let repo = Repo::init(root).unwrap();

    write(root, "a.txt", b"1\n");

    repo.commit("first", &[]).unwrap();

    let path = repo.manifest_path();

    let mut bytes = fs::read(&path).unwrap();

    assert!(Repo::open(root).unwrap().read_manifest().unwrap().is_some());

    let last = bytes.len() - 1;

    bytes[last] ^= 1;

    fs::write(&path, &bytes).unwrap();

    assert!(Repo::open(root).unwrap().read_manifest().unwrap().is_none());

    // The next status reads the files and writes it again.

    let again = Repo::open(root).unwrap();

    assert!(!again.snapshot("status").unwrap().changed);

    assert!(again.read_manifest().unwrap().is_some());

}

#[test]

fn a_commit_leaves_a_manifest_that_proves_the_working_copy_clean() {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let repo = Repo::init(root).unwrap();

    write(root, "a.txt", b"1\n");

    write(root, "b.txt", b"2\n");

    // b.txt outside the racy window, so snapshots take it as unchanged.

    let old = std::time::SystemTime::now() - std::time::Duration::from_secs(60);

    fs::File::options()

        .write(true)

        .open(root.join("b.txt"))

        .unwrap()

        .set_modified(old)

        .unwrap();

    repo.commit("first", &[]).unwrap();

    assert!(repo.clean_by_manifest().unwrap(), "after the first commit");

    repo.snapshot_status("status").unwrap();

    write(root, "a.txt", b"1\n2\n");

    repo.commit("second", &[]).unwrap();

    assert!(

        repo.clean_by_manifest().unwrap(),

        "after a commit of one file"

    );

    // Edited, seen by a status, edited again, then committed: the status

    // left the first edit's stat, which the commit must not keep.

    write(root, "a.txt", b"1\n2\n3\n");

    repo.snapshot_status("status").unwrap();

    write(root, "a.txt", b"1\n2\n3\n4\n5\n");

    repo.commit("third", &[]).unwrap();

    assert!(

        repo.clean_by_manifest().unwrap(),

        "after an edit a status saw"

    );

}