Check out only what differs, a batch at a time c5ce0367eddf

/// Text files rendered at a time by a checkout: bounds its memory.

const CHECKOUT_BATCH: usize = 2048;

        // A status snapshot: files the manifest proves unchanged are neither

        // read nor rendered, and its base is history only (oxygen#35).

        let snap = self.snapshot_status(command)?;

        let old_manifest = self.read_manifest()?.map(|(_, _, m)| m).unwrap_or_default();

        let base = &snap.base;

        let base_files = &snap.base_snapshot.files;

        let changed: HashSet<&str> = snap.statuses.iter().map(|s| s.path.as_str()).collect();

        // The target's history; its file graphs are built a batch at a time

        // for the files that must be written, and dropped after.

        let mut target = self.load_state_history(heads)?;

        let heads = target.maximal(heads);

        let tview = target.tree().view();

                .problems(tview.entries.keys().map(String::as_str)),

        report.conflicted = tview.is_conflicted();

        // Remove what the base or the working copy has and the target does

        // not. A nested repository is never deleted (spec §7.8).

        let gone: BTreeSet<&str> = base_files

            .iter()

            .filter(|(_, f)| f.kind != NodeKind::Dir)

            .map(|(p, _)| p.as_str())

            .chain(changed.iter().copied())

            .filter(|p| !tview.entries.contains_key(*p))

            .collect();

        for path in gone {

            if base_files

                .get(path)

                .is_some_and(|f| f.kind == NodeKind::Subrepo)

            {

                report

                    .notes

                    .push(format!("{path}: nested repository left in place"));

                continue;

            }

            let full = self.safe_path(path)?;

            if fs::symlink_metadata(&full).is_err() {

                continue;

            }

            fs::remove_file(&full)?;

            report.removed += 1;

            if let Some(parent) = full.parent() {

                dirs_to_prune.insert(parent.to_path_buf());

            }

        }

        // The manifest after the checkout, built from what is known.

        let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();

        let hex = |b: &[u8]| blake3::hash(b).to_hex().to_string();

        // Text files to render, attribute files first (the others need them).

        let mut texts: Vec<(String, NodeId, Mode)> = Vec::new();

        for (path, e) in &tview.entries {

            if e.kind == NodeKind::Dir || !self.in_scope(path) {

                continue;

            }

            let untouched = !changed.contains(path.as_str())

                && base_files

                    .get(path)

                    .is_some_and(|b| b.node == e.node && b.kind == e.kind && b.mode == e.mode);

            if e.kind == NodeKind::File && !e.mode.opaque {

                // Unchanged on disk (unrendered by the snapshot) and the same

                // graph in both states: nothing to read, render or write.

                let same_graph = untouched

                    && snap.base_snapshot.unrendered.contains(path)

                    && !is_attributes(path)

                    && base.graph_key(&e.node).is_some()

                    && base.graph_key(&e.node) == target.graph_key(&e.node);

                match old_manifest.entries.get(path).filter(|_| same_graph) {

                    Some(m) => {

                        entries.insert(

                            path.clone(),

                            FsEntry {

                                content_hash: Some(m.hash.clone()),

                                graph_key: m.graph.clone(),

                                unchanged: true,

                                ..FsEntry::new(NodeKind::File, e.mode.exec, Vec::new())

                            },

                        );

                    }

                    None => texts.push((path.clone(), e.node, e.mode)),

                }

                continue;

            }

            // Opaque files, symlinks and subrepos: their blobs.

            let blobs = target.live_blobs(&e.node);

            if blobs.len() > 1 {

                report.conflicted = true;

            }

            let Some(blob) = blobs.first().copied() else {

                continue;

            };

            if e.kind == NodeKind::Subrepo {

                if keep_subrepos.contains(path) {

                let pin = crate::subrepo::Pin::decode(&self.read_blob(&blob)?)?;

                let sync = self.sync_subrepo(path, &pin, false)?;

                if matches!(

                    sync,

                    SubrepoSync::Updated | SubrepoSync::KeptBookmark { .. }

                ) {

                    report.written += 1;

                report.notes.extend(sync.note(path));

                continue;

            let same_blob = untouched && base.live_blobs(&e.node) == blobs;

            if let Some(m) = old_manifest

                .entries

                .get(path)

                .filter(|_| same_blob && blobs.len() == 1)

            {

                entries.insert(

                    path.clone(),

                    FsEntry {

                        content_hash: Some(m.hash.clone()),

                        blob: (e.kind == NodeKind::File).then_some(blob),

                        ..FsEntry::new(e.kind, e.mode.exec, Vec::new())

                    },

                );

            let bytes = self.read_blob(&blob)?;

            let full = self.root.join(path);

            if e.kind == NodeKind::Symlink {

                let same = fs::read_link(&full)

                    .is_ok_and(|t| t.to_string_lossy().as_bytes() == bytes.as_slice());

                if !same {

                    self.write_symlink(path, &bytes)?;

                    report.written += 1;

                entries.insert(

                    path.clone(),

                    FsEntry {

                        content_hash: Some(hex(&bytes)),

                        ..FsEntry::new(NodeKind::Symlink, false, Vec::new())

                    },

                );

            } else {

                if !self.same_on_disk(&full, &bytes, e.mode.exec) {

                    self.write_file(path, &bytes, e.mode.exec)?;

                    report.written += 1;

                }

                entries.insert(

                    path.clone(),

                    FsEntry {

                        content_hash: Some(hex(&bytes)),

                        blob: (blobs.len() == 1).then_some(blob),

                        ..FsEntry::new(NodeKind::File, e.mode.exec, Vec::new())

                    },

                );

            }

            for (i, b) in blobs.iter().enumerate().skip(1) {

                self.write_file(

                    &format!("{path}.oxy-conflict-{}", i + 1),

                    &self.read_blob(b)?,

                    e.mode.exec,

                )?;

            }

        }

        // Render and write text files a batch at a time; attribute files

        // come first, since the others' line endings and locks follow them.

        texts.sort_by_key(|(p, _, _)| !is_attributes(p));

        let keys: HashMap<NodeId, [u8; 32]> = texts

            .iter()

            .filter_map(|(_, n, _)| Some((*n, target.graph_key(n)?)))

            .collect();

        let mut attrs = crate::attributes::Attributes::default();

        let attr_count = texts.iter().filter(|(p, _, _)| is_attributes(p)).count();

        let mut done = 0usize;

        let mut batches: Vec<&[(String, NodeId, Mode)]> = Vec::new();

        if attr_count > 0 {

            batches.push(&texts[..attr_count]);

        }

        batches.extend(texts[attr_count..].chunks(CHECKOUT_BATCH));

        for batch in batches {

            use rayon::prelude::*;

            let nodes: Vec<NodeId> = batch.iter().map(|(_, n, _)| *n).collect();

            self.load_graphs_of(&mut target, &nodes)?;

            let rendered: Vec<(Vec<u8>, bool)> = nodes

                .par_iter()

                .map(|n| {

                    let (bytes, m) = target.render_file(n);

                    (bytes, m.is_conflicted())

                })

                .collect();

            target.drop_graphs(&nodes);

            if done == 0 && attr_count > 0 {

                attrs = crate::attributes::Attributes::from_files(

                    batch

                        .iter()

                        .zip(&rendered)

                        .map(|((p, _, _), (b, _))| (p.as_str(), b.as_slice())),

                );

            }

            done += batch.len();

            for ((path, node, mode), (bytes, conflicted)) in batch.iter().zip(rendered) {

                let a = attrs.lookup(path);

                let crlf = a.eol == Some(crate::attributes::Eol::Crlf);

                let on_disk: Vec<u8> = if crlf {

                    crate::attributes::to_crlf(&bytes)

                } else {

                    bytes

                };

                if !self.same_on_disk(&self.root.join(path), &on_disk, mode.exec) {

                    self.write_file(path, &on_disk, mode.exec)?;

                    report.written += 1;

                report.conflicted |= conflicted;

                entries.insert(

                    path.clone(),

                    FsEntry {

                        content_hash: Some(hex(&on_disk)),

                        graph_key: (!conflicted && a.eol.is_none())

                            .then(|| keys.get(node).map(|k| ObjectRef(*k).to_hex()))

                            .flatten(),

                        ..FsEntry::new(NodeKind::File, mode.exec, Vec::new())

                    },

                );

        // Lockable files are read-only until locked (spec §18), whether

        // written now or left as they were.

        for (path, e) in &entries {

            if e.kind == NodeKind::File && attrs.lookup(path).lockable && !self.is_locked(path) {

                let _ = set_readonly(&self.root.join(path), true);

            }

        }

        self.write_manifest_entries(report.conflicted, false, &entries, &HashMap::new())?;

    }

    /// Whether the file at `full` already holds `bytes` with this executable

    /// bit, so writing it again can be skipped.

    fn same_on_disk(&self, full: &Path, bytes: &[u8], exec: bool) -> bool {

        match fs::symlink_metadata(full) {

            Ok(m) if m.is_file() && m.len() == bytes.len() as u64 => {

                (!self.platform.exec_bits || is_executable(&m) == exec)

                    && fs::read(full).is_ok_and(|b| b == bytes)

            }

            _ => false,

        }

/// oxygen#35: a checkout reads, renders and writes only the files whose

/// graphs differ, and renders the rest a batch at a time.

#[test]

fn checkout_touches_only_what_differs_and_renders_in_batches() {

    use std::os::unix::fs::PermissionsExt;

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let origin = root.join("origin");

    let mut repo = Repo::init(&origin).unwrap();

    // More files than one batch holds.

    let n = 2100;

    let old = std::time::SystemTime::now() - std::time::Duration::from_secs(30);

    for i in 0..n {

        let p = format!("f/{i:04}.txt");

        write(&origin, &p, format!("file {i}\n").as_bytes());

        fs::File::options()

            .write(true)

            .open(origin.join(&p))

            .unwrap()

            .set_modified(old)

            .unwrap();

    }

    repo.commit("many", &[]).unwrap();

    repo.bookmark_create("feature", None).unwrap();

    repo.bookmark_switch("feature", false).unwrap();

    write(&origin, "f/0007.txt", b"changed\n");

    repo.commit("one change", &[]).unwrap();

    // Switching back to main must not read the unchanged files.

    for i in 0..n {

        if i != 7 {

            let p = origin.join(format!("f/{i:04}.txt"));

            fs::set_permissions(&p, fs::Permissions::from_mode(0o000)).unwrap();

        }

    }

    let r = repo.bookmark_switch("main", false).unwrap();

    for i in 0..n {

        let p = origin.join(format!("f/{i:04}.txt"));

        fs::set_permissions(&p, fs::Permissions::from_mode(0o644)).unwrap();

    }

    assert_eq!((r.written, r.removed), (1, 0), "{r:?}");

    assert_eq!(fs::read(origin.join("f/0007.txt")).unwrap(), b"file 7\n");

    assert!(repo.snapshot_status("status").unwrap().statuses.is_empty());

    // A clone renders every file, across batches.

    let c = root.join("clone");

    Repo::clone(&origin.to_string_lossy(), &c).unwrap();

    for i in [0, 2047, 2048, n - 1] {

        assert_eq!(

            fs::read(c.join(format!("f/{i:04}.txt"))).unwrap(),

            format!("file {i}\n").as_bytes()

        );

    }

    let cloned = Repo::open(&c).unwrap();

    assert!(cloned

        .snapshot_status("status")

        .unwrap()

        .statuses

        .is_empty());

}

    /// Drop the built graphs of `nodes`, for a caller rendering a large

    /// tree a batch at a time. The state can no longer render those files,

    /// so it must not be used for them again.

    pub fn drop_graphs(&mut self, nodes: &[NodeId]) {

        for n in nodes {

            self.files.remove(n);

        }

    }

### Checkout and clone (2026-09-29, oxygen#35)

A checkout built and rendered every file of the target, read the whole

working copy, and after writing walked and hashed it all again for the

manifest. Switching bookmarks or cloning the Linux tree was killed at

14 GB. Now:

- a file is skipped when the manifest shows it unchanged and its graph is

  the same in both states;

- the rest are rendered 2 048 at a time, and each batch's graphs are

  dropped once written;

- the manifest is built from what the checkout already knows.

| Linux tip | oxy before | oxy now | Git |

|---|---:|---:|---:|

| Switch bookmark (one file differs) | killed at 14 GB | 7.1 to 7.4 s, 1.3 GB | 0.18 to 0.22 s, 39 MB |

| Local clone | killed at 14 GB | 98 s, 7.9 GB | 9.8 s, 469 MB |

Most of a switch is now loading history: the base's and the target's

history each decode the one huge first change (1.2 s and 364 MB each).

A clone holds the whole bundle in memory.