Commit, diff and show build only what they touch 8ab265dc4a51
Commit, working-copy diff, hunks and discard use the partial snapshot: files known unchanged are neither read nor rendered, and only the graphs of changed files are built (discarding hunks builds the rest before it renders the target). Showing a committed change builds the graphs of the files it edits and leaves the rest unrendered (Repo::change_base). At 10 000 files: diff 0.57 to 0.22 s and commit 0.58 to 0.23 s, each 244 to 52 MB; show 0.53 s. closes #20
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-29 03:29 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
8ab265dc4a51e6e258c267d96f3d7071f9dae38c18f20e949f510b17627ee5c2- Depends on
- 1d360c91a289 75036530b621 75167672ebcd 957fee30826e 982864068a59 c6aa5bd9d1c8 cc7a00565161 e7ac6c2e84e4 fce44ae9ad29
- Depended on by
- c2e1d09047db 7505194116e7 10b4fd430eab 77445673174b 192fad1e46a2 6caacfff1d4f 958fb27dda7c f7510b63cceb
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 21 atoms, 0 tree operations, 6 files
Browse the tree at this change
modified crates/oxy-core/src/repo/mod.rs +23 −2
| … | 1459 unchanged lines |
||
| 1460 | 1460 | /// Human-readable description of what a change does, with line content. |
|
| 1461 | 1461 | pub fn describe_change(&self, hash: &ChangeHash) -> Result<String> { |
|
| 1462 | 1462 | let change = self.store.get_change(hash)?; |
|
| 1463 | - | let base = self.load_state(&change.deps)?; |
|
| 1464 | - | let base_snapshot = base.materialize(); |
|
| 1463 | + | let (base, base_snapshot) = self.change_base(&change)?; |
|
| 1465 | 1464 | Ok(render_change(&change, &base, &base_snapshot)) |
|
| 1465 | + | } |
|
| 1466 | + | ||
| 1467 | + | /// The base state of `change` (its dependencies) and its rendering, as |
|
| 1468 | + | /// much as showing the change needs: the whole tree, and the graphs and |
|
| 1469 | + | /// contents of the files the change edits. Other text files are left |
|
| 1470 | + | /// unrendered ([`Snapshot::unrendered`]). |
|
| 1471 | + | pub fn change_base(&self, change: &Change) -> Result<(State, Snapshot)> { |
|
| 1472 | + | let mut base = self.load_state_history(&change.deps)?; |
|
| 1473 | + | let edited: std::collections::HashSet<NodeId> = |
|
| 1474 | + | change.edits.iter().map(|e| e.node()).collect(); |
|
| 1475 | + | let nodes: Vec<NodeId> = edited.iter().copied().collect(); |
|
| 1476 | + | self.load_graphs_of(&mut base, &nodes)?; |
|
| 1477 | + | let skip: std::collections::HashSet<NodeId> = base |
|
| 1478 | + | .tree() |
|
| 1479 | + | .view() |
|
| 1480 | + | .entries |
|
| 1481 | + | .values() |
|
| 1482 | + | .filter(|e| e.kind == crate::objects::NodeKind::File && !edited.contains(&e.node)) |
|
| 1483 | + | .map(|e| e.node) |
|
| 1484 | + | .collect(); |
|
| 1485 | + | let snapshot = base.materialize_except(&skip); |
|
| 1486 | + | Ok((base, snapshot)) |
|
| 1466 | 1487 | } |
|
| 1467 | 1488 | } |
|
| 1468 | 1489 | ||
| … | 110 unchanged lines |
modified crates/oxy-core/src/repo/snapshot.rs +16 −12
| … | 195 unchanged lines |
||
| 196 | 196 | } |
|
| 197 | 197 | ||
| 198 | 198 | /// Snapshot the working copy as [`Repo::snapshot`] does, for commands |
|
| 199 | - | /// that need only its statuses, conflicts and base history (status, |
|
| 200 | - | /// log, show). Text files known unchanged since the last manifest are |
|
| 199 | + | /// that need its statuses, conflicts and base history, and the graphs |
|
| 200 | + | /// of the files it changes: status, log, show, commit, diff, hunks and |
|
| 201 | + | /// discard. Text files known unchanged since the last manifest are |
|
| 201 | 202 | /// neither read nor rendered: the result's `base` is a history-only |
|
| 202 | 203 | /// state without their graphs, and its `base_snapshot` lists them in |
|
| 203 | - | /// `unrendered` with empty bytes. They are never conflicted. |
|
| 204 | + | /// `unrendered` with empty bytes. They are never conflicted, and no |
|
| 205 | + | /// change the working copy makes touches them. |
|
| 204 | 206 | pub fn snapshot_status(&self, command: &str) -> Result<SnapshotResult> { |
|
| 205 | 207 | self.snapshot_as(command, true) |
|
| 206 | 208 | } |
|
| … | 255 unchanged lines |
||
| 462 | 464 | /// The hunks of the working copy, for `oxy commit --interactive`: each |
|
| 463 | 465 | /// tree operation and each edit, in the order `select_hunks` numbers them. |
|
| 464 | 466 | pub fn hunks(&self) -> Result<(SnapshotResult, Vec<Hunk>)> { |
|
| 465 | - | let snap = self.snapshot("oxy commit")?; |
|
| 467 | + | let snap = self.snapshot_status("oxy commit")?; |
|
| 466 | 468 | let hunks = render_hunks(&snap.wc, &snap.base, &snap.base_snapshot); |
|
| 467 | 469 | Ok((snap, hunks)) |
|
| 468 | 470 | } |
|
| … | 8 unchanged lines |
||
| 477 | 479 | extra_deps: &[ChangeHash], |
|
| 478 | 480 | selection: &Selection, |
|
| 479 | 481 | ) -> Result<ChangeHash> { |
|
| 480 | - | let snap = self.snapshot("oxy commit")?; |
|
| 482 | + | let snap = self.snapshot_status("oxy commit")?; |
|
| 481 | 483 | if snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty() { |
|
| 482 | 484 | return fail("nothing to commit: the working copy matches its base"); |
|
| 483 | 485 | } |
|
| … | 508 unchanged lines |
||
| 992 | 994 | /// edits are undone, additions removed, removals put back. Returns the |
|
| 993 | 995 | /// number of paths touched. |
|
| 994 | 996 | pub fn discard(&self, paths: &[String]) -> Result<usize> { |
|
| 995 | - | let snap = self.snapshot("oxy discard")?; |
|
| 997 | + | let snap = self.snapshot_status("oxy discard")?; |
|
| 996 | 998 | let under = |p: &str| { |
|
| 997 | 999 | paths |
|
| 998 | 1000 | .iter() |
|
| … | 102 unchanged lines |
||
| 1101 | 1103 | partial.normalize(); |
|
| 1102 | 1104 | let mut st = snap.base.clone(); |
|
| 1103 | 1105 | st.add_unchecked(partial); |
|
| 1106 | + | // Every file of the target is rendered: build the graphs the |
|
| 1107 | + | // partial snapshot left out. |
|
| 1108 | + | self.load_all_graphs(&mut st)?; |
|
| 1104 | 1109 | let target = st.materialize(); |
|
| 1105 | 1110 | let by_node_target = target.tree.by_node(); |
|
| 1106 | 1111 | let by_node_base = snap.base_snapshot.tree.by_node(); |
|
| … | 66 unchanged lines |
||
| 1173 | 1178 | /// The diff of the working copy (`None`) or of a change, per file with |
|
| 1174 | 1179 | /// line-numbered hunks (spec: `oxy diff --json`). |
|
| 1175 | 1180 | pub fn diff_files(&self, change: Option<&ChangeHash>) -> Result<Vec<FileDiff>> { |
|
| 1176 | - | let (c, base) = match change { |
|
| 1181 | + | let (c, base, base_snapshot) = match change { |
|
| 1177 | 1182 | Some(h) => { |
|
| 1178 | 1183 | let c = self.store.get_change(h)?; |
|
| 1179 | - | let base = self.load_state(&c.deps)?; |
|
| 1180 | - | (c, base) |
|
| 1184 | + | let (base, snapshot) = self.change_base(&c)?; |
|
| 1185 | + | (c, base, snapshot) |
|
| 1181 | 1186 | } |
|
| 1182 | 1187 | None => { |
|
| 1183 | - | let snap = self.snapshot("oxy diff")?; |
|
| 1184 | - | (snap.wc, snap.base) |
|
| 1188 | + | let snap = self.snapshot_status("oxy diff")?; |
|
| 1189 | + | (snap.wc, snap.base, snap.base_snapshot) |
|
| 1185 | 1190 | } |
|
| 1186 | 1191 | }; |
|
| 1187 | - | let base_snapshot = base.materialize(); |
|
| 1188 | 1192 | Ok(diff_files_of(&c, &base, &base_snapshot)) |
|
| 1189 | 1193 | } |
|
| 1190 | 1194 | ||
| … | 994 unchanged lines |
modified crates/oxy-core/src/repo/tests.rs +52 −0
| … | 5200 unchanged lines |
||
| 5201 | 5201 | let subs = repo.subrepos().unwrap(); |
|
| 5202 | 5202 | assert!(subs[0].committed, "committed now"); |
|
| 5203 | 5203 | } |
|
| 5204 | + | ||
| 5205 | + | /// Commit, diff, hunks and discard use the partial snapshot: a file |
|
| 5206 | + | /// unchanged since the manifest is neither read nor rendered, and showing |
|
| 5207 | + | /// a change renders only what it edits, the same as from a full base. |
|
| 5208 | + | #[cfg(unix)] |
|
| 5209 | + | #[test] |
|
| 5210 | + | fn commits_and_diffs_leave_unchanged_files_alone() { |
|
| 5211 | + | use std::os::unix::fs::PermissionsExt; |
|
| 5212 | + | hermetic(); |
|
| 5213 | + | let dir = tempfile::tempdir().unwrap(); |
|
| 5214 | + | let root = dir.path(); |
|
| 5215 | + | let repo = Repo::init(root).unwrap().with_home(root.join("home")); |
|
| 5216 | + | write(root, "big.txt", b"lots\nof\nlines\n"); |
|
| 5217 | + | write(root, "a.txt", b"one\ntwo\n"); |
|
| 5218 | + | repo.commit("base", &[]).unwrap(); |
|
| 5219 | + | let old = std::time::SystemTime::now() - std::time::Duration::from_secs(30); |
|
| 5220 | + | for f in ["big.txt", "a.txt"] { |
|
| 5221 | + | fs::File::options() |
|
| 5222 | + | .write(true) |
|
| 5223 | + | .open(root.join(f)) |
|
| 5224 | + | .unwrap() |
|
| 5225 | + | .set_modified(old) |
|
| 5226 | + | .unwrap(); |
|
| 5227 | + | } |
|
| 5228 | + | assert!(repo.snapshot_status("status").unwrap().statuses.is_empty()); |
|
| 5229 | + | // big.txt cannot be read at all from here on. |
|
| 5230 | + | fs::set_permissions(root.join("big.txt"), fs::Permissions::from_mode(0o000)).unwrap(); |
|
| 5231 | + | ||
| 5232 | + | write(root, "a.txt", b"one\nTWO\nthree\n"); |
|
| 5233 | + | let diff = repo.diff_files(None).unwrap(); |
|
| 5234 | + | assert_eq!( |
|
| 5235 | + | diff.iter().map(|f| f.path.as_str()).collect::<Vec<_>>(), |
|
| 5236 | + | ["a.txt"] |
|
| 5237 | + | ); |
|
| 5238 | + | let (_, hunks) = repo.hunks().unwrap(); |
|
| 5239 | + | assert!(!hunks.is_empty() && hunks.iter().all(|h| h.path == "a.txt")); |
|
| 5240 | + | let h = repo.commit("edit a", &[]).unwrap(); |
|
| 5241 | + | ||
| 5242 | + | // Showing the change needs only a.txt's graph, and renders as a full base does. |
|
| 5243 | + | let c = repo.store().get_change(&h).unwrap(); |
|
| 5244 | + | let full = repo.load_state(&c.deps).unwrap(); |
|
| 5245 | + | let want = render_change(&c, &full, &full.materialize()); |
|
| 5246 | + | assert_eq!(repo.describe_change(&h).unwrap(), want); |
|
| 5247 | + | let (base, snap) = repo.change_base(&c).unwrap(); |
|
| 5248 | + | assert!(!base.is_full() && snap.unrendered.contains("big.txt")); |
|
| 5249 | + | ||
| 5250 | + | // Discarding works the same way. |
|
| 5251 | + | write(root, "a.txt", b"scratch\n"); |
|
| 5252 | + | assert_eq!(repo.discard(&["a.txt".into()]).unwrap(), 1); |
|
| 5253 | + | assert_eq!(fs::read(root.join("a.txt")).unwrap(), b"one\nTWO\nthree\n"); |
|
| 5254 | + | fs::set_permissions(root.join("big.txt"), fs::Permissions::from_mode(0o644)).unwrap(); |
|
| 5255 | + | } |
modified docs/backlog-overnight-2.md +1 −1
| … | 105 unchanged lines |
||
| 106 | 106 | - [ ] P5c Import microsoft/vscode's full Git history with `oxy git import` into a local scratch repository (not published): about 130 000 commits as a stress test of import, log, blame and status at scale, recorded in docs/benchmarks.md. |
|
| 107 | 107 | - [x] O10f Load file graphs lazily from a version 2 checkpoint: a state keeps unread graphs by reference and reads one when first needed. |
|
| 108 | 108 | - [x] O10g Render only what a snapshot needs, first part: the manifest records, per file, the key of the base graph it matched, so an unchanged file whose graph is untouched is neither read nor rendered; used by status, log, show, tag and bookmark. |
|
| 109 | - | - [ ] O10h The same for commit, diff, amend and the other full snapshots: a lazy base that builds only the graphs the change touches; cache materialized states. |
|
| 109 | + | - [x] O10h The same for commit, diff, hunks, discard and showing a change: only the graphs the change touches are built (caching materialized states turned out unnecessary). |
|
| 110 | 110 | - [x] O5 Concurrency: two processes committing, snapshotting and pushing on one repository; locks hold, nothing is lost. |
|
| 111 | 111 | - [x] O6 Error messages audit: every user-facing error says what to do next; `oxy doctor` checks identity, config, remotes, CA files, hub reachability. |
|
| 112 | 112 | - [x] O7 Shell completions (`oxy completions bash|zsh|fish`) and man pages generated from clap. |
|
| … | 427 unchanged lines |
modified docs/benchmarks.md +19 −0
| … | 192 unchanged lines |
||
| 193 | 193 | ||
| 194 | 194 | The same for `commit` and `diff` comes next (O10h). There, only the graphs |
|
| 195 | 195 | the change touches are needed, and materialized states can be cached. |
|
| 196 | + | ||
| 197 | + | ## 2026-09-29: O10h, commit, diff and show build only what they touch |
|
| 198 | + | ||
| 199 | + | Commit, working-copy diff, hunks and discard use the partial snapshot of |
|
| 200 | + | O10g. Files known unchanged are neither read nor rendered, and only the |
|
| 201 | + | graphs of the files the working copy changes are built. Showing a |
|
| 202 | + | committed change (`oxy show`, `oxy diff CHANGE`) builds the graphs of the |
|
| 203 | + | files that change edits and leaves the rest unrendered. |
|
| 204 | + | ||
| 205 | + | Same repository as O10g (10 000 files of 40 lines): |
|
| 206 | + | ||
| 207 | + | | Operation | After O10g | After O10h | |
|
| 208 | + | |---|---:|---:| |
|
| 209 | + | | diff (working copy) | 0.57 s, 244 MB | 0.22 s, 52 MB | |
|
| 210 | + | | commit (one file) | 0.58 s, 245 MB | 0.23 s, 52 MB | |
|
| 211 | + | | show (a one-file change) | not measured | 0.53 s, 145 MB | |
|
| 212 | + | ||
| 213 | + | Caching materialized states, planned alongside, is not needed now that |
|
| 214 | + | these commands render almost nothing. |
modified src/cli/workspace.rs +3 −4
| … | 227 unchanged lines |
||
| 228 | 228 | ui::subject(subject_of(message)) |
|
| 229 | 229 | )); |
|
| 230 | 230 | if !matches!(selection, oxy_core::repo::Selection::All) { |
|
| 231 | - | let rest = repo.snapshot("oxy commit")?; |
|
| 231 | + | let rest = repo.snapshot_status("oxy commit")?; |
|
| 232 | 232 | if !rest.statuses.is_empty() { |
|
| 233 | 233 | println!( |
|
| 234 | 234 | " {} {}", |
|
| … | 410 unchanged lines |
||
| 645 | 645 | match hash { |
|
| 646 | 646 | Some(h) => { |
|
| 647 | 647 | let c = repo.store().get_change(&h)?; |
|
| 648 | - | let base = repo.load_state(&c.deps)?; |
|
| 649 | - | let snapshot = base.materialize(); |
|
| 648 | + | let (base, snapshot) = repo.change_base(&c)?; |
|
| 650 | 649 | print!("{}", render_change(&c, &base, &snapshot)); |
|
| 651 | 650 | } |
|
| 652 | 651 | None => { |
|
| 653 | - | let snap = repo.snapshot("oxy diff")?; |
|
| 652 | + | let snap = repo.snapshot_status("oxy diff")?; |
|
| 654 | 653 | print!( |
|
| 655 | 654 | "{}", |
|
| 656 | 655 | render_change(&snap.wc, &snap.base, &snap.base_snapshot) |
|
| … | 1110 unchanged lines |