Log, blame and fsck at the scale of the Linux tree 69816d688300
log -n 100 on the Linux tip took 28 s and 4.7 GB: checking each change's signature read the whole change, which for the first one means all 1.8 GB of its text. Signature checks, the dependency walks of fetch and push, and the prompt now read a change's metadata alone (Store::get_change_meta, from its skeleton), and log takes the working copy's state from the manifest when it proves it clean. blame took a full snapshot, building every graph, to blame one file; it now loads the base's history and builds that file's graph. fsck loaded each bookmark's state whole; it now builds its graphs a batch at a time, deleted files' included, and checks a state shared by several bookmarks once. Linux tip: log -n 100 2.0 s and 435 MB, blame 1.1 s and 315 MB, fsck 123 s and 4.5 GB, where blame and fsck were killed at 14 GB. refs #35
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-29 22:58 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
69816d68830059c6155cbc8b14376c47c805a3d978e49de8daf28c648424a823- Depends on
- 135212de5370 1d360c91a289 2863456a0f6d 387335ff7f4d 6f8aeb996558 75167672ebcd 83d61e803ccd 86e1824f2d0b 94084087a876 957fee30826e 982864068a59 bc489b6c7f69 c5ce0367eddf cc7a00565161 e02bf0cd0533 e8d38e47e9d7 fce44ae9ad29 ff0af1ee299d
- Depended on by
- 33472ccc08cc 98a1620a3840 2bf22305bc48 073a2115995f ce14f0d68b15 4cce912a0953 4d6c9c1b358c 8b276e0106d1
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 26 atoms, 0 tree operations, 9 files
Browse the tree at this change
modified crates/oxy-core/src/repo/fsck.rs +18 −1
| … | 356 unchanged lines |
||
| 357 | 357 | } |
|
| 358 | 358 | } |
|
| 359 | 359 | ||
| 360 | + | /// Whether `heads` form a valid state: its history loads and every file |
|
| 361 | + | /// graph builds. Graphs are built a batch at a time and dropped, so the |
|
| 362 | + | /// check needs the memory of a batch, not of the whole tree. |
|
| 363 | + | fn check_state(&self, heads: &[ChangeHash]) -> Result<()> { |
|
| 364 | + | let mut st = self.load_state_history(heads)?; |
|
| 365 | + | let nodes = st.unbuilt_nodes(); |
|
| 366 | + | for batch in nodes.chunks(2048) { |
|
| 367 | + | self.load_graphs_of(&mut st, batch)?; |
|
| 368 | + | st.drop_graphs(batch); |
|
| 369 | + | } |
|
| 370 | + | Ok(()) |
|
| 371 | + | } |
|
| 372 | + | ||
| 360 | 373 | /// Pass 4. |
|
| 361 | 374 | fn fsck_derived(&self, rep: &mut FsckReport) -> Result<()> { |
|
| 362 | 375 | let Ok(view) = self.view() else { return Ok(()) }; |
|
| 363 | 376 | let shallow = self.is_shallow(); |
|
| 364 | 377 | if !shallow { |
|
| 378 | + | let mut checked: HashSet<ObjectRef> = HashSet::new(); |
|
| 365 | 379 | for (name, st) in &view.bookmarks { |
|
| 380 | + | if !checked.insert(*st) { |
|
| 381 | + | continue; |
|
| 382 | + | } |
|
| 366 | 383 | let Ok(s) = self.store.get_state(st) else { |
|
| 367 | 384 | continue; |
|
| 368 | 385 | }; |
|
| 369 | - | if let Err(e) = self.load_state_full(&s.heads) { |
|
| 386 | + | if let Err(e) = self.check_state(&s.heads) { |
|
| 370 | 387 | rep.error( |
|
| 371 | 388 | Some(*st), |
|
| 372 | 389 | format!("bookmark {name} does not form a valid state: {e}"), |
|
| … | 98 unchanged lines |
modified crates/oxy-core/src/repo/identity.rs +3 −3
| … | 330 unchanged lines |
||
| 331 | 331 | ||
| 332 | 332 | /// Check the signatures on a change against the author-or-delegate rule. |
|
| 333 | 333 | pub fn verify_change(&self, hash: &ChangeHash) -> Result<SignatureStatus> { |
|
| 334 | - | let change = self.store.get_change(hash)?; |
|
| 334 | + | let change = self.store.get_change_meta(hash)?; |
|
| 335 | 335 | let sigs = self.store.signatures(hash)?; |
|
| 336 | 336 | if sigs.is_empty() { |
|
| 337 | 337 | return Ok(SignatureStatus::Unsigned); |
|
| … | 19 unchanged lines |
||
| 357 | 357 | /// Verify `sig` over `target` and check the author-or-delegate rule for `change`. |
|
| 358 | 358 | fn check_signer( |
|
| 359 | 359 | &self, |
|
| 360 | - | change: &Change, |
|
| 360 | + | change: &crate::state::ChangeMeta, |
|
| 361 | 361 | sig: &Signature, |
|
| 362 | 362 | domain: Domain, |
|
| 363 | 363 | target: &ObjectRef, |
|
| … | 36 unchanged lines |
||
| 400 | 400 | if matches!(direct, SignatureStatus::Valid { .. }) { |
|
| 401 | 401 | return Ok(direct); |
|
| 402 | 402 | } |
|
| 403 | - | let change = self.store.get_change(hash)?; |
|
| 403 | + | let change = self.store.get_change_meta(hash)?; |
|
| 404 | 404 | let mut last_err = match &direct { |
|
| 405 | 405 | SignatureStatus::Invalid(e) => e.clone(), |
|
| 406 | 406 | _ => String::new(), |
|
| … | 102 unchanged lines |
modified crates/oxy-core/src/repo/inspect.rs +8 −9
| … | 58 unchanged lines |
||
| 59 | 59 | /// The introducing change, author, and date of every line of `path` in |
|
| 60 | 60 | /// the working copy's base state (spec §27). |
|
| 61 | 61 | pub fn blame(&self, path: &str) -> Result<Vec<BlameLine>> { |
|
| 62 | - | let snap = self.snapshot("oxy blame")?; |
|
| 63 | - | self.blame_state( |
|
| 64 | - | path, |
|
| 65 | - | &snap.base, |
|
| 66 | - | &snap.base_snapshot.tree, |
|
| 67 | - | "the working copy's base state", |
|
| 68 | - | ) |
|
| 62 | + | let (_, wc) = self.working_copy()?; |
|
| 63 | + | self.blame_heads(path, &wc.deps, "the working copy's base state") |
|
| 69 | 64 | } |
|
| 70 | 65 | ||
| 71 | 66 | /// Blame `path` in the state with these heads (hosted repositories). |
|
| 72 | 67 | pub fn blame_in(&self, path: &str, heads: &[ChangeHash]) -> Result<Vec<BlameLine>> { |
|
| 73 | - | // Only the one file's graph is built. |
|
| 68 | + | self.blame_heads(path, heads, "that state") |
|
| 69 | + | } |
|
| 70 | + | ||
| 71 | + | /// Blame `path` in the state with these heads, building only its graph. |
|
| 72 | + | fn blame_heads(&self, path: &str, heads: &[ChangeHash], what: &str) -> Result<Vec<BlameLine>> { |
|
| 74 | 73 | let mut state = self.load_state_history(heads)?; |
|
| 75 | 74 | let tree = state.tree().view(); |
|
| 76 | 75 | if let Some(e) = tree.entries.get(path) { |
|
| 77 | 76 | self.load_graphs_of(&mut state, &[e.node])?; |
|
| 78 | 77 | } |
|
| 79 | - | self.blame_of(&state, &tree, path) |
|
| 78 | + | self.blame_state(path, &state, &tree, what) |
|
| 80 | 79 | } |
|
| 81 | 80 | ||
| 82 | 81 | /// Blame `path` in a loaded state whose tree view the caller holds; no |
|
| … | 386 unchanged lines |
modified crates/oxy-core/src/repo/prompt.rs +1 −1
| … | 427 unchanged lines |
||
| 428 | 428 | let heads = self.heads_of(&view, &bookmark)?; |
|
| 429 | 429 | let wc_hash = view.workspaces.get(&self.workspace).copied(); |
|
| 430 | 430 | let change_id = match wc_hash { |
|
| 431 | - | Some(h) => self.store.get_change(&h)?.change_id.short(8), |
|
| 431 | + | Some(h) => self.store.get_change_meta(&h)?.change_id.short(8), |
|
| 432 | 432 | None => String::new(), |
|
| 433 | 433 | }; |
|
| 434 | 434 | // Conflictedness was recorded by the last snapshot; without a |
|
| … | 64 unchanged lines |
modified crates/oxy-core/src/repo/sync.rs +3 −3
| … | 114 unchanged lines |
||
| 115 | 115 | continue; |
|
| 116 | 116 | } |
|
| 117 | 117 | seen.insert(h); |
|
| 118 | - | stack.extend(self.store.get_change(&h)?.deps); |
|
| 118 | + | stack.extend(self.store.get_change_meta(&h)?.deps); |
|
| 119 | 119 | } |
|
| 120 | 120 | Ok(seen) |
|
| 121 | 121 | } |
|
| … | 56 unchanged lines |
||
| 178 | 178 | let mut boundary = self.boundary(); |
|
| 179 | 179 | boundary.retain(|h| !self.store.has(h)); |
|
| 180 | 180 | for h in imported { |
|
| 181 | - | for d in self.store.get_change(h)?.deps { |
|
| 181 | + | for d in self.store.get_change_meta(h)?.deps { |
|
| 182 | 182 | if !self.store.has(&d) { |
|
| 183 | 183 | boundary.insert(d); |
|
| 184 | 184 | } |
|
| … | 43 unchanged lines |
||
| 228 | 228 | continue; |
|
| 229 | 229 | } |
|
| 230 | 230 | stack.push((h, true)); |
|
| 231 | - | for d in self.store.get_change(&h)?.deps { |
|
| 231 | + | for d in self.store.get_change_meta(&h)?.deps { |
|
| 232 | 232 | if !done.contains(&d) { |
|
| 233 | 233 | stack.push((d, false)); |
|
| 234 | 234 | } |
|
| … | 1411 unchanged lines |
modified crates/oxy-core/src/state.rs +9 −0
| … | 367 unchanged lines |
||
| 368 | 368 | Ok(()) |
|
| 369 | 369 | } |
|
| 370 | 370 | ||
| 371 | + | /// Every file graph a history-only state has not built, deleted files' |
|
| 372 | + | /// included. |
|
| 373 | + | pub fn unbuilt_nodes(&self) -> Vec<NodeId> { |
|
| 374 | + | self.deferred |
|
| 375 | + | .as_ref() |
|
| 376 | + | .map(|d| d.keys().copied().collect()) |
|
| 377 | + | .unwrap_or_default() |
|
| 378 | + | } |
|
| 379 | + | ||
| 371 | 380 | /// Drop the built graphs of `nodes`, for a caller rendering a large |
|
| 372 | 381 | /// tree a batch at a time. The state can no longer render those files, |
|
| 373 | 382 | /// so it must not be used for them again. |
|
| … | 1493 unchanged lines |
modified crates/oxy-core/src/store.rs +7 −0
| … | 943 unchanged lines |
||
| 944 | 944 | Ok((c, texts)) |
|
| 945 | 945 | } |
|
| 946 | 946 | ||
| 947 | + | /// A change's metadata (id, dependencies, author, clock, message, kind) |
|
| 948 | + | /// without its contents: read from its skeleton, so neither its text |
|
| 949 | + | /// blobs nor its inline lines are read (spec §11). |
|
| 950 | + | pub fn get_change_meta(&self, r: &ObjectRef) -> Result<crate::state::ChangeMeta> { |
|
| 951 | + | Ok(crate::state::ChangeMeta::of(&self.get_change_lazy(r)?.0)) |
|
| 952 | + | } |
|
| 953 | + | ||
| 947 | 954 | fn read_text(&self, r: &ObjectRef) -> Result<(Blob, Vec<u8>)> { |
|
| 948 | 955 | let blob = Blob::decode(&self.get_typed(r, ObjectType::Blob)?)?; |
|
| 949 | 956 | let mut content = Vec::with_capacity(blob.length as usize); |
|
| … | 751 unchanged lines |
modified docs/benchmarks.md +21 −0
| … | 322 unchanged lines |
||
| 323 | 323 | history each decode the one huge first change (1.2 s and 364 MB each). |
|
| 324 | 324 | A clone holds the whole bundle in memory. |
|
| 325 | 325 | ||
| 326 | + | ### log, blame and fsck (2026-09-30, oxygen#35) |
|
| 327 | + | ||
| 328 | + | The first full run of `~/bench/linux-bench.sh` against this work found |
|
| 329 | + | `log -n 100` taking 28 s and 4.7 GB, and `blame` killed at 14 GB: |
|
| 330 | + | - **log.** Checking each change's signature read the whole change, and |
|
| 331 | + | for the first change that meant all 1.8 GB of its text. Signature |
|
| 332 | + | checks, the dependency walks of fetch and push, and the prompt now |
|
| 333 | + | read a change's metadata from its skeleton, and log answers the |
|
| 334 | + | working copy's state from the manifest when it proves it clean. |
|
| 335 | + | - **blame.** It took a full snapshot, building every graph, to blame one |
|
| 336 | + | file; it now loads the base's history and builds that file's graph. |
|
| 337 | + | - **fsck.** A bookmark's state was checked by loading it whole. It is |
|
| 338 | + | now checked a batch of graphs at a time, deleted files' graphs |
|
| 339 | + | included. |
|
| 340 | + | ||
| 341 | + | | Linux tip | Before | Now | Git | |
|
| 342 | + | |---|---:|---:|---:| |
|
| 343 | + | | `log -n 100` | 28 s, 4.7 GB | 2.0 s, 435 MB | 0.16 s | |
|
| 344 | + | | `blame MAINTAINERS` | killed at 14 GB | 1.1 s, 315 MB | 0.31 s | |
|
| 345 | + | | `fsck` | killed at 14 GB | 123 s, 4.5 GB | to measure again | |
|
| 346 | + |
modified src/cli/workspace.rs +12 −4
| … | 493 unchanged lines |
||
| 494 | 494 | as_json: bool, |
|
| 495 | 495 | ) -> Result<i32> { |
|
| 496 | 496 | let repo = open()?; |
|
| 497 | - | let snap = repo.snapshot_status("oxy log")?; |
|
| 498 | - | let conflicted = snap.base_snapshot.is_conflicted(); |
|
| 497 | + | // Whether the working copy is clean and conflicted: from the manifest |
|
| 498 | + | // when it proves the working copy clean, else from a snapshot. |
|
| 499 | + | let (conflicted, clean) = if repo.clean_by_manifest()? { |
|
| 500 | + | (false, true) |
|
| 501 | + | } else { |
|
| 502 | + | let snap = repo.snapshot_status("oxy log")?; |
|
| 503 | + | (snap.base_snapshot.is_conflicted(), snap.statuses.is_empty()) |
|
| 504 | + | }; |
|
| 499 | 505 | let entries = match (path, lines) { |
|
| 500 | 506 | (_, Some(spec)) => { |
|
| 501 | 507 | let (first, last, p) = parse_line_range(spec)?; |
|
| 502 | - | repo.line_history(&snap.base.maximal(&snap.wc.deps), p, first, last)? |
|
| 508 | + | let (_, wc) = repo.working_copy()?; |
|
| 509 | + | let base = repo.load_state_history(&wc.deps)?.maximal(&wc.deps); |
|
| 510 | + | repo.line_history(&base, p, first, last)? |
|
| 503 | 511 | } |
|
| 504 | 512 | (Some(p), None) => repo.log_for_path(p)?, |
|
| 505 | 513 | (None, None) => repo.log()?, |
|
| … | 42 unchanged lines |
||
| 548 | 556 | ) |
|
| 549 | 557 | }; |
|
| 550 | 558 | let subject = if e.is_working_copy && c.message.is_empty() { |
|
| 551 | - | ui::dim(if snap.statuses.is_empty() { |
|
| 559 | + | ui::dim(if clean { |
|
| 552 | 560 | "(clean)" |
|
| 553 | 561 | } else { |
|
| 554 | 562 | "(uncommitted changes)" |
|
| … | 1294 unchanged lines |