Commit and status no longer slow down with the number of heads 07eb99be126e
A bookmark of independent changes has many heads, and State::maximal compared every pair of them: quadratic, about 60 ms at 900 heads and 1.5 s at 4 500, run once per snapshot and twice more per commit. It is now one walk down from all the candidates, linear in the changes reached, and a property test holds it to the old definition on random graphs. Commit also reuses the state the snapshot loaded instead of loading it a second time. At 1 000 files and 1 500 changes a commit now takes 86 ms on average, flat over the whole history.
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-28 06:08 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
07eb99be126e579a08cb5868e51ce36f01e8d90ad7fc088fdcb82b06d18cd109- Depends on
- 13a0f400dff9 59171dcff221 75036530b621 75167672ebcd 982864068a59 ff0af1ee299d
- Depended on by
- ac03130a2b95 192fad1e46a2 e7ddd552b619
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 8 atoms, 0 tree operations, 3 files
Browse the tree at this change
modified crates/oxy-core/src/repo/snapshot.rs +7 −1
| … | 488 unchanged lines |
||
| 489 | 489 | let mut all: Vec<ChangeHash> = old_heads.clone(); |
|
| 490 | 490 | all.extend(old_base.iter().copied()); |
|
| 491 | 491 | all.push(hash); |
|
| 492 | - | let mut st = self.load_state(&all)?; |
|
| 492 | + | // The snapshot already loaded the working copy's base; when that also |
|
| 493 | + | // holds every bookmark head, it is the state to extend. |
|
| 494 | + | let mut st = if old_heads.iter().all(|h| snap.base.contains(h)) { |
|
| 495 | + | snap.base |
|
| 496 | + | } else { |
|
| 497 | + | self.load_state(&all)? |
|
| 498 | + | }; |
|
| 493 | 499 | st.add(change)?; |
|
| 494 | 500 | let mut with_new = old_heads.clone(); |
|
| 495 | 501 | with_new.push(hash); |
|
| … | 1495 unchanged lines |
modified crates/oxy-core/src/state.rs +49 −5
| … | 414 unchanged lines |
||
| 415 | 415 | heads |
|
| 416 | 416 | } |
|
| 417 | 417 | ||
| 418 | - | /// Maximal elements among `hashes` (those not in the closure of another). |
|
| 418 | + | /// Maximal elements among `hashes` (those not in the closure of another), |
|
| 419 | + | /// sorted. One walk down from all of them at once: whatever the walk |
|
| 420 | + | /// reaches is below some other element. Linear in the changes reached, |
|
| 421 | + | /// where comparing every pair's closures was quadratic in the number |
|
| 422 | + | /// of heads, which a bookmark of independent changes has many of. |
|
| 419 | 423 | pub fn maximal(&self, hashes: &[ChangeHash]) -> Vec<ChangeHash> { |
|
| 420 | 424 | let set: BTreeSet<ChangeHash> = hashes.iter().copied().collect(); |
|
| 421 | - | set.iter() |
|
| 422 | - | .filter(|h| !set.iter().any(|o| o != *h && self.closure(o).contains(h))) |
|
| 423 | - | .copied() |
|
| 424 | - | .collect() |
|
| 425 | + | let mut below: HashSet<ChangeHash> = HashSet::new(); |
|
| 426 | + | let mut stack: Vec<ChangeHash> = set |
|
| 427 | + | .iter() |
|
| 428 | + | .filter_map(|h| self.changes.get(h)) |
|
| 429 | + | .flat_map(|c| c.deps.iter().copied()) |
|
| 430 | + | .collect(); |
|
| 431 | + | while let Some(d) = stack.pop() { |
|
| 432 | + | if below.insert(d) { |
|
| 433 | + | if let Some(c) = self.changes.get(&d) { |
|
| 434 | + | stack.extend(c.deps.iter().copied()); |
|
| 435 | + | } |
|
| 436 | + | } |
|
| 437 | + | } |
|
| 438 | + | set.into_iter().filter(|h| !below.contains(h)).collect() |
|
| 425 | 439 | } |
|
| 426 | 440 | ||
| 427 | 441 | pub fn materialize_file(&self, node: &NodeId) -> Materialized { |
|
| … | 433 unchanged lines |
||
| 861 | 875 | assert_eq!(m, want); |
|
| 862 | 876 | assert_eq!(st.heads(), want); |
|
| 863 | 877 | } |
|
| 878 | + | ||
| 879 | + | /// The one-walk `maximal` agrees with the definition (not in the |
|
| 880 | + | /// closure of another element) on random graphs, with repeated and |
|
| 881 | + | /// unknown hashes among the input. |
|
| 882 | + | #[test] |
|
| 883 | + | fn maximal_matches_its_definition() { |
|
| 884 | + | use rand::{Rng, SeedableRng}; |
|
| 885 | + | let mut rng = rand::rngs::StdRng::seed_from_u64(7); |
|
| 886 | + | for _ in 0..20 { |
|
| 887 | + | let mut st = State::new(); |
|
| 888 | + | let mut all: Vec<ChangeHash> = Vec::new(); |
|
| 889 | + | for _ in 0..60 { |
|
| 890 | + | let deps: Vec<ChangeHash> = |
|
| 891 | + | all.iter().copied().filter(|_| rng.gen_bool(0.05)).collect(); |
|
| 892 | + | all.push(st.add(change(deps, vec![], vec![])).unwrap()); |
|
| 893 | + | } |
|
| 894 | + | let mut pick: Vec<ChangeHash> = |
|
| 895 | + | all.iter().copied().filter(|_| rng.gen_bool(0.4)).collect(); |
|
| 896 | + | pick.extend(pick.clone().into_iter().take(3)); |
|
| 897 | + | pick.push(ObjectRef([0xee; 32])); |
|
| 898 | + | let set: BTreeSet<ChangeHash> = pick.iter().copied().collect(); |
|
| 899 | + | let want: Vec<ChangeHash> = set |
|
| 900 | + | .iter() |
|
| 901 | + | .filter(|h| !set.iter().any(|o| o != *h && st.closure(o).contains(h))) |
|
| 902 | + | .copied() |
|
| 903 | + | .collect(); |
|
| 904 | + | assert_eq!(st.maximal(&pick), want); |
|
| 905 | + | assert_eq!(st.maximal(&all), st.heads()); |
|
| 906 | + | } |
|
| 907 | + | } |
|
| 864 | 908 | } |
modified docs/backlog-overnight-2.md +2 −1
| … | 43 unchanged lines |
||
| 44 | 44 | - [x] O2 `oxy fsck`: verify every object's hash, every change's references and signatures, the op log chain, and report precisely. |
|
| 45 | 45 | - [x] O3 Crash safety audit: every write goes through write_atomic or the store's batch; a test that interrupts commit and push at each step and proves the repository still opens and fscks. |
|
| 46 | 46 | - [x] O4 Benchmarks with thresholds: 10 000 files, 5 000 changes; status, commit, log, blame, push, clone; record numbers in docs. |
|
| 47 | - | - [ ] O10 Snapshot fast path (found by O4; do next on the oxy track): at 10 000 files and 5 000 changes a commit takes 3.8 s and a clean status 3.0 s, because every snapshot loads and materializes the whole base state (about 75 ms load, 75 ms materialize, 50 ms walk at 4 000 files). Skip load and materialize when every file's stat matches a manifest written for the same base; make the base state lazy for callers that need it; cache the materialized snapshot per state. Target the spec §25 numbers. |
|
| 47 | + | - [x] O10a Heads at scale (first part of O10): `State::maximal` compared every pair of heads (quadratic; about 1.5 s per call at 4 500 heads, called three times per commit); now one walk. Commit reuses the snapshot's base state instead of loading it again. |
|
| 48 | + | - [ ] O10b Snapshot fast path (rest of O10): at 10 000 files and 5 000 changes a commit takes 3.8 s and a clean status 3.0 s, because every snapshot loads and materializes the whole base state (about 75 ms load, 75 ms materialize, 50 ms walk at 4 000 files). Skip load and materialize when every file's stat matches a manifest written for the same base; make the base state lazy for callers that need it; cache the materialized snapshot per state. Target the spec §25 numbers. |
|
| 48 | 49 | - [ ] O5 Concurrency: two processes committing, snapshotting and pushing on one repository; locks hold, nothing is lost. |
|
| 49 | 50 | - [ ] O6 Error messages audit: every user-facing error says what to do next; `oxy doctor` checks identity, config, remotes, CA files, hub reachability. |
|
| 50 | 51 | - [ ] O7 Shell completions (`oxy completions bash|zsh|fish`) and man pages generated from clap. |
|
| … | 14 unchanged lines |