OXYGENHUB

Log, blame and fsck at the scale of the Linux tree 69816d688300

log -n 100 on the Linux tip took 28 s and 4.7 GB: checking each
change's signature read the whole change, which for the first one means
all 1.8 GB of its text. Signature checks, the dependency walks of fetch
and push, and the prompt now read a change's metadata alone
(Store::get_change_meta, from its skeleton), and log takes the working
copy's state from the manifest when it proves it clean. blame took a
full snapshot, building every graph, to blame one file; it now loads the
base's history and builds that file's graph. fsck loaded each
bookmark's state whole; it now builds its graphs a batch at a time,
deleted files' included, and checks a state shared by several bookmarks
once.

Linux tip: log -n 100 2.0 s and 435 MB, blame 1.1 s and 315 MB, fsck
123 s and 4.5 GB, where blame and fsck were killed at 14 GB.

refs #35
Author
Andy McCall @andymccall <[email protected]>
When
· 2026-09-29 22:58 UTC
Signature
✔ signed by Andy McCall
Hash
69816d68830059c6155cbc8b14376c47c805a3d978e49de8daf28c648424a823
Depends on
135212de5370 1d360c91a289 2863456a0f6d 387335ff7f4d 6f8aeb996558 75167672ebcd 83d61e803ccd 86e1824f2d0b 94084087a876 957fee30826e 982864068a59 bc489b6c7f69 c5ce0367eddf cc7a00565161 e02bf0cd0533 e8d38e47e9d7 fce44ae9ad29 ff0af1ee299d
Depended on by
33472ccc08cc 98a1620a3840 2bf22305bc48 073a2115995f ce14f0d68b15 4cce912a0953 4d6c9c1b358c 8b276e0106d1
In bookmarks
agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
In tags
v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
Edits
26 atoms, 0 tree operations, 9 files

Browse the tree at this change

modified crates/oxy-core/src/repo/fsck.rs +18 −1
…
356 unchanged lines
357 357
        }
358 358
    }
359 359
360 +
    /// Whether `heads` form a valid state: its history loads and every file
361 +
    /// graph builds. Graphs are built a batch at a time and dropped, so the
362 +
    /// check needs the memory of a batch, not of the whole tree.
363 +
    fn check_state(&self, heads: &[ChangeHash]) -> Result<()> {
364 +
        let mut st = self.load_state_history(heads)?;
365 +
        let nodes = st.unbuilt_nodes();
366 +
        for batch in nodes.chunks(2048) {
367 +
            self.load_graphs_of(&mut st, batch)?;
368 +
            st.drop_graphs(batch);
369 +
        }
370 +
        Ok(())
371 +
    }
372 +
360 373
    /// Pass 4.
361 374
    fn fsck_derived(&self, rep: &mut FsckReport) -> Result<()> {
362 375
        let Ok(view) = self.view() else { return Ok(()) };
363 376
        let shallow = self.is_shallow();
364 377
        if !shallow {
378 +
            let mut checked: HashSet<ObjectRef> = HashSet::new();
365 379
            for (name, st) in &view.bookmarks {
380 +
                if !checked.insert(*st) {
381 +
                    continue;
382 +
                }
366 383
                let Ok(s) = self.store.get_state(st) else {
367 384
                    continue;
368 385
                };
369 -
                if let Err(e) = self.load_state_full(&s.heads) {
386 +
                if let Err(e) = self.check_state(&s.heads) {
370 387
                    rep.error(
371 388
                        Some(*st),
372 389
                        format!("bookmark {name} does not form a valid state: {e}"),
…
98 unchanged lines
modified crates/oxy-core/src/repo/identity.rs +3 −3
…
330 unchanged lines
331 331
332 332
    /// Check the signatures on a change against the author-or-delegate rule.
333 333
    pub fn verify_change(&self, hash: &ChangeHash) -> Result<SignatureStatus> {
334 -
        let change = self.store.get_change(hash)?;
334 +
        let change = self.store.get_change_meta(hash)?;
335 335
        let sigs = self.store.signatures(hash)?;
336 336
        if sigs.is_empty() {
337 337
            return Ok(SignatureStatus::Unsigned);
…
19 unchanged lines
357 357
    /// Verify `sig` over `target` and check the author-or-delegate rule for `change`.
358 358
    fn check_signer(
359 359
        &self,
360 -
        change: &Change,
360 +
        change: &crate::state::ChangeMeta,
361 361
        sig: &Signature,
362 362
        domain: Domain,
363 363
        target: &ObjectRef,
…
36 unchanged lines
400 400
        if matches!(direct, SignatureStatus::Valid { .. }) {
401 401
            return Ok(direct);
402 402
        }
403 -
        let change = self.store.get_change(hash)?;
403 +
        let change = self.store.get_change_meta(hash)?;
404 404
        let mut last_err = match &direct {
405 405
            SignatureStatus::Invalid(e) => e.clone(),
406 406
            _ => String::new(),
…
102 unchanged lines
modified crates/oxy-core/src/repo/inspect.rs +8 −9
…
58 unchanged lines
59 59
    /// The introducing change, author, and date of every line of `path` in
60 60
    /// the working copy's base state (spec §27).
61 61
    pub fn blame(&self, path: &str) -> Result<Vec<BlameLine>> {
62 -
        let snap = self.snapshot("oxy blame")?;
63 -
        self.blame_state(
64 -
            path,
65 -
            &snap.base,
66 -
            &snap.base_snapshot.tree,
67 -
            "the working copy's base state",
68 -
        )
62 +
        let (_, wc) = self.working_copy()?;
63 +
        self.blame_heads(path, &wc.deps, "the working copy's base state")
69 64
    }
70 65
71 66
    /// Blame `path` in the state with these heads (hosted repositories).
72 67
    pub fn blame_in(&self, path: &str, heads: &[ChangeHash]) -> Result<Vec<BlameLine>> {
73 -
        // Only the one file's graph is built.
68 +
        self.blame_heads(path, heads, "that state")
69 +
    }
70 +
71 +
    /// Blame `path` in the state with these heads, building only its graph.
72 +
    fn blame_heads(&self, path: &str, heads: &[ChangeHash], what: &str) -> Result<Vec<BlameLine>> {
74 73
        let mut state = self.load_state_history(heads)?;
75 74
        let tree = state.tree().view();
76 75
        if let Some(e) = tree.entries.get(path) {
77 76
            self.load_graphs_of(&mut state, &[e.node])?;
78 77
        }
79 -
        self.blame_of(&state, &tree, path)
78 +
        self.blame_state(path, &state, &tree, what)
80 79
    }
81 80
82 81
    /// Blame `path` in a loaded state whose tree view the caller holds; no
…
386 unchanged lines
modified crates/oxy-core/src/repo/prompt.rs +1 −1
…
427 unchanged lines
428 428
        let heads = self.heads_of(&view, &bookmark)?;
429 429
        let wc_hash = view.workspaces.get(&self.workspace).copied();
430 430
        let change_id = match wc_hash {
431 -
            Some(h) => self.store.get_change(&h)?.change_id.short(8),
431 +
            Some(h) => self.store.get_change_meta(&h)?.change_id.short(8),
432 432
            None => String::new(),
433 433
        };
434 434
        // Conflictedness was recorded by the last snapshot; without a
…
64 unchanged lines
modified crates/oxy-core/src/repo/sync.rs +3 −3
…
114 unchanged lines
115 115
                continue;
116 116
            }
117 117
            seen.insert(h);
118 -
            stack.extend(self.store.get_change(&h)?.deps);
118 +
            stack.extend(self.store.get_change_meta(&h)?.deps);
119 119
        }
120 120
        Ok(seen)
121 121
    }
…
56 unchanged lines
178 178
        let mut boundary = self.boundary();
179 179
        boundary.retain(|h| !self.store.has(h));
180 180
        for h in imported {
181 -
            for d in self.store.get_change(h)?.deps {
181 +
            for d in self.store.get_change_meta(h)?.deps {
182 182
                if !self.store.has(&d) {
183 183
                    boundary.insert(d);
184 184
                }
…
43 unchanged lines
228 228
                continue;
229 229
            }
230 230
            stack.push((h, true));
231 -
            for d in self.store.get_change(&h)?.deps {
231 +
            for d in self.store.get_change_meta(&h)?.deps {
232 232
                if !done.contains(&d) {
233 233
                    stack.push((d, false));
234 234
                }
…
1411 unchanged lines
modified crates/oxy-core/src/state.rs +9 −0
…
367 unchanged lines
368 368
        Ok(())
369 369
    }
370 370
371 +
    /// Every file graph a history-only state has not built, deleted files'
372 +
    /// included.
373 +
    pub fn unbuilt_nodes(&self) -> Vec<NodeId> {
374 +
        self.deferred
375 +
            .as_ref()
376 +
            .map(|d| d.keys().copied().collect())
377 +
            .unwrap_or_default()
378 +
    }
379 +
371 380
    /// Drop the built graphs of `nodes`, for a caller rendering a large
372 381
    /// tree a batch at a time. The state can no longer render those files,
373 382
    /// so it must not be used for them again.
…
1493 unchanged lines
modified crates/oxy-core/src/store.rs +7 −0
…
943 unchanged lines
944 944
        Ok((c, texts))
945 945
    }
946 946
947 +
    /// A change's metadata (id, dependencies, author, clock, message, kind)
948 +
    /// without its contents: read from its skeleton, so neither its text
949 +
    /// blobs nor its inline lines are read (spec §11).
950 +
    pub fn get_change_meta(&self, r: &ObjectRef) -> Result<crate::state::ChangeMeta> {
951 +
        Ok(crate::state::ChangeMeta::of(&self.get_change_lazy(r)?.0))
952 +
    }
953 +
947 954
    fn read_text(&self, r: &ObjectRef) -> Result<(Blob, Vec<u8>)> {
948 955
        let blob = Blob::decode(&self.get_typed(r, ObjectType::Blob)?)?;
949 956
        let mut content = Vec::with_capacity(blob.length as usize);
…
751 unchanged lines
modified docs/benchmarks.md +21 −0
…
322 unchanged lines
323 323
history each decode the one huge first change (1.2 s and 364 MB each).
324 324
A clone holds the whole bundle in memory.
325 325
326 +
### log, blame and fsck (2026-09-30, oxygen#35)
327 +
328 +
The first full run of `~/bench/linux-bench.sh` against this work found
329 +
`log -n 100` taking 28 s and 4.7 GB, and `blame` killed at 14 GB:
330 +
- **log.** Checking each change's signature read the whole change, and
331 +
  for the first change that meant all 1.8 GB of its text. Signature
332 +
  checks, the dependency walks of fetch and push, and the prompt now
333 +
  read a change's metadata from its skeleton, and log answers the
334 +
  working copy's state from the manifest when it proves it clean.
335 +
- **blame.** It took a full snapshot, building every graph, to blame one
336 +
  file; it now loads the base's history and builds that file's graph.
337 +
- **fsck.** A bookmark's state was checked by loading it whole. It is
338 +
  now checked a batch of graphs at a time, deleted files' graphs
339 +
  included.
340 +
341 +
| Linux tip | Before | Now | Git |
342 +
|---|---:|---:|---:|
343 +
| `log -n 100` | 28 s, 4.7 GB | 2.0 s, 435 MB | 0.16 s |
344 +
| `blame MAINTAINERS` | killed at 14 GB | 1.1 s, 315 MB | 0.31 s |
345 +
| `fsck` | killed at 14 GB | 123 s, 4.5 GB | to measure again |
346 +
modified src/cli/workspace.rs +12 −4
…
493 unchanged lines
494 494
    as_json: bool,
495 495
) -> Result<i32> {
496 496
    let repo = open()?;
497 -
    let snap = repo.snapshot_status("oxy log")?;
498 -
    let conflicted = snap.base_snapshot.is_conflicted();
497 +
    // Whether the working copy is clean and conflicted: from the manifest
498 +
    // when it proves the working copy clean, else from a snapshot.
499 +
    let (conflicted, clean) = if repo.clean_by_manifest()? {
500 +
        (false, true)
501 +
    } else {
502 +
        let snap = repo.snapshot_status("oxy log")?;
503 +
        (snap.base_snapshot.is_conflicted(), snap.statuses.is_empty())
504 +
    };
499 505
    let entries = match (path, lines) {
500 506
        (_, Some(spec)) => {
501 507
            let (first, last, p) = parse_line_range(spec)?;
502 -
            repo.line_history(&snap.base.maximal(&snap.wc.deps), p, first, last)?
508 +
            let (_, wc) = repo.working_copy()?;
509 +
            let base = repo.load_state_history(&wc.deps)?.maximal(&wc.deps);
510 +
            repo.line_history(&base, p, first, last)?
503 511
        }
504 512
        (Some(p), None) => repo.log_for_path(p)?,
505 513
        (None, None) => repo.log()?,
…
42 unchanged lines
548 556
            )
549 557
        };
550 558
        let subject = if e.is_working_copy && c.message.is_empty() {
551 -
            ui::dim(if snap.statuses.is_empty() {
559 +
            ui::dim(if clean {
552 560
                "(clean)"
553 561
            } else {
554 562
                "(uncommitted changes)"
…
1294 unchanged lines