OXYGENHUB

Log from the state index, signature marks without reading changes (refs #41) 4cce912a0953

The state index's changes are now a block of their own with its own hash,
so the log reads and checks them without the rest; changes outside the
index's closure are still read from the store (test builds compare the
two). A log line's signature mark looks for signatures before reading
anything and uses the metadata the log already has, rather than reading
each change whole: on the Linux tree, where the first change holds the
whole tree, log -n 100 printed in 336 ms and now in under 1 ms.
Author
Andy McCall @andymccall <[email protected]>
When
· 2026-09-30 21:52 UTC
Signature
✔ signed by Andy McCall
Hash
4cce912a095345370b211b5340679792badd3c7dc2d302e97385711cf6143907
Depends on
05059d434aca 135212de5370 3f1b8fe25ec9 4d6c9c1b358c 69816d688300 75167672ebcd 83d61e803ccd 94084087a876 982864068a59 c6aa5bd9d1c8 cc7a00565161 f9575918477f fce44ae9ad29 ff0af1ee299d
Depended on by
74b4a3c62ea4 bf0247ee3590 ac03130a2b95 5552e69c7325 79b5a90ffd43 5cde7882d767 33472ccc08cc 2bf22305bc48 638fb21420d7 f9466021286a ce14f0d68b15
In bookmarks
agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
In tags
v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
Edits
36 atoms, 0 tree operations, 5 files

Browse the tree at this change

modified crates/oxy-core/src/repo/identity.rs +48 −6
…
330 unchanged lines
331 331
332 332
    /// Check the signatures on a change against the author-or-delegate rule.
333 333
    pub fn verify_change(&self, hash: &ChangeHash) -> Result<SignatureStatus> {
334 -
        let change = self.store.get_change_meta(hash)?;
334 +
        self.verify_change_known(hash, None)
335 +
    }
336 +
337 +
    /// `verify_change`, with the change's metadata when the caller has it
338 +
    /// (a change object can be as large as the tree it adds). Signatures
339 +
    /// are looked up first: an unsigned change needs no metadata.
340 +
    fn verify_change_known(
341 +
        &self,
342 +
        hash: &ChangeHash,
343 +
        known: Option<&crate::state::ChangeMeta>,
344 +
    ) -> Result<SignatureStatus> {
335 345
        let sigs = self.store.signatures(hash)?;
336 346
        if sigs.is_empty() {
337 347
            return Ok(SignatureStatus::Unsigned);
338 348
        }
349 +
        let read;
350 +
        let change = match known {
351 +
            Some(m) => m,
352 +
            None => {
353 +
                read = self.store.get_change_meta(hash)?;
354 +
                &read
355 +
            }
356 +
        };
339 357
        let mut last_err = String::new();
340 358
        for sig in &sigs {
341 -
            match self.check_signer(&change, sig, Domain::Change, hash) {
359 +
            match self.check_signer(change, sig, Domain::Change, hash) {
342 360
                Ok(id) => {
343 361
                    return Ok(SignatureStatus::Valid {
344 362
                        key: sig.key,
…
51 unchanged lines
396 414
    /// Like `verify_change`, but a change with no valid individual signature
397 415
    /// is also authenticated by a valid seal covering it (spec §14.4).
398 416
    pub fn authenticate_change(&self, hash: &ChangeHash) -> Result<SignatureStatus> {
399 -
        let direct = self.verify_change(hash)?;
417 +
        self.authenticate_change_known(hash, None)
418 +
    }
419 +
420 +
    /// `authenticate_change`, with the change's metadata when the caller
421 +
    /// has it (from a log, say), so the change itself need not be read.
422 +
    pub fn authenticate_change_known(
423 +
        &self,
424 +
        hash: &ChangeHash,
425 +
        known: Option<&crate::state::ChangeMeta>,
426 +
    ) -> Result<SignatureStatus> {
427 +
        let direct = self.verify_change_known(hash, known)?;
400 428
        if matches!(direct, SignatureStatus::Valid { .. }) {
401 429
            return Ok(direct);
402 430
        }
403 -
        let change = self.store.get_change_meta(hash)?;
404 431
        let mut last_err = match &direct {
405 432
            SignatureStatus::Invalid(e) => e.clone(),
406 433
            _ => String::new(),
434 +
        };
435 +
        let seals = self.store.seals_for(hash)?;
436 +
        if seals.is_empty() {
437 +
            return Ok(match last_err.is_empty() {
438 +
                true => SignatureStatus::Unsigned,
439 +
                false => SignatureStatus::Invalid(last_err),
440 +
            });
441 +
        }
442 +
        let read;
443 +
        let change = match known {
444 +
            Some(m) => m,
445 +
            None => {
446 +
                read = self.store.get_change_meta(hash)?;
447 +
                &read
448 +
            }
407 449
        };
408 -
        for seal_ref in self.store.seals_for(hash)? {
450 +
        for seal_ref in seals {
409 451
            let attest = self.store.get_attest(&seal_ref)?;
410 452
            let AttestBody::Seal { hashes } = &attest.body else {
411 453
                continue;
…
2 unchanged lines
414 456
                continue;
415 457
            }
416 458
            for sig in self.store.signatures(&seal_ref)? {
417 -
                match self.check_signer(&change, &sig, Domain::Attest, &seal_ref) {
459 +
                match self.check_signer(change, &sig, Domain::Attest, &seal_ref) {
418 460
                    Ok(id) => {
419 461
                        return Ok(SignatureStatus::Valid {
420 462
                            key: sig.key,
…
88 unchanged lines
modified crates/oxy-core/src/repo/mod.rs +64 −7
…
1129 unchanged lines
1130 1130
        Ok(st)
1131 1131
    }
1132 1132
1133 +
    /// The state index file: its heads, a hash of its changes block and one
1134 +
    /// of the rest, then the body [`State::write_index`] gives, whose first
1135 +
    /// block is the changes. Hashed in two parts so the log can read and
1136 +
    /// check the changes alone.
1133 1137
    fn read_state_index(&self, deps: &[ChangeHash]) -> Option<State> {
1134 1138
        let _t = crate::timing::phase("load: state index");
1135 1139
        let bytes = fs::read(self.state_index_path()).ok()?;
…
2 unchanged lines
1138 1142
        for _ in 0..r.count().ok()? {
1139 1143
            heads.push(r.object().ok()?);
1140 1144
        }
1141 -
        let sum = r.object().ok()?;
1142 -
        let body = &bytes[4 + 32 * heads.len() + 32..];
1143 -
        // Checked whole, so a part decoded later (the tree's maps) can
1144 -
        // trust its bytes.
1145 +
        let (changes_sum, rest_sum) = (r.object().ok()?, r.object().ok()?);
1146 +
        let body = &bytes[4 + 32 * heads.len() + 64..];
1147 +
        let split = 4 + r.count().ok()?;
1148 +
        // Checked whole, so a part decoded later (the tree's maps, a
1149 +
        // file's history) can trust its bytes.
1150 +
        let (changes, rest) = (body.get(..split)?, body.get(split..)?);
1145 1151
        let mut h = blake3::Hasher::new();
1146 -
        h.update_rayon(body);
1147 -
        if *h.finalize().as_bytes() != sum.0 {
1152 +
        h.update_rayon(rest);
1153 +
        if *blake3::hash(changes).as_bytes() != changes_sum.0
1154 +
            || *h.finalize().as_bytes() != rest_sum.0
1155 +
        {
1148 1156
            return None;
1149 1157
        }
1150 1158
        let st = State::read_index(body).ok()?;
…
3 unchanged lines
1154 1162
        let mut have = st.maximal(deps);
1155 1163
        have.sort();
1156 1164
        (have == heads).then_some(st)
1165 +
    }
1166 +
1167 +
    /// The changes of this workspace's state index (the closure of its
1168 +
    /// heads), read without the rest of the index.
1169 +
    pub(crate) fn read_state_index_changes(
1170 +
        &self,
1171 +
    ) -> Option<std::collections::HashMap<ChangeHash, std::sync::Arc<crate::state::ChangeMeta>>>
1172 +
    {
1173 +
        use std::io::Read;
1174 +
        let _t = crate::timing::phase("load: state index changes");
1175 +
        let mut f = fs::File::open(self.state_index_path()).ok()?;
1176 +
        let mut word = [0u8; 4];
1177 +
        f.read_exact(&mut word).ok()?;
1178 +
        let n = u32::from_le_bytes(word) as usize;
1179 +
        let mut head = vec![0u8; 32 * n + 64 + 4];
1180 +
        f.read_exact(&mut head).ok()?;
1181 +
        let sum = &head[32 * n..32 * n + 32];
1182 +
        let len = u32::from_le_bytes(head[32 * n + 64..].try_into().unwrap()) as usize;
1183 +
        let mut changes = vec![0u8; 4 + len];
1184 +
        changes[..4].copy_from_slice(&head[32 * n + 64..]);
1185 +
        f.read_exact(&mut changes[4..]).ok()?;
1186 +
        if blake3::hash(&changes).as_bytes() != sum {
1187 +
            return None;
1188 +
        }
1189 +
        State::read_index_changes(&changes[4..]).ok()
1157 1190
    }
1158 1191
1159 1192
    /// Save `st`, the history-only state of `deps`, as this workspace's
…
16 unchanged lines
1176 1209
        for h in &heads {
1177 1210
            w.object(h);
1178 1211
        }
1212 +
        let split = 4 + u32::from_le_bytes(body[..4].try_into().unwrap()) as usize;
1213 +
        w.fixed(blake3::hash(&body[..split]).as_bytes());
1179 1214
        let mut sum = blake3::Hasher::new();
1180 -
        sum.update_rayon(&body);
1215 +
        sum.update_rayon(&body[split..]);
1181 1216
        w.fixed(sum.finalize().as_bytes());
1182 1217
        w.fixed(&body);
1183 1218
        let path = self.state_index_path();
…
429 unchanged lines
1613 1648
        heads: &[ChangeHash],
1614 1649
    ) -> Result<Vec<(ChangeHash, crate::state::ChangeMeta)>> {
1615 1650
        let boundary = self.boundary();
1651 +
        // The state index holds the metadata of its heads' closure: a
1652 +
        // change found there is not read from the store (a change object
1653 +
        // can be as large as the tree it adds).
1654 +
        let indexed = self.read_state_index_changes();
1655 +
        let (mut stored, mut from_store) = (0usize, std::time::Duration::ZERO);
1616 1656
        let mut seen: HashSet<ChangeHash> = HashSet::new();
1617 1657
        let mut stack: Vec<ChangeHash> = heads.to_vec();
1618 1658
        let mut out = Vec::new();
…
1 unchanged lines
1620 1660
            if !seen.insert(h) {
1621 1661
                continue;
1622 1662
            }
1663 +
            if let Some(meta) = indexed.as_ref().and_then(|m| m.get(&h)) {
1664 +
                #[cfg(test)]
1665 +
                assert_eq!(
1666 +
                    **meta,
1667 +
                    self.store.get_change_meta(&h)?,
1668 +
                    "the state index holds a change's metadata"
1669 +
                );
1670 +
                stack.extend(meta.deps.iter().copied());
1671 +
                out.push((h, (**meta).clone()));
1672 +
                continue;
1673 +
            }
1674 +
            let started = std::time::Instant::now();
1623 1675
            let meta = match self.store.get_change_meta(&h) {
1624 1676
                Ok(m) => m,
1625 1677
                Err(crate::store::StoreError::NotFound(_)) if boundary.contains(&h) => continue,
1626 1678
                Err(e) => return Err(e.into()),
1627 1679
            };
1680 +
            stored += 1;
1681 +
            from_store += started.elapsed();
1628 1682
            stack.extend(meta.deps.iter().copied());
1629 1683
            out.push((h, meta));
1684 +
        }
1685 +
        if stored > 0 && crate::timing::enabled() {
1686 +
            eprintln!("timing metadata: {stored} changes read whole, {from_store:.2?}");
1630 1687
        }
1631 1688
        Ok(out)
1632 1689
    }
…
263 unchanged lines
modified crates/oxy-core/src/state.rs +44 −26
…
142 unchanged lines
143 143
/// command reads it rather than replaying history. Derived only: written
144 144
/// from a state loaded the long way, and read back as the same state.
145 145
impl State {
146 -
    const INDEX_MAGIC: &'static [u8] = b"oxy state index 2\n";
146 +
    const INDEX_MAGIC: &'static [u8] = b"oxy state index 3\n";
147 147
148 148
    /// The index of this state, or `None` when it has anything an index
149 149
    /// does not hold: a built graph, or text in memory (a change added
…
13 unchanged lines
163 163
                w.u32(*index);
164 164
            }
165 165
        };
166 +
        // The changes first, as a block a reader can take on its own (see
167 +
        // [`State::read_index_changes`]).
166 168
        let mut w = Writer::default();
167 169
        w.fixed(Self::INDEX_MAGIC);
168 170
        let mut changes: Vec<(&ChangeHash, &Arc<ChangeMeta>)> = self.changes.iter().collect();
…
12 unchanged lines
181 183
            w.bytes(m.message.as_bytes());
182 184
            w.u8(m.kind.tag() as u8);
183 185
        }
186 +
        let changes_block = std::mem::take(&mut w);
187 +
        w.bytes(&changes_block.0);
184 188
        // The tree and the files' histories as blocks of their own, so a
185 189
        // reader decodes them side by side.
186 190
        let mut tw = Writer::default();
…
128 unchanged lines
315 319
        use crate::flat::Reader;
316 320
        let bad = |what: &str| ObjectError::Schema(format!("state index: {what}"));
317 321
        let mut r = Reader::new(bytes);
322 +
        let mut st = State::new_history_only();
323 +
        {
324 +
            let _t = crate::timing::phase("index: changes");
325 +
            st.changes = Self::read_index_changes(r.bytes()?)?;
326 +
        }
327 +
        let tree_bytes = r.bytes()?;
328 +
        for _ in 0..r.count()? {
329 +
            let n = NodeId(r.fixed::<16>()?);
330 +
            let mut recs = Vec::new();
331 +
            for _ in 0..r.count()? {
332 +
                recs.push((r.object()?, r.opt_object()?, r.opt_object()?));
333 +
            }
334 +
            st.replaces.insert(n, recs);
335 +
        }
336 +
        let deferred_bytes = r.bytes()?;
337 +
        if !r.at_end() {
338 +
            return Err(bad("trailing bytes"));
339 +
        }
340 +
        let (tree, deferred) = rayon::join(
341 +
            || TreeGraph::read_flat(&mut Reader::new(tree_bytes)),
342 +
            || Self::read_deferred(deferred_bytes),
343 +
        );
344 +
        st.tree = tree?;
345 +
        st.deferred = Some(deferred?);
346 +
        Ok(st)
347 +
    }
348 +
349 +
    /// The changes of a state index: its first block, which a reader can
350 +
    /// take without the rest (the log needs no more).
351 +
    pub fn read_index_changes(
352 +
        bytes: &[u8],
353 +
    ) -> Result<HashMap<ChangeHash, Arc<ChangeMeta>>, ObjectError> {
354 +
        use crate::flat::Reader;
355 +
        let mut r = Reader::new(bytes);
318 356
        if r.fixed::<18>()?.as_slice() != Self::INDEX_MAGIC {
319 -
            return Err(bad("not a state index"));
357 +
            return Err(ObjectError::Schema("state index: not a state index".into()));
320 358
        }
321 -
        let node = |r: &mut Reader| -> Result<NodeId, ObjectError> { Ok(NodeId(r.fixed::<16>()?)) };
322 -
        let mut st = State::new_history_only();
323 -
        let t = crate::timing::phase("index: changes");
324 359
        let n = r.count()?;
325 -
        st.changes.reserve(n);
360 +
        let mut changes = HashMap::with_capacity(n);
326 361
        for _ in 0..n {
327 362
            let h = r.object()?;
328 363
            let change_id = crate::ids::ChangeId(r.fixed::<16>()?);
…
12 unchanged lines
341 376
                message: r.text()?,
342 377
                kind: ChangeKind::from_tag(r.u8()? as u64)?,
343 378
            };
344 -
            st.changes.insert(h, Arc::new(meta));
379 +
            changes.insert(h, Arc::new(meta));
345 380
        }
346 -
        drop(t);
347 -
        let tree_bytes = r.bytes()?;
348 -
        for _ in 0..r.count()? {
349 -
            let n = node(&mut r)?;
350 -
            let mut recs = Vec::new();
351 -
            for _ in 0..r.count()? {
352 -
                recs.push((r.object()?, r.opt_object()?, r.opt_object()?));
353 -
            }
354 -
            st.replaces.insert(n, recs);
355 -
        }
356 -
        let deferred_bytes = r.bytes()?;
357 381
        if !r.at_end() {
358 -
            return Err(bad("trailing bytes"));
382 +
            return Err(ObjectError::Schema("state index: trailing bytes".into()));
359 383
        }
360 -
        let (tree, deferred) = rayon::join(
361 -
            || TreeGraph::read_flat(&mut Reader::new(tree_bytes)),
362 -
            || Self::read_deferred(deferred_bytes),
363 -
        );
364 -
        st.tree = tree?;
365 -
        st.deferred = Some(deferred?);
366 -
        Ok(st)
384 +
        Ok(changes)
367 385
    }
368 386
369 387
    /// The deferred block: each node with its graph key and its body,
…
2074 unchanged lines
modified src/cli/format.rs +5 −3
…
65 unchanged lines
66 66
    })
67 67
}
68 68
69 -
/// A compact, styled authentication marker for log lines.
70 -
pub fn signature_mark(repo: &Repo, h: &ChangeHash) -> Result<String> {
71 -
    Ok(match repo.authenticate_change(h)? {
69 +
/// A compact, styled authentication marker for log lines, from the
70 +
/// change's metadata when the caller has it (so the change itself need not
71 +
/// be read).
72 +
pub fn signature_mark(repo: &Repo, h: &ChangeHash, known: Option<&ChangeMeta>) -> Result<String> {
73 +
    Ok(match repo.authenticate_change_known(h, known)? {
72 74
        SignatureStatus::Unsigned => ui::dim(format!("{} unsigned", ui::glyph("·", "-"))),
73 75
        SignatureStatus::Valid {
74 76
            sealed,
…
64 unchanged lines
modified src/cli/workspace.rs +9 −2
…
79 unchanged lines
80 80
    }
81 81
    let sp = ui::spinner("Snapshotting the working copy…");
82 82
    let snap = repo.snapshot_status("oxy status")?;
83 -
    let heads = repo.bookmark_heads()?;
83 +
    let heads = {
84 +
        let _t = oxy_core::timing::phase("status: heads");
85 +
        repo.bookmark_heads()?
86 +
    };
87 +
    let _drop = oxy_core::timing::phase("status: print and drop");
84 88
    sp.clear();
85 89
    if json {
86 90
        println!("{}", repo.status_json(&snap)?);
…
435 unchanged lines
522 526
        let snap = repo.snapshot_status("oxy log")?;
523 527
        (snap.base_snapshot.is_conflicted(), snap.statuses.is_empty())
524 528
    };
529 +
    let listing = oxy_core::timing::phase("log: entries");
525 530
    let entries = match (path, lines) {
526 531
        (_, Some(spec)) => {
527 532
            let (first, last, p) = parse_line_range(spec)?;
…
4 unchanged lines
532 537
        (Some(p), None) => repo.log_for_path(p)?,
533 538
        (None, None) => repo.log()?,
534 539
    };
540 +
    drop(listing);
541 +
    let _print = oxy_core::timing::phase("log: print");
535 542
    if as_json {
536 543
        let shown = limit.unwrap_or(entries.len()).min(entries.len());
537 544
        let mut items = Vec::new();
…
58 unchanged lines
596 603
            if e.is_working_copy {
597 604
                ui::dim("not yet committed")
598 605
            } else {
599 -
                signature_mark(&repo, &e.hash)?
606 +
                signature_mark(&repo, &e.hash, Some(c))?
600 607
            }
601 608
        );
602 609
        println!("{rail} {meta}");
…
1274 unchanged lines