OXYGENHUB

Snapshot: whole-tree passes as merge joins (refs #41) d616faf5d136

Choosing which base graphs to build and picking the changed entries walked
the whole tree with a map lookup per path; both maps are in path order, so
a merge join does it in one linear pass. The base snapshot renders by the
set of paths needed. One-file status on the Linux tree: 0.42 s (from
0.47 s).
Author
Andy McCall @andymccall <[email protected]>
When
· 2026-09-30 22:00 UTC
Signature
✔ signed by Andy McCall
Hash
d616faf5d136fe82a2e4f4bb6957281e388a4b198ceb974862958febe47f48f1
Depends on
23ff1288a5dd 6e723b81f5b9 982864068a59 cc7a00565161 d0990c2bff74 ff0af1ee299d
Depended on by
c2e1d09047db 10b4fd430eab f57bf6702628
In bookmarks
agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
In tags
v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
Edits
12 atoms, 0 tree operations, 2 files

Browse the tree at this change

modified crates/oxy-core/src/repo/snapshot.rs +77 −22
…
176 unchanged lines
177 177
        reuse: HashMap<String, (NodeId, NodeKind)>,
178 178
        fs_entries: &BTreeMap<String, FsEntry>,
179 179
    ) -> Result<BuiltChange> {
180 +
        let pick = crate::timing::phase("snapshot:   of which picking the changed");
180 181
        // A file known unchanged may still have changed mode: that alters
181 182
        // neither its size nor its time.
182 -
        let changed: BTreeMap<String, FsEntry> = fs_entries
183 -
            .iter()
184 -
            .filter(|(p, e)| {
185 -
                !e.unchanged
186 -
                    || base_snapshot
187 -
                        .files
188 -
                        .get(*p)
189 -
                        .is_none_or(|f| f.kind != e.kind || f.mode.exec != e.exec)
183 +
        let changed: BTreeMap<String, FsEntry> = join(fs_entries, &base_snapshot.files)
184 +
            .filter_map(|(p, e, f)| {
185 +
                let e = e?;
186 +
                (!e.unchanged || f.is_none_or(|f| f.kind != e.kind || f.mode.exec != e.exec))
187 +
                    .then(|| (p.clone(), e.clone()))
190 188
            })
191 -
            .map(|(p, e)| (p.clone(), e.clone()))
192 189
            .collect();
193 190
        let mut files: BTreeMap<String, crate::state::FileState> = BTreeMap::new();
194 191
        for p in changed.keys() {
…
1 unchanged lines
196 193
                files.insert(p.clone(), f.clone());
197 194
            }
198 195
        }
199 -
        for (p, f) in &base_snapshot.files {
200 -
            if f.kind == NodeKind::Dir || fs_entries.contains_key(p) {
196 +
        // Paths the base has and the disk does not.
197 +
        let gone = join(&base_snapshot.files, fs_entries)
198 +
            .filter_map(|(p, f, e)| (e.is_none()).then_some((p, f?)));
199 +
        for (p, f) in gone {
200 +
            if f.kind == NodeKind::Dir {
201 201
                continue;
202 202
            }
203 203
            files.insert(p.clone(), f.clone());
…
14 unchanged lines
218 218
            files,
219 219
            unrendered: BTreeSet::new(),
220 220
        };
221 +
        drop(pick);
222 +
        let _build = crate::timing::phase("snapshot:   of which building the change");
221 223
        // Whether anything on disk is under `dir`.
222 224
        let dir_needed = |dir: &str| {
223 225
            let prefix = format!("{dir}/");
…
206 unchanged lines
430 432
        let base_snapshot = {
431 433
            let _t = crate::timing::phase("snapshot: load and render base");
432 434
            if partial {
433 -
                let skip: HashSet<NodeId> = fs_entries
434 -
                    .iter()
435 -
                    .filter(|(_, e)| e.unchanged)
436 -
                    .filter_map(|(p, _)| view.entries.get(p).map(|e| e.node))
437 -
                    .collect();
438 -
                let need: Vec<NodeId> = view
439 -
                    .entries
440 -
                    .values()
441 -
                    .filter(|e| e.kind == NodeKind::File && !skip.contains(&e.node))
442 -
                    .map(|e| e.node)
435 +
                let sets = crate::timing::phase("snapshot:   of which choosing graphs");
436 +
                // Every file but those on disk known unchanged.
437 +
                let need: Vec<(&str, NodeId)> = join(&view.entries, &fs_entries)
438 +
                    .filter_map(|(p, v, f)| {
439 +
                        let v = v?;
440 +
                        (v.kind == NodeKind::File && !f.is_some_and(|f| f.unchanged))
441 +
                            .then_some((p.as_str(), v.node))
442 +
                    })
443 443
                    .collect();
444 +
                let render: std::collections::HashSet<&str> =
445 +
                    need.iter().map(|(p, _)| *p).collect();
446 +
                let need: Vec<NodeId> = need.iter().map(|(_, n)| *n).collect();
447 +
                drop(sets);
444 448
                {
445 449
                    let _t = crate::timing::phase("snapshot:   of which building graphs");
446 450
                    self.load_graphs_of(&mut base, &need)?;
447 451
                }
448 452
                let _t = crate::timing::phase("snapshot:   of which rendering");
449 -
                base.materialize_except(&skip)
453 +
                base.materialize_only(&render)
450 454
            } else {
451 455
                self.load_all_graphs(&mut base)?;
452 456
                base.materialize()
…
2186 unchanged lines
2639 2643
        }
2640 2644
    }
2641 2645
    (shared * 100 / a.len().max(b.len())) as u32
2646 +
}
2647 +
2648 +
/// Two maps side by side in key order: each key of either, with its value
2649 +
/// in each. Linear, with no lookups, for maps of a whole tree.
2650 +
fn join<'a, A, B>(
2651 +
    a: &'a BTreeMap<String, A>,
2652 +
    b: &'a BTreeMap<String, B>,
2653 +
) -> impl Iterator<Item = (&'a String, Option<&'a A>, Option<&'a B>)> {
2654 +
    let (mut a, mut b) = (a.iter().peekable(), b.iter().peekable());
2655 +
    std::iter::from_fn(move || match (a.peek(), b.peek()) {
2656 +
        (None, None) => None,
2657 +
        (Some(_), None) => a.next().map(|(k, v)| (k, Some(v), None)),
2658 +
        (None, Some(_)) => b.next().map(|(k, v)| (k, None, Some(v))),
2659 +
        (Some((ka, _)), Some((kb, _))) => match ka.cmp(kb) {
2660 +
            std::cmp::Ordering::Less => a.next().map(|(k, v)| (k, Some(v), None)),
2661 +
            std::cmp::Ordering::Greater => b.next().map(|(k, v)| (k, None, Some(v))),
2662 +
            std::cmp::Ordering::Equal => {
2663 +
                let (k, va) = a.next()?;
2664 +
                let (_, vb) = b.next()?;
2665 +
                Some((k, Some(va), Some(vb)))
2666 +
            }
2667 +
        },
2668 +
    })
2669 +
}
2670 +
2671 +
#[cfg(test)]
2672 +
mod join_tests {
2673 +
    use super::*;
2674 +
2675 +
    #[test]
2676 +
    fn join_pairs_every_key_in_order() {
2677 +
        let a: BTreeMap<String, u8> = [("a", 1), ("b/c", 2), ("d", 3)]
2678 +
            .into_iter()
2679 +
            .map(|(k, v)| (k.to_string(), v))
2680 +
            .collect();
2681 +
        let b: BTreeMap<String, char> = [("b", 'x'), ("b/c", 'y'), ("e", 'z')]
2682 +
            .into_iter()
2683 +
            .map(|(k, v)| (k.to_string(), v))
2684 +
            .collect();
2685 +
        let got: Vec<(String, Option<u8>, Option<char>)> = join(&a, &b)
2686 +
            .map(|(k, x, y)| (k.clone(), x.copied(), y.copied()))
2687 +
            .collect();
2688 +
        let want = vec![
2689 +
            ("a".to_string(), Some(1), None),
2690 +
            ("b".to_string(), None, Some('x')),
2691 +
            ("b/c".to_string(), Some(2), Some('y')),
2692 +
            ("d".to_string(), Some(3), None),
2693 +
            ("e".to_string(), None, Some('z')),
2694 +
        ];
2695 +
        assert_eq!(got, want);
2696 +
    }
2642 2697
}
modified crates/oxy-core/src/state.rs +14 −1
…
1187 unchanged lines
1188 1188
    /// file's graph must be loaded. For a caller that already knows those
1189 1189
    /// files are unchanged and not conflicted.
1190 1190
    pub fn materialize_except(&self, skip: &HashSet<NodeId>) -> Snapshot {
1191 +
        self.materialize_leaving(|_, e| skip.contains(&e.node))
1192 +
    }
1193 +
1194 +
    /// As [`State::materialize_except`], rendering only the text files at
1195 +
    /// `render` (a few, where the caller knows the rest unchanged).
1196 +
    pub fn materialize_only(&self, render: &std::collections::HashSet<&str>) -> Snapshot {
1197 +
        self.materialize_leaving(|p, _| !render.contains(p))
1198 +
    }
1199 +
1200 +
    fn materialize_leaving(
1201 +
        &self,
1202 +
        leave: impl Fn(&str, &crate::tree::TreeEntry) -> bool + Sync,
1203 +
    ) -> Snapshot {
1191 1204
        use rayon::prelude::*;
1192 1205
        let tree = self.tree.view();
1193 1206
        let entries: Vec<(&String, &crate::tree::TreeEntry)> = tree.entries.iter().collect();
1194 1207
        let rendered: Vec<(String, FileState, bool)> = entries
1195 1208
            .par_iter()
1196 1209
            .map(|(path, e)| {
1197 -
                if e.kind == NodeKind::File && !e.mode.opaque && skip.contains(&e.node) {
1210 +
                if e.kind == NodeKind::File && !e.mode.opaque && leave(path, e) {
1198 1211
                    let f = FileState {
1199 1212
                        node: e.node,
1200 1213
                        kind: e.kind,
…
799 unchanged lines