OXYGENHUB

Checkout through the state index; two indexes kept; racy files hashed in parallel (refs #41) 638fb21420d7

- A checkout loads its target through the state index, since the target
  becomes the working copy's base.
- A workspace keeps its last two state indexes, named by their heads, so
  moving back and forth between two states finds both; one used is
  touched, and older ones are removed.
- The clean check hashes the files in the racy window in parallel: right
  after a checkout that wrote 1 000 files, status took 1.3 s, now 0.18 s.
- Graph building has phases of its own under OXY_TIMINGS.

Checkout to the first state of the Linux tree and back: 0.83 to 0.93 s
each (from 1.4 s).
Author
Andy McCall @andymccall <[email protected]>
When
· 2026-09-30 22:54 UTC
Signature
✔ signed by Andy McCall
Hash
638fb21420d725d401e2059db04f2049b1df3154884c3fca53f65f7e14ffc346
Depends on
135212de5370 4cce912a0953 57d8dc05c9d4 982864068a59 c5ce0367eddf d3cd7ef95da6 da868d5d2646 f9466021286a f9575918477f ff0af1ee299d
Depended on by
5552e69c7325 3604179679eb acadb5ed28a1
In bookmarks
agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
In tags
v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
Edits
18 atoms, 0 tree operations, 4 files

Browse the tree at this change

modified crates/oxy-core/src/repo/checkout.rs +3 −2
…
55 unchanged lines
56 56
        let old_manifest = self.read_manifest()?.map(|(_, _, m)| m).unwrap_or_default();
57 57
        let changed: HashSet<&str> = changed_paths.iter().map(String::as_str).collect();
58 58
        // The target's history; its file graphs are built a batch at a time
59 -
        // for the files that must be written, and dropped after.
60 -
        let mut target = self.load_state_history(heads)?;
59 +
        // for the files that must be written, and dropped after. It becomes
60 +
        // the working copy's base, which the state index holds.
61 +
        let mut target = self.load_state_indexed(heads)?;
61 62
        let heads = target.maximal(heads);
62 63
        let tv = crate::timing::phase("checkout: target view");
63 64
        let tview = target.tree().view();
…
394 unchanged lines
modified crates/oxy-core/src/repo/mod.rs +63 −9
…
128 unchanged lines
129 129
/// How many changes a state index may fall behind its working copy before
130 130
/// it is written again; until then each load replays them.
131 131
const INDEX_REPLAY_LIMIT: usize = 16;
132 +
/// How many state indexes a workspace keeps.
133 +
const STATE_INDEXES_KEPT: usize = 2;
132 134
const INDEX_REPLAY_TIME: std::time::Duration = std::time::Duration::from_millis(50);
133 135
134 136
pub(crate) fn write_cache(path: &Path, data: &[u8]) -> Result<()> {
…
987 unchanged lines
1122 1124
1123 1125
    /// This workspace's state index (oxygen#41): its working copy's base,
1124 1126
    /// saved so a snapshot reads it rather than replaying history.
1125 -
    fn state_index_path(&self) -> PathBuf {
1126 -
        self.oxy
1127 -
            .join("index")
1128 -
            .join(format!("state-{}", self.workspace.to_hex()))
1127 +
    /// The state index of the state with these heads: this workspace keeps
1128 +
    /// the last few it used, named by their heads, so moving between two
1129 +
    /// states finds both.
1130 +
    fn state_index_path(&self, heads: &[ChangeHash]) -> PathBuf {
1131 +
        let mut heads = heads.to_vec();
1132 +
        heads.sort();
1133 +
        let mut h = blake3::Hasher::new();
1134 +
        for c in &heads {
1135 +
            h.update(&c.0);
1136 +
        }
1137 +
        self.oxy.join("index").join(format!(
1138 +
            "{}{}",
1139 +
            self.state_index_prefix(),
1140 +
            &h.finalize().to_hex()[..16]
1141 +
        ))
1129 1142
    }
1130 1143
1144 +
    fn state_index_prefix(&self) -> String {
1145 +
        format!("state-{}-", self.workspace.to_hex())
1146 +
    }
1147 +
1148 +
    /// This workspace's state indexes, newest first.
1149 +
    fn state_indexes(&self) -> Vec<PathBuf> {
1150 +
        let prefix = self.state_index_prefix();
1151 +
        let mut found: Vec<(std::time::SystemTime, PathBuf)> = fs::read_dir(self.oxy.join("index"))
1152 +
            .into_iter()
1153 +
            .flatten()
1154 +
            .flatten()
1155 +
            .filter(|e| e.file_name().to_string_lossy().starts_with(&prefix))
1156 +
            .filter_map(|e| Some((e.metadata().ok()?.modified().ok()?, e.path())))
1157 +
            .collect();
1158 +
        found.sort_by_key(|f| std::cmp::Reverse(f.0));
1159 +
        found.into_iter().map(|(_, p)| p).collect()
1160 +
    }
1161 +
1131 1162
    /// The history-only state of `deps` (as [`Repo::load_state_history`]),
1132 1163
    /// from this workspace's state index when it holds that history: every
1133 1164
    /// one of `deps` is in it and their heads are its heads. Otherwise it
1134 1165
    /// is loaded from history and indexed for next time.
1135 1166
    pub fn load_state_indexed(&self, deps: &[ChangeHash]) -> Result<State> {
1136 -
        if let Some((st, heads)) = self.read_state_index() {
1167 +
        if let Some((st, heads)) = self.read_state_index(deps) {
1137 1168
            let mut have = st.maximal(deps);
1138 1169
            have.sort();
1139 1170
            if deps.iter().all(|d| st.contains(d)) && have == heads {
…
55 unchanged lines
1195 1226
    /// of the rest, then the body [`State::write_index`] gives, whose first
1196 1227
    /// block is the changes. Hashed in two parts so the log can read and
1197 1228
    /// check the changes alone.
1198 -
    fn read_state_index(&self) -> Option<(State, Vec<ChangeHash>)> {
1229 +
    /// The index of `deps` if there is one, else the newest, which the
1230 +
    /// caller may bring up to date.
1231 +
    fn read_state_index(&self, deps: &[ChangeHash]) -> Option<(State, Vec<ChangeHash>)> {
1199 1232
        let _t = crate::timing::phase("load: state index");
1200 -
        let bytes = fs::read(self.state_index_path()).ok()?;
1233 +
        let exact = self.state_index_path(deps);
1234 +
        let bytes = match fs::read(&exact) {
1235 +
            Ok(b) => {
1236 +
                // Used now: the newest, so the last to be dropped.
1237 +
                let _ = fs::File::options()
1238 +
                    .write(true)
1239 +
                    .open(&exact)
1240 +
                    .and_then(|f| f.set_modified(std::time::SystemTime::now()));
1241 +
                b
1242 +
            }
1243 +
            Err(_) => fs::read(self.state_indexes().first()?).ok()?,
1244 +
        };
1201 1245
        let mut r = crate::flat::Reader::new(&bytes);
1202 1246
        let mut heads = Vec::new();
1203 1247
        for _ in 0..r.count().ok()? {
…
23 unchanged lines
1227 1271
    {
1228 1272
        use std::io::Read;
1229 1273
        let _t = crate::timing::phase("load: state index changes");
1230 -
        let mut f = fs::File::open(self.state_index_path()).ok()?;
1274 +
        let mut f = fs::File::open(self.state_indexes().first()?).ok()?;
1231 1275
        let mut word = [0u8; 4];
1232 1276
        f.read_exact(&mut word).ok()?;
1233 1277
        let n = u32::from_le_bytes(word) as usize;
…
38 unchanged lines
1272 1316
        sum.update_rayon(&body[split..]);
1273 1317
        w.fixed(sum.finalize().as_bytes());
1274 1318
        w.fixed(&body);
1275 -
        let path = self.state_index_path();
1319 +
        let path = self.state_index_path(&heads);
1276 1320
        let _ = fs::create_dir_all(self.oxy.join("index"));
1277 1321
        let _ = write_cache(&path, &w.0);
1322 +
        // The last two used: the index of a state moved away from stays for
1323 +
        // a move back. Older ones (and the unnamed one of before) go.
1324 +
        for old in self.state_indexes().iter().skip(STATE_INDEXES_KEPT) {
1325 +
            let _ = fs::remove_file(old);
1326 +
        }
1327 +
        let _ = fs::remove_file(
1328 +
            self.oxy
1329 +
                .join("index")
1330 +
                .join(format!("state-{}", self.workspace.to_hex())),
1331 +
        );
1278 1332
    }
1279 1333
1280 1334
    fn locks_dir(&self) -> PathBuf {
…
645 unchanged lines
modified crates/oxy-core/src/repo/prompt.rs +15 −8
…
430 unchanged lines
431 431
        if conflicted || uncommitted || self.root.join(".oxymodules").exists() {
432 432
            return not("conflicted, uncommitted or subrepos");
433 433
        }
434 -
        for (rel, e) in &m.entries {
435 -
            if e.mtime_ns + 2_000_000_000 < m.written_ns {
436 -
                continue;
437 -
            }
438 -
            let path = self.root.join(rel);
434 +
        // Entries in the racy window are hashed, in parallel (a checkout
435 +
        // just wrote them all).
436 +
        use rayon::prelude::*;
437 +
        let racy: Vec<(&String, &ManifestEntry)> = m
438 +
            .entries
439 +
            .iter()
440 +
            .filter(|(_, e)| e.mtime_ns + 2_000_000_000 >= m.written_ns)
441 +
            .collect();
442 +
        let root = &self.root;
443 +
        let changed = racy.par_iter().find_any(|(rel, e)| {
444 +
            let path = root.join(rel.as_str());
439 445
            let same = match fs::symlink_metadata(&path) {
440 446
                Ok(md) if md.file_type().is_symlink() => fs::read_link(&path).is_ok_and(|t| {
441 447
                    *blake3::hash(t.to_string_lossy().as_bytes()).as_bytes() == e.hash
…
4 unchanged lines
446 452
                }
447 453
                _ => false,
448 454
            };
449 -
            if !same {
450 -
                return not(&format!("{rel} changed within the racy window"));
451 -
            }
455 +
            !same
456 +
        });
457 +
        if let Some((rel, _)) = changed {
458 +
            return not(&format!("{rel} changed within the racy window"));
452 459
        }
453 460
        if self.quick_dirty_keeping(true)? != Some(false) {
454 461
            return not("a file differs");
…
161 unchanged lines
modified crates/oxy-core/src/state.rs +5 −0
…
665 unchanged lines
666 666
    }));
667 667
    // And the encodings of changes whose inline inserts are needed,
668 668
    // each read once.
669 +
    let reading = crate::timing::phase("graphs: read changes and blobs");
669 670
    let mut changes: HashMap<ChangeHash, Arc<Vec<u8>>> = HashMap::new();
670 671
    for t in taken.iter().flat_map(|(_, d)| d.texts.iter()) {
671 672
        if let TextRef::Inline { change, .. } = t {
…
10 unchanged lines
682 683
    if blobs.len() != refs.len() {
683 684
        return Err(ObjectError::Schema("graph blobs: wrong count".into()));
684 685
    }
686 +
    drop(reading);
687 +
    let lines = crate::timing::phase("graphs: lines of inserts");
685 688
    let mut texts = blobs.split_off(bases).into_iter();
686 689
    let mut blobs = blobs.into_iter();
687 690
    // Pair each graph with its blob, then build them all in parallel:
…
43 unchanged lines
731 734
        };
732 735
        work.push((node, d, bytes));
733 736
    }
737 +
    drop(lines);
738 +
    let _t = crate::timing::phase("graphs: build");
734 739
    use rayon::prelude::*;
735 740
    let built: Vec<(NodeId, FileGraph)> = work
736 741
        .into_par_iter()
…
1739 unchanged lines