Checkout through the state index; two indexes kept; racy files hashed in parallel (refs #41) 638fb21420d7
- A checkout loads its target through the state index, since the target becomes the working copy's base. - A workspace keeps its last two state indexes, named by their heads, so moving back and forth between two states finds both; one used is touched, and older ones are removed. - The clean check hashes the files in the racy window in parallel: right after a checkout that wrote 1 000 files, status took 1.3 s, now 0.18 s. - Graph building has phases of its own under OXY_TIMINGS. Checkout to the first state of the Linux tree and back: 0.83 to 0.93 s each (from 1.4 s).
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-30 22:54 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
638fb21420d725d401e2059db04f2049b1df3154884c3fca53f65f7e14ffc346- Depends on
- 135212de5370 4cce912a0953 57d8dc05c9d4 982864068a59 c5ce0367eddf d3cd7ef95da6 da868d5d2646 f9466021286a f9575918477f ff0af1ee299d
- Depended on by
- 5552e69c7325 3604179679eb acadb5ed28a1
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 18 atoms, 0 tree operations, 4 files
Browse the tree at this change
modified crates/oxy-core/src/repo/checkout.rs +3 −2
| … | 55 unchanged lines |
||
| 56 | 56 | let old_manifest = self.read_manifest()?.map(|(_, _, m)| m).unwrap_or_default(); |
|
| 57 | 57 | let changed: HashSet<&str> = changed_paths.iter().map(String::as_str).collect(); |
|
| 58 | 58 | // The target's history; its file graphs are built a batch at a time |
|
| 59 | - | // for the files that must be written, and dropped after. |
|
| 60 | - | let mut target = self.load_state_history(heads)?; |
|
| 59 | + | // for the files that must be written, and dropped after. It becomes |
|
| 60 | + | // the working copy's base, which the state index holds. |
|
| 61 | + | let mut target = self.load_state_indexed(heads)?; |
|
| 61 | 62 | let heads = target.maximal(heads); |
|
| 62 | 63 | let tv = crate::timing::phase("checkout: target view"); |
|
| 63 | 64 | let tview = target.tree().view(); |
|
| … | 394 unchanged lines |
modified crates/oxy-core/src/repo/mod.rs +63 −9
| … | 128 unchanged lines |
||
| 129 | 129 | /// How many changes a state index may fall behind its working copy before |
|
| 130 | 130 | /// it is written again; until then each load replays them. |
|
| 131 | 131 | const INDEX_REPLAY_LIMIT: usize = 16; |
|
| 132 | + | /// How many state indexes a workspace keeps. |
|
| 133 | + | const STATE_INDEXES_KEPT: usize = 2; |
|
| 132 | 134 | const INDEX_REPLAY_TIME: std::time::Duration = std::time::Duration::from_millis(50); |
|
| 133 | 135 | ||
| 134 | 136 | pub(crate) fn write_cache(path: &Path, data: &[u8]) -> Result<()> { |
|
| … | 987 unchanged lines |
||
| 1122 | 1124 | ||
| 1123 | 1125 | /// This workspace's state index (oxygen#41): its working copy's base, |
|
| 1124 | 1126 | /// saved so a snapshot reads it rather than replaying history. |
|
| 1125 | - | fn state_index_path(&self) -> PathBuf { |
|
| 1126 | - | self.oxy |
|
| 1127 | - | .join("index") |
|
| 1128 | - | .join(format!("state-{}", self.workspace.to_hex())) |
|
| 1127 | + | /// The state index of the state with these heads: this workspace keeps |
|
| 1128 | + | /// the last few it used, named by their heads, so moving between two |
|
| 1129 | + | /// states finds both. |
|
| 1130 | + | fn state_index_path(&self, heads: &[ChangeHash]) -> PathBuf { |
|
| 1131 | + | let mut heads = heads.to_vec(); |
|
| 1132 | + | heads.sort(); |
|
| 1133 | + | let mut h = blake3::Hasher::new(); |
|
| 1134 | + | for c in &heads { |
|
| 1135 | + | h.update(&c.0); |
|
| 1136 | + | } |
|
| 1137 | + | self.oxy.join("index").join(format!( |
|
| 1138 | + | "{}{}", |
|
| 1139 | + | self.state_index_prefix(), |
|
| 1140 | + | &h.finalize().to_hex()[..16] |
|
| 1141 | + | )) |
|
| 1129 | 1142 | } |
|
| 1130 | 1143 | ||
| 1144 | + | fn state_index_prefix(&self) -> String { |
|
| 1145 | + | format!("state-{}-", self.workspace.to_hex()) |
|
| 1146 | + | } |
|
| 1147 | + | ||
| 1148 | + | /// This workspace's state indexes, newest first. |
|
| 1149 | + | fn state_indexes(&self) -> Vec<PathBuf> { |
|
| 1150 | + | let prefix = self.state_index_prefix(); |
|
| 1151 | + | let mut found: Vec<(std::time::SystemTime, PathBuf)> = fs::read_dir(self.oxy.join("index")) |
|
| 1152 | + | .into_iter() |
|
| 1153 | + | .flatten() |
|
| 1154 | + | .flatten() |
|
| 1155 | + | .filter(|e| e.file_name().to_string_lossy().starts_with(&prefix)) |
|
| 1156 | + | .filter_map(|e| Some((e.metadata().ok()?.modified().ok()?, e.path()))) |
|
| 1157 | + | .collect(); |
|
| 1158 | + | found.sort_by_key(|f| std::cmp::Reverse(f.0)); |
|
| 1159 | + | found.into_iter().map(|(_, p)| p).collect() |
|
| 1160 | + | } |
|
| 1161 | + | ||
| 1131 | 1162 | /// The history-only state of `deps` (as [`Repo::load_state_history`]), |
|
| 1132 | 1163 | /// from this workspace's state index when it holds that history: every |
|
| 1133 | 1164 | /// one of `deps` is in it and their heads are its heads. Otherwise it |
|
| 1134 | 1165 | /// is loaded from history and indexed for next time. |
|
| 1135 | 1166 | pub fn load_state_indexed(&self, deps: &[ChangeHash]) -> Result<State> { |
|
| 1136 | - | if let Some((st, heads)) = self.read_state_index() { |
|
| 1167 | + | if let Some((st, heads)) = self.read_state_index(deps) { |
|
| 1137 | 1168 | let mut have = st.maximal(deps); |
|
| 1138 | 1169 | have.sort(); |
|
| 1139 | 1170 | if deps.iter().all(|d| st.contains(d)) && have == heads { |
|
| … | 55 unchanged lines |
||
| 1195 | 1226 | /// of the rest, then the body [`State::write_index`] gives, whose first |
|
| 1196 | 1227 | /// block is the changes. Hashed in two parts so the log can read and |
|
| 1197 | 1228 | /// check the changes alone. |
|
| 1198 | - | fn read_state_index(&self) -> Option<(State, Vec<ChangeHash>)> { |
|
| 1229 | + | /// The index of `deps` if there is one, else the newest, which the |
|
| 1230 | + | /// caller may bring up to date. |
|
| 1231 | + | fn read_state_index(&self, deps: &[ChangeHash]) -> Option<(State, Vec<ChangeHash>)> { |
|
| 1199 | 1232 | let _t = crate::timing::phase("load: state index"); |
|
| 1200 | - | let bytes = fs::read(self.state_index_path()).ok()?; |
|
| 1233 | + | let exact = self.state_index_path(deps); |
|
| 1234 | + | let bytes = match fs::read(&exact) { |
|
| 1235 | + | Ok(b) => { |
|
| 1236 | + | // Used now: the newest, so the last to be dropped. |
|
| 1237 | + | let _ = fs::File::options() |
|
| 1238 | + | .write(true) |
|
| 1239 | + | .open(&exact) |
|
| 1240 | + | .and_then(|f| f.set_modified(std::time::SystemTime::now())); |
|
| 1241 | + | b |
|
| 1242 | + | } |
|
| 1243 | + | Err(_) => fs::read(self.state_indexes().first()?).ok()?, |
|
| 1244 | + | }; |
|
| 1201 | 1245 | let mut r = crate::flat::Reader::new(&bytes); |
|
| 1202 | 1246 | let mut heads = Vec::new(); |
|
| 1203 | 1247 | for _ in 0..r.count().ok()? { |
|
| … | 23 unchanged lines |
||
| 1227 | 1271 | { |
|
| 1228 | 1272 | use std::io::Read; |
|
| 1229 | 1273 | let _t = crate::timing::phase("load: state index changes"); |
|
| 1230 | - | let mut f = fs::File::open(self.state_index_path()).ok()?; |
|
| 1274 | + | let mut f = fs::File::open(self.state_indexes().first()?).ok()?; |
|
| 1231 | 1275 | let mut word = [0u8; 4]; |
|
| 1232 | 1276 | f.read_exact(&mut word).ok()?; |
|
| 1233 | 1277 | let n = u32::from_le_bytes(word) as usize; |
|
| … | 38 unchanged lines |
||
| 1272 | 1316 | sum.update_rayon(&body[split..]); |
|
| 1273 | 1317 | w.fixed(sum.finalize().as_bytes()); |
|
| 1274 | 1318 | w.fixed(&body); |
|
| 1275 | - | let path = self.state_index_path(); |
|
| 1319 | + | let path = self.state_index_path(&heads); |
|
| 1276 | 1320 | let _ = fs::create_dir_all(self.oxy.join("index")); |
|
| 1277 | 1321 | let _ = write_cache(&path, &w.0); |
|
| 1322 | + | // The last two used: the index of a state moved away from stays for |
|
| 1323 | + | // a move back. Older ones (and the unnamed one of before) go. |
|
| 1324 | + | for old in self.state_indexes().iter().skip(STATE_INDEXES_KEPT) { |
|
| 1325 | + | let _ = fs::remove_file(old); |
|
| 1326 | + | } |
|
| 1327 | + | let _ = fs::remove_file( |
|
| 1328 | + | self.oxy |
|
| 1329 | + | .join("index") |
|
| 1330 | + | .join(format!("state-{}", self.workspace.to_hex())), |
|
| 1331 | + | ); |
|
| 1278 | 1332 | } |
|
| 1279 | 1333 | ||
| 1280 | 1334 | fn locks_dir(&self) -> PathBuf { |
|
| … | 645 unchanged lines |
modified crates/oxy-core/src/repo/prompt.rs +15 −8
| … | 430 unchanged lines |
||
| 431 | 431 | if conflicted || uncommitted || self.root.join(".oxymodules").exists() { |
|
| 432 | 432 | return not("conflicted, uncommitted or subrepos"); |
|
| 433 | 433 | } |
|
| 434 | - | for (rel, e) in &m.entries { |
|
| 435 | - | if e.mtime_ns + 2_000_000_000 < m.written_ns { |
|
| 436 | - | continue; |
|
| 437 | - | } |
|
| 438 | - | let path = self.root.join(rel); |
|
| 434 | + | // Entries in the racy window are hashed, in parallel (a checkout |
|
| 435 | + | // just wrote them all). |
|
| 436 | + | use rayon::prelude::*; |
|
| 437 | + | let racy: Vec<(&String, &ManifestEntry)> = m |
|
| 438 | + | .entries |
|
| 439 | + | .iter() |
|
| 440 | + | .filter(|(_, e)| e.mtime_ns + 2_000_000_000 >= m.written_ns) |
|
| 441 | + | .collect(); |
|
| 442 | + | let root = &self.root; |
|
| 443 | + | let changed = racy.par_iter().find_any(|(rel, e)| { |
|
| 444 | + | let path = root.join(rel.as_str()); |
|
| 439 | 445 | let same = match fs::symlink_metadata(&path) { |
|
| 440 | 446 | Ok(md) if md.file_type().is_symlink() => fs::read_link(&path).is_ok_and(|t| { |
|
| 441 | 447 | *blake3::hash(t.to_string_lossy().as_bytes()).as_bytes() == e.hash |
|
| … | 4 unchanged lines |
||
| 446 | 452 | } |
|
| 447 | 453 | _ => false, |
|
| 448 | 454 | }; |
|
| 449 | - | if !same { |
|
| 450 | - | return not(&format!("{rel} changed within the racy window")); |
|
| 451 | - | } |
|
| 455 | + | !same |
|
| 456 | + | }); |
|
| 457 | + | if let Some((rel, _)) = changed { |
|
| 458 | + | return not(&format!("{rel} changed within the racy window")); |
|
| 452 | 459 | } |
|
| 453 | 460 | if self.quick_dirty_keeping(true)? != Some(false) { |
|
| 454 | 461 | return not("a file differs"); |
|
| … | 161 unchanged lines |
modified crates/oxy-core/src/state.rs +5 −0
| … | 665 unchanged lines |
||
| 666 | 666 | })); |
|
| 667 | 667 | // And the encodings of changes whose inline inserts are needed, |
|
| 668 | 668 | // each read once. |
|
| 669 | + | let reading = crate::timing::phase("graphs: read changes and blobs"); |
|
| 669 | 670 | let mut changes: HashMap<ChangeHash, Arc<Vec<u8>>> = HashMap::new(); |
|
| 670 | 671 | for t in taken.iter().flat_map(|(_, d)| d.texts.iter()) { |
|
| 671 | 672 | if let TextRef::Inline { change, .. } = t { |
|
| … | 10 unchanged lines |
||
| 682 | 683 | if blobs.len() != refs.len() { |
|
| 683 | 684 | return Err(ObjectError::Schema("graph blobs: wrong count".into())); |
|
| 684 | 685 | } |
|
| 686 | + | drop(reading); |
|
| 687 | + | let lines = crate::timing::phase("graphs: lines of inserts"); |
|
| 685 | 688 | let mut texts = blobs.split_off(bases).into_iter(); |
|
| 686 | 689 | let mut blobs = blobs.into_iter(); |
|
| 687 | 690 | // Pair each graph with its blob, then build them all in parallel: |
|
| … | 43 unchanged lines |
||
| 731 | 734 | }; |
|
| 732 | 735 | work.push((node, d, bytes)); |
|
| 733 | 736 | } |
|
| 737 | + | drop(lines); |
|
| 738 | + | let _t = crate::timing::phase("graphs: build"); |
|
| 734 | 739 | use rayon::prelude::*; |
|
| 735 | 740 | let built: Vec<(NodeId, FileGraph)> = work |
|
| 736 | 741 | .into_par_iter() |
|
| … | 1739 unchanged lines |