Stat cache for text files: a stat-clean file whose recorded hash is the base's rendering takes the base's bytes instead of being read; edits are always read (spec 12.4) 696bbf29d3eb
closes #3
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-28 16:46 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
696bbf29d3eb44ccc65408604e49b8bd3c38a2ab779f9e66cdef9b9148a8c7e1- Depends on
- 0bdc3f91a39f 4dee277c327a 75167672ebcd 982864068a59 b7f102fcbb17 bc489b6c7f69 c863c7a7a2d1 e7ddd552b619 ff0af1ee299d
- Depended on by
- c5ce0367eddf cc7a00565161 afaba5f1306d
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 16 atoms, 0 tree operations, 5 files
Browse the tree at this change
modified crates/oxy-core/src/repo/checkout.rs +1 −1
| … | 162 unchanged lines |
||
| 163 | 163 | _ => None, |
|
| 164 | 164 | }) |
|
| 165 | 165 | .collect(); |
|
| 166 | - | let entries = self.walk_working_copy_with(None, &Self::tracked_kinds(&target))?; |
|
| 166 | + | let entries = self.walk_working_copy_with(None, &Self::tracked_kinds(&target), None)?; |
|
| 167 | 167 | self.write_manifest_entries(report.conflicted, false, &entries, &blobs)?; |
|
| 168 | 168 | let _ = self.run_hook( |
|
| 169 | 169 | "post-checkout", |
|
| … | 76 unchanged lines |
modified crates/oxy-core/src/repo/mod.rs +35 −10
| … | 978 unchanged lines |
||
| 979 | 979 | } |
|
| 980 | 980 | ||
| 981 | 981 | fn walk_working_copy(&self) -> Result<BTreeMap<String, FsEntry>> { |
|
| 982 | - | self.walk_working_copy_with(None, &BTreeMap::new()) |
|
| 982 | + | self.walk_working_copy_with(None, &BTreeMap::new(), None) |
|
| 983 | 983 | } |
|
| 984 | 984 | ||
| 985 | 985 | /// A walker over the working copy that never enters a nested repository |
|
| … | 40 unchanged lines |
||
| 1026 | 1026 | &self, |
|
| 1027 | 1027 | cache: Option<&prompt::Manifest>, |
|
| 1028 | 1028 | tracked: &BTreeMap<String, (NodeKind, Option<ObjectRef>)>, |
|
| 1029 | + | base: Option<&crate::state::Snapshot>, |
|
| 1029 | 1030 | ) -> Result<BTreeMap<String, FsEntry>> { |
|
| 1030 | 1031 | let nested = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); |
|
| 1031 | 1032 | let wb = self.walker(&nested); |
|
| … | 40 unchanged lines |
||
| 1072 | 1073 | let read: Vec<(String, FsEntry)> = files |
|
| 1073 | 1074 | .par_iter() |
|
| 1074 | 1075 | .map(|(rel, path, meta)| { |
|
| 1075 | - | Ok((rel.clone(), read_file_entry(rel, path, meta, cache)?)) |
|
| 1076 | + | let base_text = base.and_then(|b| match b.files.get(rel).map(|f| &f.content) { |
|
| 1077 | + | Some(crate::state::FileContent::Text { bytes, .. }) => { |
|
| 1078 | + | Some(bytes.as_slice()) |
|
| 1079 | + | } |
|
| 1080 | + | _ => None, |
|
| 1081 | + | }); |
|
| 1082 | + | Ok(( |
|
| 1083 | + | rel.clone(), |
|
| 1084 | + | read_file_entry(rel, path, meta, cache, base_text)?, |
|
| 1085 | + | )) |
|
| 1076 | 1086 | }) |
|
| 1077 | 1087 | .collect::<Result<Vec<_>>>()?; |
|
| 1078 | 1088 | for (rel, mut e) in read { |
|
| … | 96 unchanged lines |
||
| 1175 | 1185 | meta: &fs::Metadata, |
|
| 1176 | 1186 | cache: Option<&prompt::Manifest>, |
|
| 1177 | 1187 | ) -> Result<FsEntry> { |
|
| 1178 | - | let mut e = read_file_entry(rel, path, meta, cache)?; |
|
| 1188 | + | let mut e = read_file_entry(rel, path, meta, cache, None)?; |
|
| 1179 | 1189 | if e.content.is_empty() && e.blob.is_some_and(|b| !self.store.has(&b)) { |
|
| 1180 | 1190 | e.content = fs::read(path)?; |
|
| 1181 | 1191 | e.blob = None; |
|
| … | 222 unchanged lines |
||
| 1404 | 1414 | /// size and mtime match the manifest outside the racy window. Touches no |
|
| 1405 | 1415 | /// repository state, so the walk runs it in parallel; the caller checks |
|
| 1406 | 1416 | /// that a cached blob is still in the store. |
|
| 1417 | + | /// Read one working-copy file. With a manifest whose stat for it is |
|
| 1418 | + | /// unchanged (and not racy), an opaque file is not read at all (its stored |
|
| 1419 | + | /// blob stands in), and a text file whose recorded hash equals the base |
|
| 1420 | + | /// state's rendering of that path takes the base's bytes instead of reading |
|
| 1421 | + | /// the disk (spec §12.3): equal hashes mean equal bytes, so the change built |
|
| 1422 | + | /// from it is the same. |
|
| 1407 | 1423 | fn read_file_entry( |
|
| 1408 | 1424 | rel: &str, |
|
| 1409 | 1425 | path: &Path, |
|
| 1410 | 1426 | meta: &fs::Metadata, |
|
| 1411 | 1427 | cache: Option<&prompt::Manifest>, |
|
| 1428 | + | base_text: Option<&[u8]>, |
|
| 1412 | 1429 | ) -> Result<FsEntry> { |
|
| 1413 | 1430 | let exec = is_executable(meta); |
|
| 1414 | 1431 | let cached = cache.and_then(|m| { |
|
| … | 10 unchanged lines |
||
| 1425 | 1442 | blob: e.blob, |
|
| 1426 | 1443 | content_hash: Some(e.hash.clone()), |
|
| 1427 | 1444 | }, |
|
| 1428 | - | Some(e) => FsEntry { |
|
| 1429 | - | kind: NodeKind::File, |
|
| 1430 | - | exec, |
|
| 1431 | - | content: fs::read(path)?, |
|
| 1432 | - | blob: None, |
|
| 1433 | - | content_hash: Some(e.hash.clone()), |
|
| 1434 | - | }, |
|
| 1445 | + | Some(e) => { |
|
| 1446 | + | let from_base = base_text.filter(|b| { |
|
| 1447 | + | b.len() as u64 == e.size && blake3::hash(b).to_hex().as_str() == e.hash |
|
| 1448 | + | }); |
|
| 1449 | + | FsEntry { |
|
| 1450 | + | kind: NodeKind::File, |
|
| 1451 | + | exec, |
|
| 1452 | + | content: match from_base { |
|
| 1453 | + | Some(b) => b.to_vec(), |
|
| 1454 | + | None => fs::read(path)?, |
|
| 1455 | + | }, |
|
| 1456 | + | blob: None, |
|
| 1457 | + | content_hash: Some(e.hash.clone()), |
|
| 1458 | + | } |
|
| 1459 | + | } |
|
| 1435 | 1460 | None => FsEntry::new(NodeKind::File, exec, fs::read(path)?), |
|
| 1436 | 1461 | }) |
|
| 1437 | 1462 | } |
modified crates/oxy-core/src/repo/snapshot.rs +6 −2
| … | 73 unchanged lines |
||
| 74 | 74 | reuse: HashMap<String, (NodeId, NodeKind)>, |
|
| 75 | 75 | ) -> Result<BuiltChange> { |
|
| 76 | 76 | let mut fs_entries = |
|
| 77 | - | self.walk_working_copy_with(None, &Self::tracked_kinds(base_snapshot))?; |
|
| 77 | + | self.walk_working_copy_with(None, &Self::tracked_kinds(base_snapshot), None)?; |
|
| 78 | 78 | self.platform_normalize(&mut fs_entries, base_snapshot); |
|
| 79 | 79 | self.build_from_entries(base, base_snapshot, reuse, &fs_entries) |
|
| 80 | 80 | } |
|
| … | 137 unchanged lines |
||
| 218 | 218 | let manifest = self.read_manifest()?.map(|(_, _, m)| m); |
|
| 219 | 219 | let mut fs_entries = { |
|
| 220 | 220 | let _t = crate::timing::phase("snapshot: read working copy"); |
|
| 221 | - | self.walk_working_copy_with(manifest.as_ref(), &Self::tracked_kinds(&base_snapshot))? |
|
| 221 | + | self.walk_working_copy_with( |
|
| 222 | + | manifest.as_ref(), |
|
| 223 | + | &Self::tracked_kinds(&base_snapshot), |
|
| 224 | + | Some(&base_snapshot), |
|
| 225 | + | )? |
|
| 222 | 226 | }; |
|
| 223 | 227 | self.platform_normalize(&mut fs_entries, &base_snapshot); |
|
| 224 | 228 | let built = { |
|
| … | 1861 unchanged lines |
modified crates/oxy-core/src/repo/tests.rs +85 −0
| … | 4596 unchanged lines |
||
| 4597 | 4597 | ); |
|
| 4598 | 4598 | assert!(repo.blame_of(&state, &tree, "missing.txt").is_err()); |
|
| 4599 | 4599 | } |
|
| 4600 | + | ||
| 4601 | + | /// A text file whose stat matches the manifest and whose recorded hash is |
|
| 4602 | + | /// the base's content is taken from the base without reading it; anything |
|
| 4603 | + | /// else is read, so no edit is missed (spec §12.3). |
|
| 4604 | + | #[cfg(unix)] |
|
| 4605 | + | #[test] |
|
| 4606 | + | fn stat_cache_takes_unchanged_text_from_the_base_and_misses_no_edit() { |
|
| 4607 | + | use std::os::unix::fs::PermissionsExt; |
|
| 4608 | + | hermetic(); |
|
| 4609 | + | let dir = tempfile::tempdir().unwrap(); |
|
| 4610 | + | let root = dir.path(); |
|
| 4611 | + | let repo = Repo::init(root).unwrap().with_home(root.join("home")); |
|
| 4612 | + | write(root, "a.txt", b"one\ntwo\n"); |
|
| 4613 | + | write(root, "b.txt", b"bee\n"); |
|
| 4614 | + | repo.commit("base", &[]).unwrap(); |
|
| 4615 | + | let age = |p: &str| { |
|
| 4616 | + | let old = std::time::SystemTime::now() - std::time::Duration::from_secs(30); |
|
| 4617 | + | fs::File::options() |
|
| 4618 | + | .write(true) |
|
| 4619 | + | .open(root.join(p)) |
|
| 4620 | + | .unwrap() |
|
| 4621 | + | .set_modified(old) |
|
| 4622 | + | .unwrap(); |
|
| 4623 | + | }; |
|
| 4624 | + | age("a.txt"); |
|
| 4625 | + | age("b.txt"); |
|
| 4626 | + | assert!( |
|
| 4627 | + | !repo.snapshot("status").unwrap().changed, |
|
| 4628 | + | "records the aged stat" |
|
| 4629 | + | ); |
|
| 4630 | + | ||
| 4631 | + | // Clean and equal to the base: not read at all. |
|
| 4632 | + | fs::set_permissions(root.join("a.txt"), fs::Permissions::from_mode(0o000)).unwrap(); |
|
| 4633 | + | let snap = repo.snapshot("status").unwrap(); |
|
| 4634 | + | assert!( |
|
| 4635 | + | !snap.changed && snap.statuses.is_empty(), |
|
| 4636 | + | "{:?}", |
|
| 4637 | + | snap.statuses |
|
| 4638 | + | ); |
|
| 4639 | + | fs::set_permissions(root.join("a.txt"), fs::Permissions::from_mode(0o644)).unwrap(); |
|
| 4640 | + | ||
| 4641 | + | // An uncommitted edit that is older than the manifest: stat-clean, but |
|
| 4642 | + | // its hash is not the base's, so it is read and stays visible. |
|
| 4643 | + | write(root, "b.txt", b"bee edited\n"); |
|
| 4644 | + | age("b.txt"); |
|
| 4645 | + | let first = repo.snapshot("status").unwrap(); |
|
| 4646 | + | assert_eq!( |
|
| 4647 | + | statuses(&first), |
|
| 4648 | + | vec![("b.txt".to_string(), ChangeStatus::Modified)] |
|
| 4649 | + | ); |
|
| 4650 | + | let again = repo.snapshot("status").unwrap(); |
|
| 4651 | + | assert_eq!( |
|
| 4652 | + | statuses(&again), |
|
| 4653 | + | vec![("b.txt".to_string(), ChangeStatus::Modified)], |
|
| 4654 | + | "still seen once stat-clean" |
|
| 4655 | + | ); |
|
| 4656 | + | assert!(!again.changed, "and the working copy hashes the same"); |
|
| 4657 | + | fs::set_permissions(root.join("b.txt"), fs::Permissions::from_mode(0o000)).unwrap(); |
|
| 4658 | + | assert!( |
|
| 4659 | + | repo.snapshot("status").is_err(), |
|
| 4660 | + | "an edited file must be read, never taken from the base" |
|
| 4661 | + | ); |
|
| 4662 | + | fs::set_permissions(root.join("b.txt"), fs::Permissions::from_mode(0o644)).unwrap(); |
|
| 4663 | + | ||
| 4664 | + | // A same-size edit made just now is racy and read. |
|
| 4665 | + | write(root, "a.txt", b"one\nTWO\n"); |
|
| 4666 | + | assert_eq!( |
|
| 4667 | + | statuses(&repo.snapshot("status").unwrap()), |
|
| 4668 | + | vec![ |
|
| 4669 | + | ("a.txt".to_string(), ChangeStatus::Modified), |
|
| 4670 | + | ("b.txt".to_string(), ChangeStatus::Modified) |
|
| 4671 | + | ] |
|
| 4672 | + | ); |
|
| 4673 | + | // Committing makes the edits the base: both are clean and served from it again. |
|
| 4674 | + | repo.commit("edits", &[]).unwrap(); |
|
| 4675 | + | age("a.txt"); |
|
| 4676 | + | age("b.txt"); |
|
| 4677 | + | assert!(repo.snapshot("status").unwrap().statuses.is_empty()); |
|
| 4678 | + | fs::set_permissions(root.join("a.txt"), fs::Permissions::from_mode(0o000)).unwrap(); |
|
| 4679 | + | fs::set_permissions(root.join("b.txt"), fs::Permissions::from_mode(0o000)).unwrap(); |
|
| 4680 | + | let snap = repo.snapshot("status").unwrap(); |
|
| 4681 | + | assert!(!snap.changed && snap.statuses.is_empty()); |
|
| 4682 | + | fs::set_permissions(root.join("a.txt"), fs::Permissions::from_mode(0o644)).unwrap(); |
|
| 4683 | + | fs::set_permissions(root.join("b.txt"), fs::Permissions::from_mode(0o644)).unwrap(); |
|
| 4684 | + | } |
modified docs/spec.md +1 −1
| … | 493 unchanged lines |
||
| 494 | 494 | ||
| 495 | 495 | Implementations MUST support a stat-based crawl with a persistent cache keyed by at least `(path, size, mtime)` and SHOULD support a file-system monitor (Watchman or a native inotify/FSEvents/ReadDirectoryChangesW watcher) for the latency targets in §25. |
|
| 496 | 496 | ||
| 497 | - | The reference implementation's cache is the workspace **manifest** written after every snapshot, commit, and checkout: per file its size, mtime, content hash, and, for an opaque file, the ObjectRef of its stored blob. On the next snapshot a file whose size and mtime match is not read: an opaque file contributes its recorded blob, a text file keeps its recorded hash. A file whose mtime lies within two seconds of the manifest's own write time is never trusted (the racy-stat window), so a modification that preserves size and lands in the same timestamp tick is still seen. An implementation MAY use a file-system monitor instead of, or in addition to, the manifest. |
|
| 497 | + | The reference implementation's cache is the workspace **manifest** written after every snapshot, commit, and checkout: per file its size, mtime, content hash, and, for an opaque file, the ObjectRef of its stored blob. On the next snapshot a file whose size and mtime match is not read: an opaque file contributes its recorded blob, a text file keeps its recorded hash, and when that hash equals the hash of the base state's rendering of the path, its bytes are taken from the base instead of the disk (equal hashes mean equal bytes; a stat-clean file that differs from the base, such as an older uncommitted edit, is always read). A file whose mtime lies within two seconds of the manifest's own write time is never trusted (the racy-stat window), so a modification that preserves size and lands in the same timestamp tick is still seen. An implementation MAY use a file-system monitor instead of, or in addition to, the manifest. |
|
| 498 | 498 | ||
| 499 | 499 | ## 13. Storage engine |
|
| 500 | 500 | ||
| … | 504 unchanged lines |