Snapshot: the manifest is a stat cache; unchanged opaque files are not read and contribute their recorded blob, with a two-second racy-stat window (status on 1 GiB 2.4 s to 4 ms; spec 0.6s) b7f102fcbb17
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-27 00:11 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
b7f102fcbb1757bfafa4a431d6ef79ef1eb451708c1da946278617e22075e3ea- Depends on
- 135212de5370 23b217d75848 2b872d074ddb 614541c3cebd 6e723b81f5b9 7421309f5372 79ab8528ff62 8f10a07a1dbe 96f7610fa666 982864068a59 c3069537d7c5 cdb5beba7f05 ff0af1ee299d
- Depended on by
- 0389c3e82f6c 1ec3f008e96a 1966f5db15ee 76740263089c d0990c2bff74 40a40bb7b798 0554aaa7ddef eaf3e9068180 c5ce0367eddf da868d5d2646 86e1824f2d0b dfb23a3a8cfa f7510b63cceb cc7a00565161 696bbf29d3eb 0bdc3f91a39f c863c7a7a2d1 75167672ebcd 957fee30826e 4dee277c327a 9ac1fbe211b4 50a046e439e9 ee06a6c557b7
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 53 atoms, 0 tree operations, 9 files
Browse the tree at this change
modified crates/oxy-core/src/repo/checkout.rs +10 −1
| … | 99 unchanged lines |
||
| 100 | 100 | view.workspaces.insert(self.workspace, wc_hash); |
|
| 101 | 101 | self.record_op(&[op_ref], command, view)?; |
|
| 102 | 102 | report.conflicted = target.is_conflicted(); |
|
| 103 | - | self.write_manifest(report.conflicted, false)?; |
|
| 103 | + | let blobs: HashMap<String, ObjectRef> = target |
|
| 104 | + | .files |
|
| 105 | + | .iter() |
|
| 106 | + | .filter_map(|(p, f)| match &f.content { |
|
| 107 | + | FileContent::Opaque { blobs } if f.kind == NodeKind::File && blobs.len() == 1 => Some((p.clone(), blobs[0])), |
|
| 108 | + | _ => None, |
|
| 109 | + | }) |
|
| 110 | + | .collect(); |
|
| 111 | + | let entries = self.walk_working_copy()?; |
|
| 112 | + | self.write_manifest_entries(report.conflicted, false, &entries, &blobs)?; |
|
| 104 | 113 | let _ = self.run_hook( |
|
| 105 | 114 | "post-checkout", |
|
| 106 | 115 | &format!("{{\"event\":\"post-checkout\",\"heads\":{},\"written\":{},\"removed\":{}}}", json_array(heads.iter().map(|h| json_str(&h.to_hex()))), report.written, report.removed), |
|
| … | 67 unchanged lines |
modified crates/oxy-core/src/repo/git.rs +1 −1
| … | 497 unchanged lines |
||
| 498 | 498 | c |
|
| 499 | 499 | } |
|
| 500 | 500 | }; |
|
| 501 | - | entries.insert(path, FsEntry { kind, exec, content }); |
|
| 501 | + | entries.insert(path, FsEntry::new(kind, exec, content)); |
|
| 502 | 502 | } |
|
| 503 | 503 | ||
| 504 | 504 | let built = self.build_from_entries(&base, &base_snapshot, HashMap::new(), &entries)?; |
|
| … | 146 unchanged lines |
modified crates/oxy-core/src/repo/history.rs +1 −1
| … | 415 unchanged lines |
||
| 416 | 416 | None => continue, |
|
| 417 | 417 | }, |
|
| 418 | 418 | }; |
|
| 419 | - | out.insert(p.clone(), FsEntry { kind: src.kind, exec: src.mode.exec, content }); |
|
| 419 | + | out.insert(p.clone(), FsEntry::new(src.kind, src.mode.exec, content)); |
|
| 420 | 420 | } |
|
| 421 | 421 | Ok(out) |
|
| 422 | 422 | }; |
|
| … | 63 unchanged lines |
modified crates/oxy-core/src/repo/mod.rs +42 −4
| … | 103 unchanged lines |
||
| 104 | 104 | pub(crate) struct FsEntry { |
|
| 105 | 105 | pub kind: NodeKind, |
|
| 106 | 106 | pub exec: bool, |
|
| 107 | - | /// File content, or a symlink's target. |
|
| 107 | + | /// File content, or a symlink's target. Empty when `blob` is set. |
|
| 108 | 108 | pub content: Vec<u8>, |
|
| 109 | + | /// The stored blob of an opaque file whose size and mtime are unchanged |
|
| 110 | + | /// since the last manifest (spec §12.3 stat cache); the content was not |
|
| 111 | + | /// read. |
|
| 112 | + | pub blob: Option<ObjectRef>, |
|
| 113 | + | /// BLAKE3 of the content, carried over from the manifest when the stat |
|
| 114 | + | /// is unchanged, so the manifest can be rewritten without rehashing. |
|
| 115 | + | pub content_hash: Option<String>, |
|
| 109 | 116 | } |
|
| 110 | 117 | ||
| 118 | + | impl FsEntry { |
|
| 119 | + | pub(crate) fn new(kind: NodeKind, exec: bool, content: Vec<u8>) -> FsEntry { |
|
| 120 | + | FsEntry { kind, exec, content, blob: None, content_hash: None } |
|
| 121 | + | } |
|
| 122 | + | } |
|
| 123 | + | ||
| 111 | 124 | pub struct Repo { |
|
| 112 | 125 | root: PathBuf, |
|
| 113 | 126 | oxy: PathBuf, |
|
| … | 446 unchanged lines |
||
| 560 | 573 | } |
|
| 561 | 574 | ||
| 562 | 575 | fn walk_working_copy(&self) -> Result<BTreeMap<String, FsEntry>> { |
|
| 576 | + | self.walk_working_copy_with(None) |
|
| 577 | + | } |
|
| 578 | + | ||
| 579 | + | /// Walk the working copy; with a manifest, a file whose size and mtime |
|
| 580 | + | /// match it is not read: an opaque file yields its known blob, a text |
|
| 581 | + | /// file is read but keeps its known hash. Files modified within two |
|
| 582 | + | /// seconds of the manifest are never trusted (a racy stat). |
|
| 583 | + | fn walk_working_copy_with(&self, cache: Option<&prompt::Manifest>) -> Result<BTreeMap<String, FsEntry>> { |
|
| 563 | 584 | let mut wb = ignore::WalkBuilder::new(&self.root); |
|
| 564 | 585 | wb.hidden(false) |
|
| 565 | 586 | .git_ignore(false) |
|
| … | 18 unchanged lines |
||
| 584 | 605 | } |
|
| 585 | 606 | if ft.is_symlink() { |
|
| 586 | 607 | let target = fs::read_link(path)?; |
|
| 587 | - | out.insert(rel, FsEntry { kind: NodeKind::Symlink, exec: false, content: target.to_string_lossy().as_bytes().to_vec() }); |
|
| 608 | + | out.insert(rel, FsEntry::new(NodeKind::Symlink, false, target.to_string_lossy().as_bytes().to_vec())); |
|
| 588 | 609 | } else if ft.is_file() { |
|
| 589 | - | let exec = is_executable(&entry.metadata().map_err(|e| RepoError::Msg(e.to_string()))?); |
|
| 590 | - | out.insert(rel, FsEntry { kind: NodeKind::File, exec, content: fs::read(path)? }); |
|
| 610 | + | let meta = entry.metadata().map_err(|e| RepoError::Msg(e.to_string()))?; |
|
| 611 | + | let exec = is_executable(&meta); |
|
| 612 | + | let cached = cache.and_then(|m| { |
|
| 613 | + | let e = m.entries.get(&rel)?; |
|
| 614 | + | let mtime_ns = prompt::mtime_ns(&meta); |
|
| 615 | + | let racy = mtime_ns + 2_000_000_000 >= m.written_ns; |
|
| 616 | + | (e.size == meta.len() && e.mtime_ns == mtime_ns && !racy).then_some(e) |
|
| 617 | + | }); |
|
| 618 | + | match cached { |
|
| 619 | + | Some(e) if e.blob.is_some_and(|b| self.store.has(&b)) => { |
|
| 620 | + | out.insert(rel, FsEntry { kind: NodeKind::File, exec, content: vec![], blob: e.blob, content_hash: Some(e.hash.clone()) }); |
|
| 621 | + | } |
|
| 622 | + | Some(e) => { |
|
| 623 | + | out.insert(rel, FsEntry { kind: NodeKind::File, exec, content: fs::read(path)?, blob: None, content_hash: Some(e.hash.clone()) }); |
|
| 624 | + | } |
|
| 625 | + | None => { |
|
| 626 | + | out.insert(rel, FsEntry::new(NodeKind::File, exec, fs::read(path)?)); |
|
| 627 | + | } |
|
| 628 | + | } |
|
| 591 | 629 | } |
|
| 592 | 630 | } |
|
| 593 | 631 | Ok(out) |
|
| … | 116 unchanged lines |
modified crates/oxy-core/src/repo/prompt.rs +82 −20
| … | 73 unchanged lines |
||
| 74 | 74 | } |
|
| 75 | 75 | ||
| 76 | 76 | /// Path → (size, mtime in nanoseconds, BLAKE3 hex) as last seen. |
|
| 77 | - | type Manifest = BTreeMap<String, (u64, u128, String)>; |
|
| 77 | + | /// One manifest entry: the file's stat, its content hash, and the stored |
|
| 78 | + | /// blob for an opaque file. |
|
| 79 | + | #[derive(Clone, Debug)] |
|
| 80 | + | pub(crate) struct ManifestEntry { |
|
| 81 | + | pub size: u64, |
|
| 82 | + | pub mtime_ns: u128, |
|
| 83 | + | pub hash: String, |
|
| 84 | + | pub blob: Option<ObjectRef>, |
|
| 85 | + | } |
|
| 86 | + | ||
| 87 | + | #[derive(Clone, Debug, Default)] |
|
| 88 | + | pub(crate) struct Manifest { |
|
| 89 | + | /// When the manifest was written, for the racy-stat guard. |
|
| 90 | + | pub written_ns: u128, |
|
| 91 | + | pub entries: BTreeMap<String, ManifestEntry>, |
|
| 92 | + | } |
|
| 78 | 93 | ||
| 94 | + | pub(crate) fn mtime_ns(m: &fs::Metadata) -> u128 { |
|
| 95 | + | m.modified().ok().and_then(|t| t.duration_since(UNIX_EPOCH).ok()).map(|d| d.as_nanos()).unwrap_or(0) |
|
| 96 | + | } |
|
| 97 | + | ||
| 79 | 98 | pub const DEFAULT_PROMPT_FORMAT: &str = "%b %c%d%x%A%D"; |
|
| 80 | 99 | ||
| 81 | - | /// One manifest line per file: size, mtime in nanoseconds, hash, path. |
|
| 82 | - | fn manifest_line(path: &str, size: u64, mtime_ns: u128, hash: &blake3::Hash) -> String { |
|
| 83 | - | format!("{size}\t{mtime_ns}\t{}\t{path}\n", hash.to_hex()) |
|
| 100 | + | /// One manifest line per file: size, mtime in nanoseconds, hash, blob ref |
|
| 101 | + | /// or `-`, path. |
|
| 102 | + | fn manifest_line(path: &str, size: u64, mtime_ns: u128, hash: &str, blob: Option<ObjectRef>) -> String { |
|
| 103 | + | format!("{size}\t{mtime_ns}\t{hash}\t{}\t{path}\n", blob.map(|b| b.to_hex()).unwrap_or_else(|| "-".into())) |
|
| 84 | 104 | } |
|
| 85 | 105 | ||
| 86 | 106 | impl Repo { |
|
| … | 5 unchanged lines |
||
| 92 | 112 | /// checkout), with header lines saying whether the base state is |
|
| 93 | 113 | /// conflicted and whether the working-copy change holds uncommitted work. |
|
| 94 | 114 | pub(crate) fn write_manifest(&self, conflicted: bool, uncommitted: bool) -> Result<()> { |
|
| 95 | - | let mut text = format!("# conflicted={}\n# uncommitted={}\n", conflicted as u8, uncommitted as u8); |
|
| 96 | - | for (path, e) in self.walk_working_copy()? { |
|
| 97 | - | let full = self.root.join(&path); |
|
| 115 | + | let entries = self.walk_working_copy()?; |
|
| 116 | + | self.write_manifest_entries(conflicted, uncommitted, &entries, &HashMap::new()) |
|
| 117 | + | } |
|
| 118 | + | ||
| 119 | + | /// Write the manifest from entries already walked; `blobs` names the |
|
| 120 | + | /// stored blob of each opaque path so the next walk can skip reading it. |
|
| 121 | + | pub(crate) fn write_manifest_entries(&self, conflicted: bool, uncommitted: bool, entries: &BTreeMap<String, FsEntry>, blobs: &HashMap<String, ObjectRef>) -> Result<()> { |
|
| 122 | + | let written = std::time::SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0); |
|
| 123 | + | let mut text = format!("# conflicted={}\n# uncommitted={}\n# written={written}\n", conflicted as u8, uncommitted as u8); |
|
| 124 | + | for (path, e) in entries { |
|
| 125 | + | let full = self.root.join(path); |
|
| 98 | 126 | let (size, mtime_ns) = match fs::symlink_metadata(&full) { |
|
| 99 | - | Ok(m) => (m.len(), m.modified().ok().and_then(|t| t.duration_since(UNIX_EPOCH).ok()).map(|d| d.as_nanos()).unwrap_or(0)), |
|
| 127 | + | Ok(m) => (m.len(), mtime_ns(&m)), |
|
| 100 | 128 | Err(_) => (e.content.len() as u64, 0), |
|
| 101 | 129 | }; |
|
| 102 | - | text.push_str(&manifest_line(&path, size, mtime_ns, &blake3::hash(&e.content))); |
|
| 130 | + | let hash = match &e.content_hash { |
|
| 131 | + | Some(h) => h.clone(), |
|
| 132 | + | None => blake3::hash(&e.content).to_hex().to_string(), |
|
| 133 | + | }; |
|
| 134 | + | let blob = blobs.get(path).copied().or(e.blob); |
|
| 135 | + | text.push_str(&manifest_line(path, size, mtime_ns, &hash, blob)); |
|
| 103 | 136 | } |
|
| 104 | 137 | let p = self.manifest_path(); |
|
| 105 | 138 | fs::create_dir_all(p.parent().unwrap())?; |
|
| 106 | 139 | write_atomic(&p, text.as_bytes()) |
|
| 107 | 140 | } |
|
| 108 | 141 | ||
| 142 | + | /// Change only the header flags of the current manifest (after a commit |
|
| 143 | + | /// the files are as the snapshot left them). |
|
| 144 | + | pub(crate) fn rewrite_manifest_flags(&self, conflicted: bool, uncommitted: bool) -> Result<()> { |
|
| 145 | + | let p = self.manifest_path(); |
|
| 146 | + | let text = match fs::read_to_string(&p) { |
|
| 147 | + | Ok(t) => t, |
|
| 148 | + | Err(_) => return self.write_manifest(conflicted, uncommitted), |
|
| 149 | + | }; |
|
| 150 | + | let mut out = String::with_capacity(text.len()); |
|
| 151 | + | for line in text.lines() { |
|
| 152 | + | if line.starts_with("# conflicted=") { |
|
| 153 | + | out.push_str(&format!("# conflicted={}\n", conflicted as u8)); |
|
| 154 | + | } else if line.starts_with("# uncommitted=") { |
|
| 155 | + | out.push_str(&format!("# uncommitted={}\n", uncommitted as u8)); |
|
| 156 | + | } else { |
|
| 157 | + | out.push_str(line); |
|
| 158 | + | out.push('\n'); |
|
| 159 | + | } |
|
| 160 | + | } |
|
| 161 | + | write_atomic(&p, out.as_bytes()) |
|
| 162 | + | } |
|
| 163 | + | ||
| 109 | 164 | /// Forget the manifest after an operation that changed the working-copy |
|
| 110 | 165 | /// change without a snapshot (undo, rewrites); the prompt shows blank |
|
| 111 | 166 | /// markers until the next snapshot. |
|
| … | 1 unchanged lines |
||
| 113 | 168 | let _ = fs::remove_file(self.manifest_path()); |
|
| 114 | 169 | } |
|
| 115 | 170 | ||
| 116 | - | fn read_manifest(&self) -> Result<Option<(bool, bool, Manifest)>> { |
|
| 171 | + | pub(crate) fn read_manifest(&self) -> Result<Option<(bool, bool, Manifest)>> { |
|
| 117 | 172 | let text = match fs::read_to_string(self.manifest_path()) { |
|
| 118 | 173 | Ok(t) => t, |
|
| 119 | 174 | Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), |
|
| 120 | 175 | Err(e) => return Err(e.into()), |
|
| 121 | 176 | }; |
|
| 122 | - | let mut out = BTreeMap::new(); |
|
| 177 | + | let mut out = Manifest::default(); |
|
| 123 | 178 | let mut conflicted = false; |
|
| 124 | 179 | let mut uncommitted = false; |
|
| 125 | 180 | for line in text.lines() { |
|
| … | 5 unchanged lines |
||
| 131 | 186 | uncommitted = v.trim() == "1"; |
|
| 132 | 187 | continue; |
|
| 133 | 188 | } |
|
| 134 | - | let f: Vec<&str> = line.splitn(4, '\t').collect(); |
|
| 135 | - | if f.len() == 4 { |
|
| 136 | - | out.insert(f[3].to_string(), (f[0].parse().unwrap_or(0), f[1].parse().unwrap_or(0), f[2].to_string())); |
|
| 189 | + | if let Some(v) = line.strip_prefix("# written=") { |
|
| 190 | + | out.written_ns = v.trim().parse().unwrap_or(u128::MAX); |
|
| 191 | + | continue; |
|
| 192 | + | } |
|
| 193 | + | let f: Vec<&str> = line.splitn(5, '\t').collect(); |
|
| 194 | + | if f.len() == 5 { |
|
| 195 | + | out.entries.insert( |
|
| 196 | + | f[4].to_string(), |
|
| 197 | + | ManifestEntry { size: f[0].parse().unwrap_or(0), mtime_ns: f[1].parse().unwrap_or(0), hash: f[2].to_string(), blob: ObjectRef::from_hex(f[3]) }, |
|
| 198 | + | ); |
|
| 137 | 199 | } |
|
| 138 | 200 | } |
|
| 139 | 201 | Ok(Some((conflicted, uncommitted, out))) |
|
| … | 20 unchanged lines |
||
| 160 | 222 | if !self.in_scope(&rel) { |
|
| 161 | 223 | continue; |
|
| 162 | 224 | } |
|
| 163 | - | let Some((size, mtime_ns, hash)) = manifest.get(&rel) else { return Ok(Some(true)) }; |
|
| 225 | + | let Some(me) = manifest.entries.get(&rel) else { return Ok(Some(true)) }; |
|
| 164 | 226 | seen += 1; |
|
| 165 | 227 | let m = fs::symlink_metadata(entry.path())?; |
|
| 166 | - | let now_mtime = m.modified().ok().and_then(|t| t.duration_since(UNIX_EPOCH).ok()).map(|d| d.as_nanos()).unwrap_or(0); |
|
| 228 | + | let now_mtime = mtime_ns(&m); |
|
| 167 | 229 | if ft.is_symlink() { |
|
| 168 | 230 | let target = fs::read_link(entry.path())?; |
|
| 169 | - | if blake3::hash(target.to_string_lossy().as_bytes()).to_hex().as_str() != hash { |
|
| 231 | + | if blake3::hash(target.to_string_lossy().as_bytes()).to_hex().as_str() != me.hash { |
|
| 170 | 232 | return Ok(Some(true)); |
|
| 171 | 233 | } |
|
| 172 | 234 | continue; |
|
| 173 | 235 | } |
|
| 174 | - | if m.len() != *size { |
|
| 236 | + | if m.len() != me.size { |
|
| 175 | 237 | return Ok(Some(true)); |
|
| 176 | 238 | } |
|
| 177 | - | if now_mtime != *mtime_ns && blake3::hash(&fs::read(entry.path())?).to_hex().as_str() != hash { |
|
| 239 | + | if now_mtime != me.mtime_ns && blake3::hash(&fs::read(entry.path())?).to_hex().as_str() != me.hash { |
|
| 178 | 240 | return Ok(Some(true)); |
|
| 179 | 241 | } |
|
| 180 | 242 | } |
|
| 181 | - | Ok(Some(seen != manifest.len())) |
|
| 243 | + | Ok(Some(seen != manifest.entries.len())) |
|
| 182 | 244 | } |
|
| 183 | 245 | ||
| 184 | 246 | /// Prompt information without recording anything. |
|
| … | 26 unchanged lines |
modified crates/oxy-core/src/repo/snapshot.rs +33 −10
| … | 35 unchanged lines |
||
| 36 | 36 | pub contributors: Vec<ChangeHash>, |
|
| 37 | 37 | pub resolves: bool, |
|
| 38 | 38 | pub statuses: Vec<PathStatus>, |
|
| 39 | + | /// The stored blob of every opaque path, for the manifest's stat cache. |
|
| 40 | + | pub blobs: HashMap<String, ObjectRef>, |
|
| 39 | 41 | } |
|
| 40 | 42 | ||
| 41 | 43 | /// Result of snapshotting the working copy. |
|
| … | 62 unchanged lines |
||
| 104 | 106 | } |
|
| 105 | 107 | let mut statuses = b.statuses; |
|
| 106 | 108 | statuses.sort_by(|a, b| a.path.cmp(&b.path)); |
|
| 107 | - | Ok(BuiltChange { tree_ops: b.tree_ops, edits: b.edits, contributors: b.contributors.into_iter().collect(), resolves: b.resolves, statuses }) |
|
| 109 | + | Ok(BuiltChange { tree_ops: b.tree_ops, edits: b.edits, contributors: b.contributors.into_iter().collect(), resolves: b.resolves, statuses, blobs: b.blobs }) |
|
| 108 | 110 | } |
|
| 109 | 111 | ||
| 110 | 112 | /// Store a change and, when `core.sign = change` and we are its author, |
|
| … | 29 unchanged lines |
||
| 140 | 142 | // Reuse the NodeIds the previous working-copy version assigned to new |
|
| 141 | 143 | // paths, so an unchanged working copy hashes identically (spec §12.3). |
|
| 142 | 144 | let reuse = added_node_paths(&old_wc, &base_snapshot); |
|
| 143 | - | let built = self.build_against(&base, &base_snapshot, reuse)?; |
|
| 145 | + | // The last manifest lets unchanged opaque files go unread. |
|
| 146 | + | let manifest = self.read_manifest()?.map(|(_, _, m)| m); |
|
| 147 | + | let fs_entries = self.walk_working_copy_with(manifest.as_ref())?; |
|
| 148 | + | let built = self.build_from_entries(&base, &base_snapshot, reuse, &fs_entries)?; |
|
| 144 | 149 | ||
| 145 | 150 | let mut wc = Change { |
|
| 146 | 151 | change_id: old_wc.change_id, |
|
| … | 24 unchanged lines |
||
| 171 | 176 | self.record_op(&[op_ref], &format!("{command}{SNAPSHOT_SUFFIX}"), view)?; |
|
| 172 | 177 | } |
|
| 173 | 178 | // The prompt compares against this without recording anything. |
|
| 174 | - | self.write_manifest(base_snapshot.is_conflicted(), !wc.tree_ops.is_empty() || !wc.edits.is_empty())?; |
|
| 179 | + | self.write_manifest_entries(base_snapshot.is_conflicted(), !wc.tree_ops.is_empty() || !wc.edits.is_empty(), &fs_entries, &built.blobs)?; |
|
| 175 | 180 | Ok(SnapshotResult { wc_hash: new_hash, wc, base, base_snapshot, statuses, changed, resolves, contributors }) |
|
| 176 | 181 | } |
|
| 177 | 182 | ||
| … | 98 unchanged lines |
||
| 276 | 281 | view.bookmarks.insert(self.config.bookmark.clone(), self.store.put_state(&StateObject::new(new_heads))?); |
|
| 277 | 282 | view.workspaces.insert(self.workspace, wc_hash); |
|
| 278 | 283 | self.record_op(&[op_ref], &format!("oxy commit -m {message:?}"), view)?; |
|
| 279 | - | self.write_manifest(snap.base_snapshot.is_conflicted() && !snap.resolves, false)?; |
|
| 284 | + | self.rewrite_manifest_flags(snap.base_snapshot.is_conflicted() && !snap.resolves, false)?; |
|
| 280 | 285 | self.maybe_autopack()?; |
|
| 281 | 286 | let _ = self.run_hook("post-commit", &format!("{{\"event\":\"post-commit\",\"change_id\":{},\"hash\":{},\"message\":{}}}", json_str(&snap.wc.change_id.to_hex()), json_str(&hash.to_hex()), json_str(message))); |
|
| 282 | 287 | Ok(hash) |
|
| … | 13 unchanged lines |
||
| 296 | 301 | contributors: BTreeSet<ChangeHash>, |
|
| 297 | 302 | resolves: bool, |
|
| 298 | 303 | attrs: crate::attributes::Attributes, |
|
| 304 | + | blobs: HashMap<String, ObjectRef>, |
|
| 299 | 305 | } |
|
| 300 | 306 | ||
| 301 | 307 | impl<'a> ChangeBuilder<'a> { |
|
| … | 5 unchanged lines |
||
| 307 | 313 | dir_nodes.insert(p.clone(), f.node); |
|
| 308 | 314 | } |
|
| 309 | 315 | } |
|
| 310 | - | ChangeBuilder { base, base_snapshot, tree_ops: vec![], edits: vec![], statuses: vec![], dir_nodes, reuse, contributors: BTreeSet::new(), resolves: false, attrs: crate::attributes::Attributes::default() } |
|
| 316 | + | ChangeBuilder { base, base_snapshot, tree_ops: vec![], edits: vec![], statuses: vec![], dir_nodes, reuse, contributors: BTreeSet::new(), resolves: false, attrs: crate::attributes::Attributes::default(), blobs: HashMap::new() } |
|
| 311 | 317 | } |
|
| 312 | 318 | ||
| 313 | 319 | fn with_attributes(mut self, attrs: crate::attributes::Attributes) -> Self { |
|
| … | 3 unchanged lines |
||
| 317 | 323 | ||
| 318 | 324 | /// Whether a file is opaque: forced by `binary`, forced text by `text` |
|
| 319 | 325 | /// only when it has no NUL bytes, else sniffed. |
|
| 326 | + | /// The blob for an opaque entry: the cached one, else stored now. |
|
| 327 | + | fn blob_of(&mut self, repo: &Repo, path: &str, e: &FsEntry, nocompress: bool) -> Result<ObjectRef> { |
|
| 328 | + | let b = match e.blob { |
|
| 329 | + | Some(b) => b, |
|
| 330 | + | None => repo.store_blob_for(&e.content, nocompress)?, |
|
| 331 | + | }; |
|
| 332 | + | self.blobs.insert(path.to_string(), b); |
|
| 333 | + | Ok(b) |
|
| 334 | + | } |
|
| 335 | + | ||
| 320 | 336 | fn opaque_for(&self, path: &str, e: &FsEntry) -> bool { |
|
| 337 | + | if e.blob.is_some() { |
|
| 338 | + | return true; |
|
| 339 | + | } |
|
| 321 | 340 | if e.kind == NodeKind::Symlink { |
|
| 322 | 341 | return true; |
|
| 323 | 342 | } |
|
| … | 45 unchanged lines |
||
| 369 | 388 | let nocompress = self.attrs.lookup(path).nocompress; |
|
| 370 | 389 | self.tree_ops.push(TreeOp::AddNode { node, kind: e.kind, mode: Mode { exec: e.exec, opaque }, parent, name: name.to_string() }); |
|
| 371 | 390 | if opaque { |
|
| 372 | - | let blob = repo.store_blob_for(&e.content, nocompress)?; |
|
| 391 | + | let blob = self.blob_of(repo, path, e, nocompress)?; |
|
| 373 | 392 | self.edits.push(Edit::Replace { node, old: None, new: Some(blob) }); |
|
| 374 | 393 | } else if !e.content.is_empty() { |
|
| 375 | 394 | let content = self.stored_text(path, &e.content); |
|
| … | 86 unchanged lines |
||
| 462 | 481 | if !live.is_empty() { |
|
| 463 | 482 | self.edits.push(Edit::Delete { node, atoms: ranges(&live) }); |
|
| 464 | 483 | } |
|
| 465 | - | let blob = repo.store_blob_for(&e.content, nocompress)?; |
|
| 484 | + | let blob = self.blob_of(repo, path, e, nocompress)?; |
|
| 466 | 485 | self.edits.push(Edit::Replace { node, old: None, new: Some(blob) }); |
|
| 467 | 486 | status = Some(ChangeStatus::Modified); |
|
| 468 | 487 | } else if bytes != &disk_text { |
|
| … | 13 unchanged lines |
||
| 482 | 501 | } |
|
| 483 | 502 | } |
|
| 484 | 503 | FileContent::Opaque { blobs } => { |
|
| 485 | - | let new_ref = Repo::blob_ref(&e.content); |
|
| 504 | + | let new_ref = e.blob.unwrap_or_else(|| Repo::blob_ref(&e.content)); |
|
| 505 | + | if e.kind == NodeKind::File && wants_opaque { |
|
| 506 | + | // Unchanged or replaced, the file stays opaque: remember its blob. |
|
| 507 | + | self.blobs.insert(path.to_string(), new_ref); |
|
| 508 | + | } |
|
| 486 | 509 | if blobs.len() > 1 { |
|
| 487 | 510 | if !blobs.contains(&new_ref) { |
|
| 488 | 511 | self.statuses.push(PathStatus { path: path.to_string(), status: ChangeStatus::ConflictEdited }); |
|
| … | 17 unchanged lines |
||
| 506 | 529 | }); |
|
| 507 | 530 | } |
|
| 508 | 531 | } else { |
|
| 509 | - | let blob = repo.store_blob_for(&e.content, nocompress)?; |
|
| 532 | + | let blob = self.blob_of(repo, path, e, nocompress)?; |
|
| 510 | 533 | self.edits.push(Edit::Replace { node, old: current, new: Some(blob) }); |
|
| 511 | 534 | } |
|
| 512 | 535 | status = Some(ChangeStatus::Modified); |
|
| … | 227 unchanged lines |
||
| 740 | 763 | similarity(o, &e.content) |
|
| 741 | 764 | } |
|
| 742 | 765 | } |
|
| 743 | - | (None, Some(b)) if Repo::blob_ref(&e.content) == b => 100, |
|
| 766 | + | (None, Some(b)) if e.blob.unwrap_or_else(|| Repo::blob_ref(&e.content)) == b => 100, |
|
| 744 | 767 | _ => 0, |
|
| 745 | 768 | }; |
|
| 746 | 769 | if score >= RENAME_THRESHOLD { |
|
| … | 44 unchanged lines |
modified crates/oxy-core/src/repo/tests.rs +50 −0
| … | 1931 unchanged lines |
||
| 1932 | 1932 | total |
|
| 1933 | 1933 | } |
|
| 1934 | 1934 | ||
| 1935 | + | /// An opaque file whose size and mtime match the manifest is not read on |
|
| 1936 | + | /// the next snapshot: its blob comes from the manifest's stat cache. |
|
| 1937 | + | #[cfg(unix)] |
|
| 1938 | + | #[test] |
|
| 1939 | + | fn stat_cache_skips_reading_unchanged_opaque_files() { |
|
| 1940 | + | use std::os::unix::fs::PermissionsExt; |
|
| 1941 | + | hermetic(); |
|
| 1942 | + | let dir = tempfile::tempdir().unwrap(); |
|
| 1943 | + | let root = dir.path(); |
|
| 1944 | + | let repo = Repo::init(root).unwrap().with_home(root.join("home")); |
|
| 1945 | + | let mut big = vec![0u8; 300 * 1024]; |
|
| 1946 | + | for (i, b) in big.iter_mut().enumerate() { |
|
| 1947 | + | *b = (i as u32).wrapping_mul(2654435761).to_le_bytes()[1]; |
|
| 1948 | + | } |
|
| 1949 | + | big[0] = 0; |
|
| 1950 | + | write(root, "big.bin", &big); |
|
| 1951 | + | write(root, "a.txt", b"text\n"); |
|
| 1952 | + | let h1 = repo.commit("base", &[]).unwrap(); |
|
| 1953 | + | let (_, _, m) = repo.read_manifest().unwrap().unwrap(); |
|
| 1954 | + | assert!(m.entries["big.bin"].blob.is_some(), "opaque files record their blob"); |
|
| 1955 | + | assert!(m.entries["a.txt"].blob.is_none()); |
|
| 1956 | + | let blob = m.entries["big.bin"].blob.unwrap(); |
|
| 1957 | + | assert!(repo.store().has(&blob)); |
|
| 1958 | + | ||
| 1959 | + | // Files modified within two seconds of the manifest are racy and still |
|
| 1960 | + | // read; age the file so the cache applies, then snapshot to record it. |
|
| 1961 | + | let old = std::time::SystemTime::now() - std::time::Duration::from_secs(30); |
|
| 1962 | + | fs::File::options().write(true).open(root.join("big.bin")).unwrap().set_modified(old).unwrap(); |
|
| 1963 | + | assert!(!repo.snapshot("status").unwrap().changed); |
|
| 1964 | + | // Now the file cannot be read at all; the snapshot succeeds without it. |
|
| 1965 | + | fs::set_permissions(root.join("big.bin"), fs::Permissions::from_mode(0o000)).unwrap(); |
|
| 1966 | + | let snap = repo.snapshot("status").unwrap(); |
|
| 1967 | + | assert!(!snap.changed && snap.statuses.is_empty(), "{:?}", snap.statuses); |
|
| 1968 | + | fs::set_permissions(root.join("big.bin"), fs::Permissions::from_mode(0o644)).unwrap(); |
|
| 1969 | + | // A real change (new mtime, new content) is seen and committed correctly. |
|
| 1970 | + | let mut big2 = big.clone(); |
|
| 1971 | + | big2[10] ^= 0xff; |
|
| 1972 | + | write(root, "big.bin", &big2); |
|
| 1973 | + | let snap = repo.snapshot("status").unwrap(); |
|
| 1974 | + | assert_eq!(snap.statuses.iter().map(|s| (s.path.as_str(), s.status)).collect::<Vec<_>>(), vec![("big.bin", ChangeStatus::Modified)]); |
|
| 1975 | + | let h2 = repo.commit("edit", &[]).unwrap(); |
|
| 1976 | + | let files = repo.load_state(&[h2]).unwrap().materialize().files; |
|
| 1977 | + | assert_eq!(files["big.bin"].content, FileContent::Opaque { blobs: vec![Repo::blob_ref(&big2)] }); |
|
| 1978 | + | // Checkout writes a manifest with blobs too, so the next status is cached. |
|
| 1979 | + | repo.checkout(&[h1], false, "checkout").unwrap(); |
|
| 1980 | + | let (_, _, m) = repo.read_manifest().unwrap().unwrap(); |
|
| 1981 | + | assert_eq!(m.entries["big.bin"].blob, Some(Repo::blob_ref(&big))); |
|
| 1982 | + | assert!(!repo.snapshot("status").unwrap().changed); |
|
| 1983 | + | } |
|
| 1984 | + |
modified docs/plan.md +2 −1
| … | 31 unchanged lines |
||
| 32 | 32 | - [x] Rename detection on Git import (similarity ≥ 50%); `oxy git fetch`/`push` against a Git remote via a mirror. (Done: detection lives in the change builder, so working-copy renames are tracked too; fetch/push through `.oxy/git/<key>/mirror.git` with fast-forward chaining; spec 0.6p.) |
|
| 33 | 33 | - [x] Hardening: property tests for the CBOR decoder and pack reader against random and mutated inputs; a concurrent-push test against the server; a 1 GB single-file benchmark for chunking and blobs. (Done: random/mutated CBOR never panics and accepted input is canonical; 300 rounds of pack/index corruption fail cleanly, with new index bounds checks; 8 concurrent HTTP pushes serialise with pull-and-retry; `bench_one_gibibyte_file` is an ignored release-mode test, run with `OXY_BENCH_DIR` on a real disk.) |
|
| 34 | 34 | - [x] Performance follow-up from the 1 GiB benchmark: commit takes 5.6 s on tmpfs but 63 s on the SSD, and clone 49 s, because every object is written with its own fsync (13,715 chunks). Write committed and fetched objects straight into a pack with one fsync, or batch loose writes. (Done: store write batches; small batches stay loose, large ones spill to one pack. SSD: commit 63 s → 8.8 s, clone 49 s → 15.9 s; spec 0.6r.) |
|
| 35 | - | - [ ] Performance follow-up: `oxy status` re-hashes a clean 1 GiB working copy (2.4 s). Record size and mtime in the manifest and skip hashing files whose stat is unchanged (spec allows a stat cache). |
|
| 35 | + | - [x] Performance follow-up: `oxy status` re-hashes a clean 1 GiB working copy (2.4 s). Record size and mtime in the manifest and skip hashing files whose stat is unchanged (spec allows a stat cache). (Done: manifest records blob refs for opaque files; unchanged files are not read; two-second racy window; status on a clean 1 GiB copy 2.4 s → 4 ms; spec 0.6s.) |
|
| 36 | 36 | - [ ] Documentation: `docs/guide.md` (user guide by task) and `docs/protocol.md` (for other implementers, derived from spec §16 and the wire encodings). |
|
| 37 | 37 | - [ ] Spec review pass: resolve or re-word every "MAY ... until" allowance introduced during implementation, and close what can be closed in §27. |
|
| 38 | 38 | ||
| 39 | 39 | ## Log |
|
| 40 | 40 | ||
| 41 | 41 | - 2026-09-26 23:01 plan created. |
|
| 42 | + | - 2026-09-27 09:10 iteration 16: manifest stat cache (status 2.4 s → 4 ms on 1 GiB). 112 tests. |
|
| 42 | 43 | - 2026-09-27 08:40 iteration 15: batched object writes (commit 8.8 s, clone 15.9 s for 1 GiB on SSD). 111 tests. |
|
| 43 | 44 | - 2026-09-27 08:05 iteration 14: hardening (CBOR and pack fuzz-style tests, concurrent pushes, 1 GiB benchmark: write 0.7 s, commit 63 s on SSD / 5.6 s on tmpfs, status 2.4 s, clone 49 s, read back 1.0 s, store 1025.6 MiB, 13,715 objects). 110 tests, 1 ignored benchmark. |
|
| 44 | 45 | - 2026-09-27 07:20 iteration 13: rename detection (snapshots and Git import) and oxy git fetch/push via a mirror (spec 0.6p). 108 tests. |
|
| … | 12 unchanged lines |
modified docs/spec.md +6 −2
| … | 1 unchanged lines |
||
| 2 | 2 | ||
| 3 | 3 | | | | |
|
| 4 | 4 | |---|---| |
|
| 5 | - | | **Status** | Draft 0.6r | |
|
| 5 | + | | **Status** | Draft 0.6s | |
|
| 6 | 6 | | **Format version** | 1 | |
|
| 7 | 7 | | **Date** | 2026-09-26 | |
|
| 8 | 8 | | **Audience** | Implementers, security reviewers, and contributors | |
|
| … | 458 unchanged lines |
||
| 467 | 467 | ||
| 468 | 468 | Implementations MUST support a file-system monitor (Watchman or a native inotify/FSEvents/ReadDirectoryChangesW watcher) and MUST fall back to a stat-based crawl with a persistent cache keyed by `(path, size, mtime, inode, ctime)`. Snapshot latency targets are in §25. |
|
| 469 | 469 | ||
| 470 | + | The reference implementation's cache is the workspace **manifest** written after every snapshot, commit, and checkout: per file its size, mtime, content hash, and, for an opaque file, the ObjectRef of its stored blob. On the next snapshot a file whose size and mtime match is not read: an opaque file contributes its recorded blob, a text file keeps its recorded hash. A file whose mtime lies within two seconds of the manifest's own write time is never trusted (the racy-stat window), so a modification that preserves size and lands in the same timestamp tick is still seen. An implementation MAY use a file-system monitor instead of, or in addition to, the manifest. |
|
| 471 | + | ||
| 470 | 472 | ## 13. Storage engine |
|
| 471 | 473 | ||
| 472 | 474 | ### 13.1 Layout |
|
| … | 374 unchanged lines |
||
| 847 | 849 | ||
| 848 | 850 | --- |
|
| 849 | 851 | ||
| 850 | - | *Changelog.* Draft 0.6r (2026-09-27): batched object writes (§13.4). |
|
| 852 | + | *Changelog.* Draft 0.6s (2026-09-27): the manifest as a stat cache with a racy-stat window (§12.4). |
|
| 853 | + | ||
| 854 | + | Draft 0.6r (2026-09-27): batched object writes (§13.4). |
|
| 851 | 855 | ||
| 852 | 856 | Draft 0.6q (2026-09-27): pack index validation rules (§13.1). |
|
| 853 | 857 | ||
| … | 49 unchanged lines |