Status snapshots skip files unchanged since the manifest cc7a00565161
The manifest records, per text file equal to the base, the key of the base graph it matched: the node, its checkpoint graph and the set of later changes that edited it. When a file's stat and key both still match, the snapshot neither reads nor renders it. Status, log, show, tag and bookmark use a partial snapshot that builds only the other graphs; commit and diff build the whole base as before. .oxyattributes files are always read, and any change to them turns skipping off. Spec §12.4, Draft 0.8k. At 10 000 files a clean status takes 0.22 s instead of 0.55 s and 51 MB instead of 259 MB; log 0.38 s instead of about 0.75 s. closes #11
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-28 21:25 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
cc7a00565161b0f3cdee591a879b4088d2018526dd7a8b9a846ab6699765e07a- Depends on
- 0bdc3f91a39f 132b884b9a8d 135212de5370 1d360c91a289 3c5c6ad2518a 4e88e774777a 614541c3cebd 696bbf29d3eb 75036530b621 75167672ebcd 7b9809e67c3d 84b9e9ac0ab4 8f10a07a1dbe 94084087a876 982864068a59 9ac1fbe211b4 afaba5f1306d b037ade16953 b7f102fcbb17 c6aa5bd9d1c8 e3d071cecfe2 e7ddd552b619 e8d38e47e9d7 eae595568832 fce44ae9ad29 ff0af1ee299d
- Depended on by
- c2e1d09047db b689779e9e74 5552e69c7325 2c9ddac3c387 e96a430c78f3 cd4eb55e3b3d 1ec3f008e96a 091fef80755d f9466021286a d616faf5d136 4cce912a0953 36e51aa806c6 3f1b8fe25ec9 1966f5db15ee 76740263089c 06d16dc2bc13 992467d31d26 d0990c2bff74 f9575918477f 40a40bb7b798 e83354cdac22 d0d52107b074 57d8dc05c9d4 519c7c061bcb 01a47cd7f0c8 8b276e0106d1 69816d688300 0554aaa7ddef a4fad5b085c7 c5ce0367eddf da868d5d2646 86e1824f2d0b dfb23a3a8cfa 8ab265dc4a51 f8c695764c95 54cd08a1e7bf
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 64 atoms, 0 tree operations, 11 files
Browse the tree at this change
modified crates/oxy-core/src/repo/hosted.rs +1 −4
| … | 41 unchanged lines |
||
| 42 | 42 | FsEntry::new(*kind, f.mode.exec, bytes.clone()) |
|
| 43 | 43 | } |
|
| 44 | 44 | (kind, FileContent::Opaque { blobs }) => FsEntry { |
|
| 45 | - | kind: *kind, |
|
| 46 | - | exec: f.mode.exec, |
|
| 47 | - | content: vec![], |
|
| 48 | 45 | blob: blobs.first().copied(), |
|
| 49 | - | content_hash: None, |
|
| 46 | + | ..FsEntry::new(*kind, f.mode.exec, vec![]) |
|
| 50 | 47 | }, |
|
| 51 | 48 | }; |
|
| 52 | 49 | entries.insert(path.clone(), entry); |
|
| … | 518 unchanged lines |
modified crates/oxy-core/src/repo/mod.rs +91 −55
| … | 25 unchanged lines |
||
| 26 | 26 | use crate::subrepo::Pin; |
|
| 27 | 27 | use crate::tree::OpKey; |
|
| 28 | 28 | use rand::RngCore; |
|
| 29 | - | use std::collections::{BTreeMap, BTreeSet, HashMap}; |
|
| 29 | + | use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; |
|
| 30 | 30 | use std::fs; |
|
| 31 | 31 | use std::io::Write; |
|
| 32 | 32 | use std::path::{Path, PathBuf}; |
|
| … | 96 unchanged lines |
||
| 129 | 129 | /// BLAKE3 of the content, carried over from the manifest when the stat |
|
| 130 | 130 | /// is unchanged, so the manifest can be rewritten without rehashing. |
|
| 131 | 131 | pub content_hash: Option<String>, |
|
| 132 | + | /// A text file known equal to the base's rendering without reading it: |
|
| 133 | + | /// its stat matches the manifest and the manifest recorded it equal to |
|
| 134 | + | /// a base file graph with the same key (spec §12.3). `content` is empty. |
|
| 135 | + | pub unchanged: bool, |
|
| 136 | + | /// The base graph key this file equals, for the next manifest. |
|
| 137 | + | pub graph_key: Option<String>, |
|
| 132 | 138 | } |
|
| 133 | 139 | ||
| 134 | 140 | impl FsEntry { |
|
| … | 4 unchanged lines |
||
| 139 | 145 | content, |
|
| 140 | 146 | blob: None, |
|
| 141 | 147 | content_hash: None, |
|
| 148 | + | unchanged: false, |
|
| 149 | + | graph_key: None, |
|
| 142 | 150 | } |
|
| 143 | 151 | } |
|
| 144 | 152 | } |
|
| … | 647 unchanged lines |
||
| 792 | 800 | /// Build these file graphs in a history-only state. |
|
| 793 | 801 | pub fn load_graphs_of(&self, state: &mut State, nodes: &[crate::ids::NodeId]) -> Result<()> { |
|
| 794 | 802 | Ok(state.load_graphs(nodes, |refs| { |
|
| 803 | + | self.read_blobs(refs).map_err(|e| { |
|
| 804 | + | crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}")) |
|
| 805 | + | }) |
|
| 806 | + | })?) |
|
| 807 | + | } |
|
| 808 | + | ||
| 809 | + | /// Build every graph a history-only state still lacks, making it full. |
|
| 810 | + | pub fn load_all_graphs(&self, state: &mut State) -> Result<()> { |
|
| 811 | + | Ok(state.load_all_graphs(|refs| { |
|
| 795 | 812 | self.read_blobs(refs).map_err(|e| { |
|
| 796 | 813 | crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}")) |
|
| 797 | 814 | }) |
|
| … | 280 unchanged lines |
||
| 1078 | 1095 | } |
|
| 1079 | 1096 | ||
| 1080 | 1097 | /// Walk the working copy; with a manifest, a file whose size and mtime |
|
| 1081 | - | /// match it is not read: an opaque file yields its known blob, a text |
|
| 1082 | - | /// file is read but keeps its known hash. Files modified within two |
|
| 1083 | - | /// seconds of the manifest are never trusted (a racy stat). |
|
| 1098 | + | /// match it is not read: an opaque file yields its known blob, and a |
|
| 1099 | + | /// text file whose recorded graph key equals its base graph's key in |
|
| 1100 | + | /// `keys` (path → key) is marked unchanged; other text files are read |
|
| 1101 | + | /// but keep their known hash. Files modified within two seconds of the |
|
| 1102 | + | /// manifest are never trusted (a racy stat). |
|
| 1103 | + | /// |
|
| 1104 | + | /// `.oxyattributes` files are always read, since their content is |
|
| 1105 | + | /// needed; and when any of them is not known unchanged (added, removed, |
|
| 1106 | + | /// edited, or changed in the base), no text file is taken as unchanged, |
|
| 1107 | + | /// because attributes decide how a file compares with the base. |
|
| 1084 | 1108 | /// |
|
| 1085 | 1109 | /// `.oxyignore` applies to untracked paths only: every path in `tracked` |
|
| 1086 | 1110 | /// (the base state's files) that exists on disk is included even when |
|
| … | 2 unchanged lines |
||
| 1089 | 1113 | &self, |
|
| 1090 | 1114 | cache: Option<&prompt::Manifest>, |
|
| 1091 | 1115 | tracked: &BTreeMap<String, (NodeKind, Option<ObjectRef>)>, |
|
| 1092 | - | base: Option<&crate::state::Snapshot>, |
|
| 1116 | + | keys: Option<&HashMap<String, String>>, |
|
| 1093 | 1117 | ) -> Result<BTreeMap<String, FsEntry>> { |
|
| 1094 | 1118 | let nested = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); |
|
| 1095 | 1119 | let wb = self.walker(&nested); |
|
| … | 34 unchanged lines |
||
| 1130 | 1154 | files.push((rel, path.to_path_buf(), meta)); |
|
| 1131 | 1155 | } |
|
| 1132 | 1156 | } |
|
| 1157 | + | let keys = keys.filter(|k| { |
|
| 1158 | + | let on_disk: HashSet<&str> = files.iter().map(|(r, _, _)| r.as_str()).collect(); |
|
| 1159 | + | let attrs_unchanged = |
|
| 1160 | + | files |
|
| 1161 | + | .iter() |
|
| 1162 | + | .filter(|(r, _, _)| is_attributes(r)) |
|
| 1163 | + | .all(|(rel, _, meta)| { |
|
| 1164 | + | stat_cached(rel, meta, cache) |
|
| 1165 | + | .is_some_and(|e| e.graph.is_some() && e.graph.as_ref() == k.get(rel)) |
|
| 1166 | + | }); |
|
| 1167 | + | let none_removed = tracked |
|
| 1168 | + | .keys() |
|
| 1169 | + | .filter(|r| is_attributes(r)) |
|
| 1170 | + | .all(|r| on_disk.contains(r.as_str())); |
|
| 1171 | + | attrs_unchanged && none_removed |
|
| 1172 | + | }); |
|
| 1133 | 1173 | // Reading files is most of a walk's cost; do it in parallel. |
|
| 1134 | 1174 | { |
|
| 1135 | 1175 | use rayon::prelude::*; |
|
| 1136 | 1176 | let read: Vec<(String, FsEntry)> = files |
|
| 1137 | 1177 | .par_iter() |
|
| 1138 | 1178 | .map(|(rel, path, meta)| { |
|
| 1139 | - | let base_text = base.and_then(|b| match b.files.get(rel).map(|f| &f.content) { |
|
| 1140 | - | Some(crate::state::FileContent::Text { bytes, .. }) => { |
|
| 1141 | - | Some(bytes.as_slice()) |
|
| 1142 | - | } |
|
| 1143 | - | _ => None, |
|
| 1144 | - | }); |
|
| 1145 | - | Ok(( |
|
| 1146 | - | rel.clone(), |
|
| 1147 | - | read_file_entry(rel, path, meta, cache, base_text)?, |
|
| 1148 | - | )) |
|
| 1179 | + | let key = keys |
|
| 1180 | + | .filter(|_| !is_attributes(rel)) |
|
| 1181 | + | .and_then(|k| k.get(rel)) |
|
| 1182 | + | .map(String::as_str); |
|
| 1183 | + | Ok((rel.clone(), read_file_entry(rel, path, meta, cache, key)?)) |
|
| 1149 | 1184 | }) |
|
| 1150 | 1185 | .collect::<Result<Vec<_>>>()?; |
|
| 1151 | 1186 | for (rel, mut e) in read { |
|
| … | 56 unchanged lines |
||
| 1208 | 1243 | out.insert( |
|
| 1209 | 1244 | rel.clone(), |
|
| 1210 | 1245 | FsEntry { |
|
| 1211 | - | kind: NodeKind::Subrepo, |
|
| 1212 | - | exec: false, |
|
| 1213 | - | content: vec![], |
|
| 1214 | 1246 | blob: Some(*b), |
|
| 1215 | - | content_hash: None, |
|
| 1247 | + | ..FsEntry::new(NodeKind::Subrepo, false, vec![]) |
|
| 1216 | 1248 | }, |
|
| 1217 | 1249 | ); |
|
| 1218 | 1250 | } |
|
| … | 253 unchanged lines |
||
| 1472 | 1504 | Some(i) => (&path[..i], &path[i + 1..]), |
|
| 1473 | 1505 | None => ("", path), |
|
| 1474 | 1506 | } |
|
| 1507 | + | } |
|
| 1508 | + | ||
| 1509 | + | /// Whether `rel` is an attributes file (spec §7.6). |
|
| 1510 | + | fn is_attributes(rel: &str) -> bool { |
|
| 1511 | + | rel == ".oxyattributes" || rel.ends_with("/.oxyattributes") |
|
| 1475 | 1512 | } |
|
| 1476 | 1513 | ||
| 1477 | - | /// One regular file's entry from disk, or from the stat cache when its |
|
| 1478 | - | /// size and mtime match the manifest outside the racy window. Touches no |
|
| 1479 | - | /// repository state, so the walk runs it in parallel; the caller checks |
|
| 1480 | - | /// that a cached blob is still in the store. |
|
| 1481 | - | /// Read one working-copy file. With a manifest whose stat for it is |
|
| 1482 | - | /// unchanged (and not racy), an opaque file is not read at all (its stored |
|
| 1483 | - | /// blob stands in), and a text file whose recorded hash equals the base |
|
| 1484 | - | /// state's rendering of that path takes the base's bytes instead of reading |
|
| 1485 | - | /// the disk (spec §12.3): equal hashes mean equal bytes, so the change built |
|
| 1486 | - | /// from it is the same. |
|
| 1514 | + | /// The manifest entry for `rel` when its size and mtime match it outside |
|
| 1515 | + | /// the racy window. |
|
| 1516 | + | fn stat_cached<'m>( |
|
| 1517 | + | rel: &str, |
|
| 1518 | + | meta: &fs::Metadata, |
|
| 1519 | + | cache: Option<&'m prompt::Manifest>, |
|
| 1520 | + | ) -> Option<&'m prompt::ManifestEntry> { |
|
| 1521 | + | let m = cache?; |
|
| 1522 | + | let e = m.entries.get(rel)?; |
|
| 1523 | + | let mtime_ns = prompt::mtime_ns(meta); |
|
| 1524 | + | let racy = mtime_ns + 2_000_000_000 >= m.written_ns; |
|
| 1525 | + | (e.size == meta.len() && e.mtime_ns == mtime_ns && !racy).then_some(e) |
|
| 1526 | + | } |
|
| 1527 | + | ||
| 1528 | + | /// Read one working-copy file, or serve it from the stat cache (spec |
|
| 1529 | + | /// §12.3). With a manifest whose stat for it is unchanged (and not racy), |
|
| 1530 | + | /// an opaque file is not read at all (its stored blob stands in), and a |
|
| 1531 | + | /// text file the manifest recorded equal to a base graph whose key is |
|
| 1532 | + | /// `key`, the key the base has now, is not read either: equal keys mean |
|
| 1533 | + | /// equal renderings, so it still equals the base. Touches no repository |
|
| 1534 | + | /// state, so the walk runs it in parallel; the caller checks that a cached |
|
| 1535 | + | /// blob is still in the store. |
|
| 1487 | 1536 | fn read_file_entry( |
|
| 1488 | 1537 | rel: &str, |
|
| 1489 | 1538 | path: &Path, |
|
| 1490 | 1539 | meta: &fs::Metadata, |
|
| 1491 | 1540 | cache: Option<&prompt::Manifest>, |
|
| 1492 | - | base_text: Option<&[u8]>, |
|
| 1541 | + | key: Option<&str>, |
|
| 1493 | 1542 | ) -> Result<FsEntry> { |
|
| 1494 | 1543 | let exec = is_executable(meta); |
|
| 1495 | - | let cached = cache.and_then(|m| { |
|
| 1496 | - | let e = m.entries.get(rel)?; |
|
| 1497 | - | let mtime_ns = prompt::mtime_ns(meta); |
|
| 1498 | - | let racy = mtime_ns + 2_000_000_000 >= m.written_ns; |
|
| 1499 | - | (e.size == meta.len() && e.mtime_ns == mtime_ns && !racy).then_some(e) |
|
| 1500 | - | }); |
|
| 1501 | - | Ok(match cached { |
|
| 1544 | + | Ok(match stat_cached(rel, meta, cache) { |
|
| 1502 | 1545 | Some(e) if e.blob.is_some() => FsEntry { |
|
| 1503 | - | kind: NodeKind::File, |
|
| 1504 | - | exec, |
|
| 1505 | - | content: vec![], |
|
| 1506 | 1546 | blob: e.blob, |
|
| 1547 | + | content_hash: Some(e.hash.clone()), |
|
| 1548 | + | ..FsEntry::new(NodeKind::File, exec, vec![]) |
|
| 1549 | + | }, |
|
| 1550 | + | Some(e) if key.is_some() && e.graph.as_deref() == key => FsEntry { |
|
| 1551 | + | content_hash: Some(e.hash.clone()), |
|
| 1552 | + | unchanged: true, |
|
| 1553 | + | graph_key: e.graph.clone(), |
|
| 1554 | + | ..FsEntry::new(NodeKind::File, exec, vec![]) |
|
| 1555 | + | }, |
|
| 1556 | + | Some(e) => FsEntry { |
|
| 1507 | 1557 | content_hash: Some(e.hash.clone()), |
|
| 1558 | + | ..FsEntry::new(NodeKind::File, exec, fs::read(path)?) |
|
| 1508 | 1559 | }, |
|
| 1509 | - | Some(e) => { |
|
| 1510 | - | let from_base = base_text.filter(|b| { |
|
| 1511 | - | b.len() as u64 == e.size && blake3::hash(b).to_hex().as_str() == e.hash |
|
| 1512 | - | }); |
|
| 1513 | - | FsEntry { |
|
| 1514 | - | kind: NodeKind::File, |
|
| 1515 | - | exec, |
|
| 1516 | - | content: match from_base { |
|
| 1517 | - | Some(b) => b.to_vec(), |
|
| 1518 | - | None => fs::read(path)?, |
|
| 1519 | - | }, |
|
| 1520 | - | blob: None, |
|
| 1521 | - | content_hash: Some(e.hash.clone()), |
|
| 1522 | - | } |
|
| 1523 | - | } |
|
| 1524 | 1560 | None => FsEntry::new(NodeKind::File, exec, fs::read(path)?), |
|
| 1525 | 1561 | }) |
|
| 1526 | 1562 | } |
modified crates/oxy-core/src/repo/prompt.rs +20 −7
| … | 132 unchanged lines |
||
| 133 | 133 | pub mtime_ns: u128, |
|
| 134 | 134 | pub hash: String, |
|
| 135 | 135 | pub blob: Option<ObjectRef>, |
|
| 136 | + | /// The graph key (`State::graph_key`) of the base file this text file |
|
| 137 | + | /// was found equal to, rendered, byte for byte. |
|
| 138 | + | pub graph: Option<String>, |
|
| 136 | 139 | } |
|
| 137 | 140 | ||
| 138 | 141 | #[derive(Clone, Debug, Default)] |
|
| … | 18 unchanged lines |
||
| 157 | 160 | pub const GLYPH_PROMPT_FORMAT: &str = "%i %b %c %s%U%O"; |
|
| 158 | 161 | ||
| 159 | 162 | /// One manifest line per file: size, mtime in nanoseconds, hash, blob ref |
|
| 160 | - | /// or `-`, path. |
|
| 163 | + | /// or `-`, graph key or `-`, path. |
|
| 161 | 164 | fn manifest_line( |
|
| 162 | 165 | path: &str, |
|
| 163 | 166 | size: u64, |
|
| 164 | 167 | mtime_ns: u128, |
|
| 165 | 168 | hash: &str, |
|
| 166 | 169 | blob: Option<ObjectRef>, |
|
| 170 | + | graph: Option<&str>, |
|
| 167 | 171 | ) -> String { |
|
| 168 | 172 | format!( |
|
| 169 | - | "{size}\t{mtime_ns}\t{hash}\t{}\t{path}\n", |
|
| 170 | - | blob.map(|b| b.to_hex()).unwrap_or_else(|| "-".into()) |
|
| 173 | + | "{size}\t{mtime_ns}\t{hash}\t{}\t{}\t{path}\n", |
|
| 174 | + | blob.map(|b| b.to_hex()).unwrap_or_else(|| "-".into()), |
|
| 175 | + | graph.unwrap_or("-") |
|
| 171 | 176 | ) |
|
| 172 | 177 | } |
|
| 173 | 178 | ||
| … | 41 unchanged lines |
||
| 215 | 220 | None => blake3::hash(&e.content).to_hex().to_string(), |
|
| 216 | 221 | }; |
|
| 217 | 222 | let blob = blobs.get(path).copied().or(e.blob); |
|
| 218 | - | text.push_str(&manifest_line(path, size, mtime_ns, &hash, blob)); |
|
| 223 | + | text.push_str(&manifest_line( |
|
| 224 | + | path, |
|
| 225 | + | size, |
|
| 226 | + | mtime_ns, |
|
| 227 | + | &hash, |
|
| 228 | + | blob, |
|
| 229 | + | e.graph_key.as_deref(), |
|
| 230 | + | )); |
|
| 219 | 231 | } |
|
| 220 | 232 | let p = self.manifest_path(); |
|
| 221 | 233 | fs::create_dir_all(p.parent().unwrap())?; |
|
| … | 51 unchanged lines |
||
| 273 | 285 | out.written_ns = v.trim().parse().unwrap_or(u128::MAX); |
|
| 274 | 286 | continue; |
|
| 275 | 287 | } |
|
| 276 | - | let f: Vec<&str> = line.splitn(5, '\t').collect(); |
|
| 277 | - | if f.len() == 5 { |
|
| 288 | + | let f: Vec<&str> = line.splitn(6, '\t').collect(); |
|
| 289 | + | if f.len() == 6 { |
|
| 278 | 290 | out.entries.insert( |
|
| 279 | - | f[4].to_string(), |
|
| 291 | + | f[5].to_string(), |
|
| 280 | 292 | ManifestEntry { |
|
| 281 | 293 | size: f[0].parse().unwrap_or(0), |
|
| 282 | 294 | mtime_ns: f[1].parse().unwrap_or(0), |
|
| 283 | 295 | hash: f[2].to_string(), |
|
| 284 | 296 | blob: ObjectRef::from_hex(f[3]), |
|
| 297 | + | graph: (f[4] != "-").then(|| f[4].to_string()), |
|
| 285 | 298 | }, |
|
| 286 | 299 | ); |
|
| 287 | 300 | } |
|
| … | 159 unchanged lines |
modified crates/oxy-core/src/repo/snapshot.rs +112 −17
| … | 189 unchanged lines |
||
| 190 | 190 | } |
|
| 191 | 191 | ||
| 192 | 192 | /// Snapshot the working copy into the working-copy change (spec §12.3). |
|
| 193 | + | /// The result carries the full base state and its whole rendering. |
|
| 193 | 194 | pub fn snapshot(&self, command: &str) -> Result<SnapshotResult> { |
|
| 195 | + | self.snapshot_as(command, false) |
|
| 196 | + | } |
|
| 197 | + | ||
| 198 | + | /// Snapshot the working copy as [`Repo::snapshot`] does, for commands |
|
| 199 | + | /// that need only its statuses, conflicts and base history (status, |
|
| 200 | + | /// log, show). Text files known unchanged since the last manifest are |
|
| 201 | + | /// neither read nor rendered: the result's `base` is a history-only |
|
| 202 | + | /// state without their graphs, and its `base_snapshot` lists them in |
|
| 203 | + | /// `unrendered` with empty bytes. They are never conflicted. |
|
| 204 | + | pub fn snapshot_status(&self, command: &str) -> Result<SnapshotResult> { |
|
| 205 | + | self.snapshot_as(command, true) |
|
| 206 | + | } |
|
| 207 | + | ||
| 208 | + | fn snapshot_as(&self, command: &str, partial: bool) -> Result<SnapshotResult> { |
|
| 194 | 209 | let _lock = self.lock_snapshot()?; |
|
| 195 | 210 | let batch = self.store.begin_batch()?; |
|
| 196 | - | let result = self.snapshot_locked(command); |
|
| 211 | + | let result = self.snapshot_locked(command, partial); |
|
| 197 | 212 | batch.finish()?; |
|
| 198 | 213 | result |
|
| 199 | 214 | } |
|
| 200 | 215 | ||
| 201 | - | fn snapshot_locked(&self, command: &str) -> Result<SnapshotResult> { |
|
| 216 | + | fn snapshot_locked(&self, command: &str, partial: bool) -> Result<SnapshotResult> { |
|
| 202 | 217 | let _all = crate::timing::phase("snapshot"); |
|
| 203 | 218 | let (op_ref, op) = self.head_op()?; |
|
| 204 | 219 | let (old_hash, old_wc) = self.working_copy()?; |
|
| 205 | - | let base = { |
|
| 206 | - | let _t = crate::timing::phase("snapshot: load base state"); |
|
| 207 | - | self.load_state(&old_wc.deps)? |
|
| 220 | + | // The base's history first: its graph keys say which files the |
|
| 221 | + | // manifest already knows equal to it. |
|
| 222 | + | let mut base = { |
|
| 223 | + | let _t = crate::timing::phase("snapshot: load base history"); |
|
| 224 | + | self.load_state_history(&old_wc.deps)? |
|
| 208 | 225 | }; |
|
| 226 | + | let view = base.tree().view(); |
|
| 227 | + | let keys: HashMap<String, String> = view |
|
| 228 | + | .entries |
|
| 229 | + | .iter() |
|
| 230 | + | .filter(|(_, e)| e.kind == NodeKind::File && !e.mode.opaque) |
|
| 231 | + | .filter_map(|(p, e)| { |
|
| 232 | + | let k = base.graph_key(&e.node)?; |
|
| 233 | + | Some((p.clone(), crate::hash::ObjectRef(k).to_hex())) |
|
| 234 | + | }) |
|
| 235 | + | .collect(); |
|
| 236 | + | let manifest = self.read_manifest()?.map(|(_, _, m)| m); |
|
| 237 | + | let tracked: BTreeMap<String, (NodeKind, Option<ObjectRef>)> = { |
|
| 238 | + | let _t = crate::timing::phase("snapshot: tracked paths"); |
|
| 239 | + | let mut t = BTreeMap::new(); |
|
| 240 | + | for (p, e) in &view.entries { |
|
| 241 | + | if e.kind == NodeKind::Dir { |
|
| 242 | + | continue; |
|
| 243 | + | } |
|
| 244 | + | let blob = if e.kind != NodeKind::File || e.mode.opaque { |
|
| 245 | + | base.live_blobs(&e.node).first().copied() |
|
| 246 | + | } else { |
|
| 247 | + | None |
|
| 248 | + | }; |
|
| 249 | + | t.insert(p.clone(), (e.kind, blob)); |
|
| 250 | + | } |
|
| 251 | + | t |
|
| 252 | + | }; |
|
| 253 | + | let mut fs_entries = { |
|
| 254 | + | let _t = crate::timing::phase("snapshot: read working copy"); |
|
| 255 | + | self.walk_working_copy_with(manifest.as_ref(), &tracked, Some(&keys))? |
|
| 256 | + | }; |
|
| 209 | 257 | let base_snapshot = { |
|
| 210 | - | let _t = crate::timing::phase("snapshot: materialize base"); |
|
| 211 | - | base.materialize() |
|
| 258 | + | let _t = crate::timing::phase("snapshot: load and render base"); |
|
| 259 | + | if partial { |
|
| 260 | + | let skip: HashSet<NodeId> = fs_entries |
|
| 261 | + | .iter() |
|
| 262 | + | .filter(|(_, e)| e.unchanged) |
|
| 263 | + | .filter_map(|(p, _)| view.entries.get(p).map(|e| e.node)) |
|
| 264 | + | .collect(); |
|
| 265 | + | let need: Vec<NodeId> = view |
|
| 266 | + | .entries |
|
| 267 | + | .values() |
|
| 268 | + | .filter(|e| e.kind == NodeKind::File && !skip.contains(&e.node)) |
|
| 269 | + | .map(|e| e.node) |
|
| 270 | + | .collect(); |
|
| 271 | + | self.load_graphs_of(&mut base, &need)?; |
|
| 272 | + | base.materialize_except(&skip) |
|
| 273 | + | } else { |
|
| 274 | + | self.load_all_graphs(&mut base)?; |
|
| 275 | + | base.materialize() |
|
| 276 | + | } |
|
| 212 | 277 | }; |
|
| 213 | 278 | ||
| 214 | 279 | // Reuse the NodeIds the previous working-copy version assigned to new |
|
| 215 | 280 | // paths, so an unchanged working copy hashes identically (spec §12.3). |
|
| 216 | 281 | let reuse = added_node_paths(&old_wc, &base_snapshot); |
|
| 217 | - | // The last manifest lets unchanged opaque files go unread. |
|
| 218 | - | let manifest = self.read_manifest()?.map(|(_, _, m)| m); |
|
| 219 | - | let mut fs_entries = { |
|
| 220 | - | let _t = crate::timing::phase("snapshot: read working copy"); |
|
| 221 | - | self.walk_working_copy_with( |
|
| 222 | - | manifest.as_ref(), |
|
| 223 | - | &Self::tracked_kinds(&base_snapshot), |
|
| 224 | - | Some(&base_snapshot), |
|
| 225 | - | )? |
|
| 226 | - | }; |
|
| 227 | 282 | self.platform_normalize(&mut fs_entries, &base_snapshot); |
|
| 228 | 283 | let built = { |
|
| 229 | 284 | let _t = crate::timing::phase("snapshot: diff against base"); |
|
| 230 | 285 | self.build_from_entries(&base, &base_snapshot, reuse, &fs_entries)? |
|
| 231 | 286 | }; |
|
| 287 | + | // Text files found equal to an unconflicted base rendering keep its |
|
| 288 | + | // graph key, so the next snapshot need not read or render them. |
|
| 289 | + | { |
|
| 290 | + | let changed: HashSet<&str> = built.statuses.iter().map(|s| s.path.as_str()).collect(); |
|
| 291 | + | for (p, e) in fs_entries.iter_mut() { |
|
| 292 | + | if e.unchanged || e.kind != NodeKind::File || changed.contains(p.as_str()) { |
|
| 293 | + | continue; |
|
| 294 | + | } |
|
| 295 | + | e.graph_key = match base_snapshot.files.get(p).map(|f| &f.content) { |
|
| 296 | + | Some(FileContent::Text { |
|
| 297 | + | bytes, |
|
| 298 | + | materialized, |
|
| 299 | + | }) if !materialized.is_conflicted() && *bytes == e.content => { |
|
| 300 | + | keys.get(p).cloned() |
|
| 301 | + | } |
|
| 302 | + | _ => None, |
|
| 303 | + | }; |
|
| 304 | + | } |
|
| 305 | + | } |
|
| 232 | 306 | ||
| 233 | 307 | let mut wc = Change { |
|
| 234 | 308 | change_id: old_wc.change_id, |
|
| … | 1321 unchanged lines |
||
| 1556 | 1630 | let entry = &self.base_snapshot.tree.entries[path]; |
|
| 1557 | 1631 | let node = expected.node; |
|
| 1558 | 1632 | let mode_atom = entry.mode_atom.map(op_ref); |
|
| 1633 | + | if e.unchanged { |
|
| 1634 | + | // Known equal to the base's unconflicted rendering (spec |
|
| 1635 | + | // §12.3); only the executable bit can differ. |
|
| 1636 | + | if e.exec != expected.mode.exec { |
|
| 1637 | + | if let Some(old) = mode_atom { |
|
| 1638 | + | self.tree_ops.push(TreeOp::SetMode { |
|
| 1639 | + | node, |
|
| 1640 | + | old, |
|
| 1641 | + | mode: Mode { |
|
| 1642 | + | exec: e.exec, |
|
| 1643 | + | opaque: false, |
|
| 1644 | + | }, |
|
| 1645 | + | }); |
|
| 1646 | + | self.statuses.push(PathStatus { |
|
| 1647 | + | path: path.to_string(), |
|
| 1648 | + | status: ChangeStatus::ModeChanged, |
|
| 1649 | + | }); |
|
| 1650 | + | } |
|
| 1651 | + | } |
|
| 1652 | + | return Ok(()); |
|
| 1653 | + | } |
|
| 1559 | 1654 | let mut status: Option<ChangeStatus> = None; |
|
| 1560 | 1655 | let wants_opaque = self.opaque_for(path, e); |
|
| 1561 | 1656 | let nocompress = self.attrs.lookup(path).nocompress; |
|
| … | 528 unchanged lines |
modified crates/oxy-core/src/repo/tests.rs +134 −0
| … | 4805 unchanged lines |
||
| 4806 | 4806 | .to_string() |
|
| 4807 | 4807 | .contains("does not hold identity")); |
|
| 4808 | 4808 | } |
|
| 4809 | + | ||
| 4810 | + | /// Status snapshots (spec §12.3): a text file whose stat matches the |
|
| 4811 | + | /// manifest, and whose base graph key is the one the manifest recorded it |
|
| 4812 | + | /// equal to, is neither read nor rendered; any edit, and any change to the |
|
| 4813 | + | /// file's graph in the base, brings it back. |
|
| 4814 | + | #[cfg(unix)] |
|
| 4815 | + | #[test] |
|
| 4816 | + | fn status_snapshots_skip_files_unchanged_since_the_manifest() { |
|
| 4817 | + | use std::os::unix::fs::PermissionsExt; |
|
| 4818 | + | hermetic(); |
|
| 4819 | + | let dir = tempfile::tempdir().unwrap(); |
|
| 4820 | + | let root = dir.path(); |
|
| 4821 | + | let repo = Repo::init(root).unwrap().with_home(root.join("home")); |
|
| 4822 | + | for (p, c) in [("a.txt", "one\n"), ("b.txt", "bee\n"), ("d/c.txt", "sea\n")] { |
|
| 4823 | + | write(root, p, c.as_bytes()); |
|
| 4824 | + | } |
|
| 4825 | + | repo.commit("base", &[]).unwrap(); |
|
| 4826 | + | let age = |p: &str| { |
|
| 4827 | + | let old = std::time::SystemTime::now() - std::time::Duration::from_secs(30); |
|
| 4828 | + | fs::File::options() |
|
| 4829 | + | .write(true) |
|
| 4830 | + | .open(root.join(p)) |
|
| 4831 | + | .unwrap() |
|
| 4832 | + | .set_modified(old) |
|
| 4833 | + | .unwrap(); |
|
| 4834 | + | }; |
|
| 4835 | + | let lock = |p: &str, mode: u32| { |
|
| 4836 | + | fs::set_permissions(root.join(p), fs::Permissions::from_mode(mode)).unwrap() |
|
| 4837 | + | }; |
|
| 4838 | + | for p in ["a.txt", "b.txt", "d/c.txt"] { |
|
| 4839 | + | age(p); |
|
| 4840 | + | } |
|
| 4841 | + | assert!(repo.snapshot("status").unwrap().statuses.is_empty()); |
|
| 4842 | + | let (_, _, m) = repo.read_manifest().unwrap().unwrap(); |
|
| 4843 | + | assert!( |
|
| 4844 | + | m.entries.values().all(|e| e.graph.is_some()), |
|
| 4845 | + | "keys recorded" |
|
| 4846 | + | ); |
|
| 4847 | + | ||
| 4848 | + | // Unreadable files prove they are not read; the status snapshot does |
|
| 4849 | + | // not render them either, and its base lacks their graphs. |
|
| 4850 | + | lock("a.txt", 0o000); |
|
| 4851 | + | lock("b.txt", 0o000); |
|
| 4852 | + | let snap = repo.snapshot_status("status").unwrap(); |
|
| 4853 | + | assert!(!snap.changed && snap.statuses.is_empty()); |
|
| 4854 | + | assert_eq!( |
|
| 4855 | + | snap.base_snapshot |
|
| 4856 | + | .unrendered |
|
| 4857 | + | .iter() |
|
| 4858 | + | .cloned() |
|
| 4859 | + | .collect::<Vec<_>>(), |
|
| 4860 | + | vec!["a.txt", "b.txt", "d/c.txt"] |
|
| 4861 | + | ); |
|
| 4862 | + | assert!(!snap.base.is_full()); |
|
| 4863 | + | assert!(!snap.base_snapshot.is_conflicted()); |
|
| 4864 | + | // A full snapshot does not read them either, but renders everything. |
|
| 4865 | + | let full = repo.snapshot("status").unwrap(); |
|
| 4866 | + | assert!(full.statuses.is_empty() && full.base.is_full()); |
|
| 4867 | + | assert!(full.base_snapshot.unrendered.is_empty()); |
|
| 4868 | + | assert!(matches!(&full.base_snapshot.files["a.txt"].content, |
|
| 4869 | + | FileContent::Text { bytes, .. } if bytes == b"one\n")); |
|
| 4870 | + | ||
| 4871 | + | // An edit is seen; the file is rendered, the others still are not. |
|
| 4872 | + | write(root, "d/c.txt", b"sea edited\n"); |
|
| 4873 | + | let snap = repo.snapshot_status("status").unwrap(); |
|
| 4874 | + | assert_eq!( |
|
| 4875 | + | statuses(&snap), |
|
| 4876 | + | vec![("d/c.txt".to_string(), ChangeStatus::Modified)] |
|
| 4877 | + | ); |
|
| 4878 | + | assert!(!snap.base_snapshot.unrendered.contains("d/c.txt")); |
|
| 4879 | + | // So is a mode change of an unread file. |
|
| 4880 | + | lock("b.txt", 0o100); |
|
| 4881 | + | assert_eq!( |
|
| 4882 | + | statuses(&repo.snapshot_status("status").unwrap()), |
|
| 4883 | + | vec![ |
|
| 4884 | + | ("b.txt".to_string(), ChangeStatus::ModeChanged), |
|
| 4885 | + | ("d/c.txt".to_string(), ChangeStatus::Modified) |
|
| 4886 | + | ] |
|
| 4887 | + | ); |
|
| 4888 | + | lock("b.txt", 0o000); |
|
| 4889 | + | ||
| 4890 | + | // Committing c.txt changes its graph in the base, not a.txt's: a.txt |
|
| 4891 | + | // stays unread, c.txt is read once and then skipped too. |
|
| 4892 | + | repo.commit_selection("edit c", &[], &Selection::Paths(vec!["d/c.txt".into()])) |
|
| 4893 | + | .unwrap(); |
|
| 4894 | + | age("d/c.txt"); |
|
| 4895 | + | let snap = repo.snapshot_status("status").unwrap(); |
|
| 4896 | + | assert!(snap.statuses.is_empty(), "{:?}", snap.statuses); |
|
| 4897 | + | assert!(snap.base_snapshot.unrendered.contains("a.txt")); |
|
| 4898 | + | lock("d/c.txt", 0o000); |
|
| 4899 | + | let snap = repo.snapshot_status("status").unwrap(); |
|
| 4900 | + | assert!(snap.statuses.is_empty()); |
|
| 4901 | + | assert!(snap.base_snapshot.unrendered.contains("d/c.txt")); |
|
| 4902 | + | for p in ["a.txt", "b.txt", "d/c.txt"] { |
|
| 4903 | + | lock(p, 0o644); |
|
| 4904 | + | } |
|
| 4905 | + | } |
|
| 4906 | + | ||
| 4907 | + | /// Attributes decide how a file compares with the base, so a changed |
|
| 4908 | + | /// `.oxyattributes` makes every text file be compared again. |
|
| 4909 | + | #[cfg(unix)] |
|
| 4910 | + | #[test] |
|
| 4911 | + | fn a_changed_attributes_file_disables_skipping_unchanged_files() { |
|
| 4912 | + | hermetic(); |
|
| 4913 | + | let dir = tempfile::tempdir().unwrap(); |
|
| 4914 | + | let root = dir.path(); |
|
| 4915 | + | let repo = Repo::init(root).unwrap().with_home(root.join("home")); |
|
| 4916 | + | write(root, "a.txt", b"crlf\r\n"); |
|
| 4917 | + | repo.commit("base", &[]).unwrap(); |
|
| 4918 | + | let old = std::time::SystemTime::now() - std::time::Duration::from_secs(30); |
|
| 4919 | + | fs::File::options() |
|
| 4920 | + | .write(true) |
|
| 4921 | + | .open(root.join("a.txt")) |
|
| 4922 | + | .unwrap() |
|
| 4923 | + | .set_modified(old) |
|
| 4924 | + | .unwrap(); |
|
| 4925 | + | assert!(repo.snapshot_status("status").unwrap().statuses.is_empty()); |
|
| 4926 | + | assert!(repo |
|
| 4927 | + | .snapshot_status("status") |
|
| 4928 | + | .unwrap() |
|
| 4929 | + | .base_snapshot |
|
| 4930 | + | .unrendered |
|
| 4931 | + | .contains("a.txt")); |
|
| 4932 | + | // With eol=lf, a.txt as stored no longer equals the base. |
|
| 4933 | + | write(root, ".oxyattributes", b"*.txt eol=lf\n"); |
|
| 4934 | + | let snap = repo.snapshot_status("status").unwrap(); |
|
| 4935 | + | assert_eq!( |
|
| 4936 | + | statuses(&snap), |
|
| 4937 | + | vec![ |
|
| 4938 | + | (".oxyattributes".to_string(), ChangeStatus::Added), |
|
| 4939 | + | ("a.txt".to_string(), ChangeStatus::Modified) |
|
| 4940 | + | ] |
|
| 4941 | + | ); |
|
| 4942 | + | } |
modified crates/oxy-core/src/state.rs +210 −14
| … | 140 unchanged lines |
||
| 141 | 141 | pub struct Snapshot { |
|
| 142 | 142 | pub tree: TreeView, |
|
| 143 | 143 | pub files: BTreeMap<String, FileState>, |
|
| 144 | + | /// Text files left unrendered by [`State::materialize_except`]: their |
|
| 145 | + | /// `bytes` are empty and they are known not to be conflicted. |
|
| 146 | + | pub unrendered: BTreeSet<String>, |
|
| 144 | 147 | } |
|
| 145 | 148 | ||
| 146 | 149 | impl Snapshot { |
|
| … | 139 unchanged lines |
||
| 286 | 289 | load(&refs)? |
|
| 287 | 290 | } |
|
| 288 | 291 | .into_iter(); |
|
| 289 | - | for (node, d) in taken.iter_mut() { |
|
| 290 | - | let mut graph = match d.base.take() { |
|
| 291 | - | Some(DeferredBase::Inline(v)) => FileGraph::from_value(&v)?, |
|
| 292 | - | Some(DeferredBase::Blob(_)) => { |
|
| 293 | - | let bytes = blobs |
|
| 292 | + | // Pair each graph with its blob, then build them all in parallel: |
|
| 293 | + | // graphs are independent. |
|
| 294 | + | let mut work: Vec<(NodeId, Deferred, Option<Vec<u8>>)> = Vec::with_capacity(taken.len()); |
|
| 295 | + | for (node, d) in taken.drain(..) { |
|
| 296 | + | let bytes = match &d.base { |
|
| 297 | + | Some(DeferredBase::Blob(_)) => Some( |
|
| 298 | + | blobs |
|
| 294 | 299 | .next() |
|
| 295 | - | .ok_or_else(|| ObjectError::Schema("graph blobs: wrong count".into()))?; |
|
| 296 | - | FileGraph::from_value(&cbor::decode(&bytes)?)? |
|
| 297 | - | } |
|
| 298 | - | None => FileGraph::default(), |
|
| 300 | + | .ok_or_else(|| ObjectError::Schema("graph blobs: wrong count".into()))?, |
|
| 301 | + | ), |
|
| 302 | + | _ => None, |
|
| 299 | 303 | }; |
|
| 300 | - | for (h, edits) in std::mem::take(&mut d.edits) { |
|
| 301 | - | apply_file_edits(&mut graph, h, edits); |
|
| 302 | - | } |
|
| 303 | - | self.files.insert(*node, graph); |
|
| 304 | + | work.push((node, d, bytes)); |
|
| 304 | 305 | } |
|
| 306 | + | use rayon::prelude::*; |
|
| 307 | + | let built: Vec<(NodeId, FileGraph)> = work |
|
| 308 | + | .into_par_iter() |
|
| 309 | + | .map(|(node, mut d, bytes)| { |
|
| 310 | + | let mut graph = match (d.base.take(), bytes) { |
|
| 311 | + | (Some(DeferredBase::Inline(v)), _) => FileGraph::from_value(&v)?, |
|
| 312 | + | (Some(DeferredBase::Blob(_)), Some(bytes)) => { |
|
| 313 | + | FileGraph::from_value(&cbor::decode(&bytes)?)? |
|
| 314 | + | } |
|
| 315 | + | _ => FileGraph::default(), |
|
| 316 | + | }; |
|
| 317 | + | for (h, edits) in std::mem::take(&mut d.edits) { |
|
| 318 | + | apply_file_edits(&mut graph, h, edits); |
|
| 319 | + | } |
|
| 320 | + | Ok((node, graph)) |
|
| 321 | + | }) |
|
| 322 | + | .collect::<Result<_, ObjectError>>()?; |
|
| 323 | + | self.files.extend(built); |
|
| 324 | + | Ok(()) |
|
| 325 | + | } |
|
| 326 | + | ||
| 327 | + | /// Build every graph a history-only state has not built, making it full. |
|
| 328 | + | pub fn load_all_graphs( |
|
| 329 | + | &mut self, |
|
| 330 | + | load: impl FnOnce(&[ObjectRef]) -> Result<Vec<Vec<u8>>, ObjectError>, |
|
| 331 | + | ) -> Result<(), ObjectError> { |
|
| 332 | + | let nodes: Vec<NodeId> = match &self.deferred { |
|
| 333 | + | Some(d) => d.keys().copied().collect(), |
|
| 334 | + | None => return Ok(()), |
|
| 335 | + | }; |
|
| 336 | + | self.load_graphs(&nodes, load)?; |
|
| 337 | + | self.deferred = None; |
|
| 305 | 338 | Ok(()) |
|
| 339 | + | } |
|
| 340 | + | ||
| 341 | + | /// A key for the graph of `node` in a history-only state that has not |
|
| 342 | + | /// built it: equal keys mean equal graphs, so equal renderings (a |
|
| 343 | + | /// rendering depends only on the graph and on the changes' own |
|
| 344 | + | /// immutable metadata). It names the node, its checkpoint base (a blob, |
|
| 345 | + | /// an inline value, or none) and the set of later changes that edited |
|
| 346 | + | /// the file. |
|
| 347 | + | /// `None` for a full state or a graph already built. |
|
| 348 | + | pub fn graph_key(&self, node: &NodeId) -> Option<[u8; 32]> { |
|
| 349 | + | let deferred = self.deferred.as_ref()?; |
|
| 350 | + | if self.files.contains_key(node) { |
|
| 351 | + | return None; |
|
| 352 | + | } |
|
| 353 | + | let mut h = blake3::Hasher::new(); |
|
| 354 | + | h.update(b"oxy graph key 1\0"); |
|
| 355 | + | // A change's edits are applied per node: the node is part of it. |
|
| 356 | + | h.update(&node.0); |
|
| 357 | + | let Some(d) = deferred.get(node) else { |
|
| 358 | + | // Never edited: the empty graph. |
|
| 359 | + | h.update(&[0]); |
|
| 360 | + | return Some(*h.finalize().as_bytes()); |
|
| 361 | + | }; |
|
| 362 | + | match &d.base { |
|
| 363 | + | None => h.update(&[0]), |
|
| 364 | + | Some(DeferredBase::Blob(r)) => h.update(&[1]).update(&r.0), |
|
| 365 | + | Some(DeferredBase::Inline(v)) => h.update(&[2]).update(&cbor::encode(v)), |
|
| 366 | + | }; |
|
| 367 | + | let mut changes: Vec<&ChangeHash> = d.edits.iter().map(|(c, _)| c).collect(); |
|
| 368 | + | changes.sort(); |
|
| 369 | + | changes.dedup(); |
|
| 370 | + | h.update(&(changes.len() as u64).to_le_bytes()); |
|
| 371 | + | for c in changes { |
|
| 372 | + | h.update(&c.0); |
|
| 373 | + | } |
|
| 374 | + | Some(*h.finalize().as_bytes()) |
|
| 306 | 375 | } |
|
| 307 | 376 | ||
| 308 | 377 | pub fn len(&self) -> usize { |
|
| … | 609 unchanged lines |
||
| 918 | 987 | .map(|(path, e)| (path.to_string(), self.file_state(e))) |
|
| 919 | 988 | .collect(); |
|
| 920 | 989 | let files: BTreeMap<String, FileState> = rendered.into_iter().collect(); |
|
| 921 | - | Snapshot { tree, files } |
|
| 990 | + | Snapshot { |
|
| 991 | + | tree, |
|
| 992 | + | files, |
|
| 993 | + | unrendered: BTreeSet::new(), |
|
| 994 | + | } |
|
| 995 | + | } |
|
| 996 | + | ||
| 997 | + | /// Like [`State::materialize`], leaving the text files of nodes in |
|
| 998 | + | /// `skip` unrendered (listed in [`Snapshot::unrendered`]); every other |
|
| 999 | + | /// file's graph must be loaded. For a caller that already knows those |
|
| 1000 | + | /// files are unchanged and not conflicted. |
|
| 1001 | + | pub fn materialize_except(&self, skip: &HashSet<NodeId>) -> Snapshot { |
|
| 1002 | + | use rayon::prelude::*; |
|
| 1003 | + | let tree = self.tree.view(); |
|
| 1004 | + | let entries: Vec<(&String, &crate::tree::TreeEntry)> = tree.entries.iter().collect(); |
|
| 1005 | + | let rendered: Vec<(String, FileState, bool)> = entries |
|
| 1006 | + | .par_iter() |
|
| 1007 | + | .map(|(path, e)| { |
|
| 1008 | + | if e.kind == NodeKind::File && !e.mode.opaque && skip.contains(&e.node) { |
|
| 1009 | + | let f = FileState { |
|
| 1010 | + | node: e.node, |
|
| 1011 | + | kind: e.kind, |
|
| 1012 | + | mode: e.mode, |
|
| 1013 | + | content: FileContent::Text { |
|
| 1014 | + | bytes: Vec::new(), |
|
| 1015 | + | materialized: Materialized::default(), |
|
| 1016 | + | }, |
|
| 1017 | + | }; |
|
| 1018 | + | (path.to_string(), f, true) |
|
| 1019 | + | } else { |
|
| 1020 | + | (path.to_string(), self.file_state(e), false) |
|
| 1021 | + | } |
|
| 1022 | + | }) |
|
| 1023 | + | .collect(); |
|
| 1024 | + | let mut files = BTreeMap::new(); |
|
| 1025 | + | let mut unrendered = BTreeSet::new(); |
|
| 1026 | + | for (p, f, skipped) in rendered { |
|
| 1027 | + | if skipped { |
|
| 1028 | + | unrendered.insert(p.clone()); |
|
| 1029 | + | } |
|
| 1030 | + | files.insert(p, f); |
|
| 1031 | + | } |
|
| 1032 | + | Snapshot { |
|
| 1033 | + | tree, |
|
| 1034 | + | files, |
|
| 1035 | + | unrendered, |
|
| 1036 | + | } |
|
| 922 | 1037 | } |
|
| 923 | 1038 | ||
| 924 | 1039 | fn file_state(&self, e: &crate::tree::TreeEntry) -> FileState { |
|
| … | 306 unchanged lines |
||
| 1231 | 1346 | /// History-only states (O10f): the same state as a full load once its |
|
| 1232 | 1347 | /// graphs are built, by replay and from a version 2 checkpoint with |
|
| 1233 | 1348 | /// later changes; unloaded graphs are never read as empty. |
|
| 1349 | + | #[test] |
|
| 1350 | + | fn graph_keys_change_exactly_when_a_graph_does() { |
|
| 1351 | + | let (f, g) = (NodeId::random(), NodeId::random()); |
|
| 1352 | + | let add = |node: NodeId, name: &str| TreeOp::AddNode { |
|
| 1353 | + | node, |
|
| 1354 | + | kind: NodeKind::File, |
|
| 1355 | + | mode: Mode::TEXT, |
|
| 1356 | + | parent: NodeId::ROOT, |
|
| 1357 | + | name: name.into(), |
|
| 1358 | + | }; |
|
| 1359 | + | let ins = |node: NodeId, line: &[u8]| Edit::Insert { |
|
| 1360 | + | node, |
|
| 1361 | + | after: AtomRef::Start, |
|
| 1362 | + | before: AtomRef::End, |
|
| 1363 | + | lines: vec![line.to_vec()], |
|
| 1364 | + | }; |
|
| 1365 | + | let c1 = change( |
|
| 1366 | + | vec![], |
|
| 1367 | + | vec![add(f, "f"), add(g, "g")], |
|
| 1368 | + | vec![ins(f, b"f\n"), ins(g, b"g\n")], |
|
| 1369 | + | ); |
|
| 1370 | + | let h1 = c1.hash(); |
|
| 1371 | + | let c2 = change(vec![h1], vec![], vec![ins(g, b"g2\n")]); |
|
| 1372 | + | let h2 = c2.hash(); |
|
| 1373 | + | let c3 = change(vec![h2], vec![], vec![ins(f, b"f2\n")]); |
|
| 1374 | + | let load = |cs: &[&Change]| { |
|
| 1375 | + | let mut st = State::new_history_only(); |
|
| 1376 | + | for c in cs { |
|
| 1377 | + | st.add((*c).clone()).unwrap(); |
|
| 1378 | + | } |
|
| 1379 | + | st |
|
| 1380 | + | }; |
|
| 1381 | + | let s1 = load(&[&c1]); |
|
| 1382 | + | let s2 = load(&[&c1, &c2]); |
|
| 1383 | + | let s3 = load(&[&c1, &c2, &c3]); |
|
| 1384 | + | let key = |s: &State, n: &NodeId| s.graph_key(n).unwrap(); |
|
| 1385 | + | assert_eq!(key(&s1, &f), key(&load(&[&c1]), &f), "same state, same key"); |
|
| 1386 | + | assert_eq!(key(&s1, &f), key(&s2, &f), "an edit elsewhere keeps it"); |
|
| 1387 | + | assert_ne!(key(&s1, &g), key(&s2, &g), "an edit of the file changes it"); |
|
| 1388 | + | assert_ne!(key(&s2, &f), key(&s3, &f)); |
|
| 1389 | + | assert_ne!(key(&s1, &f), key(&s1, &g), "different files differ"); |
|
| 1390 | + | let mut built = load(&[&c1]); |
|
| 1391 | + | built.load_graphs(&[f], |_| unreachable!()).unwrap(); |
|
| 1392 | + | assert!(built.graph_key(&f).is_none(), "a built graph has no key"); |
|
| 1393 | + | let mut full = State::new(); |
|
| 1394 | + | full.add(c1.clone()).unwrap(); |
|
| 1395 | + | assert!(full.graph_key(&f).is_none(), "nor has a full state"); |
|
| 1396 | + | ||
| 1397 | + | // Through a version 2 checkpoint of c1, keys name its graph blobs |
|
| 1398 | + | // or inline values: stable across loads, changed by later edits. |
|
| 1399 | + | let graphs = full.checkpoint_file_graphs(); |
|
| 1400 | + | let index = full.checkpoint_index(&graphs, |enc| { |
|
| 1401 | + | crate::hash::hash_object(crate::hash::ObjectType::Blob, enc) |
|
| 1402 | + | }); |
|
| 1403 | + | let cp = |extra: &[&Change]| { |
|
| 1404 | + | let mut st = State::from_checkpoint_history(&index).unwrap(); |
|
| 1405 | + | for c in extra { |
|
| 1406 | + | st.add((*c).clone()).unwrap(); |
|
| 1407 | + | } |
|
| 1408 | + | st |
|
| 1409 | + | }; |
|
| 1410 | + | assert_eq!(key(&cp(&[]), &f), key(&cp(&[]), &f)); |
|
| 1411 | + | assert_eq!(key(&cp(&[]), &f), key(&cp(&[&c2]), &f)); |
|
| 1412 | + | assert_ne!(key(&cp(&[]), &g), key(&cp(&[&c2]), &g)); |
|
| 1413 | + | assert_ne!(key(&cp(&[&c2]), &f), key(&cp(&[&c2, &c3]), &f)); |
|
| 1414 | + | ||
| 1415 | + | // Partial rendering: skipped files are listed and empty, the others |
|
| 1416 | + | // render as in a full state. |
|
| 1417 | + | let mut part = load(&[&c1, &c2]); |
|
| 1418 | + | part.load_graphs(&[g], |_| unreachable!()).unwrap(); |
|
| 1419 | + | let snap = part.materialize_except(&[f].into_iter().collect()); |
|
| 1420 | + | assert_eq!(snap.unrendered, ["f".to_string()].into_iter().collect()); |
|
| 1421 | + | let mut whole = State::new(); |
|
| 1422 | + | whole.add(c1.clone()).unwrap(); |
|
| 1423 | + | whole.add(c2.clone()).unwrap(); |
|
| 1424 | + | assert_eq!(snap.files["g"], whole.materialize().files["g"]); |
|
| 1425 | + | assert!( |
|
| 1426 | + | matches!(&snap.files["f"].content, FileContent::Text { bytes, .. } if bytes.is_empty()) |
|
| 1427 | + | ); |
|
| 1428 | + | } |
|
| 1429 | + | ||
| 1234 | 1430 | #[test] |
|
| 1235 | 1431 | fn history_only_states_build_the_same_graphs_on_demand() { |
|
| 1236 | 1432 | let small = NodeId::random(); |
|
| … | 346 unchanged lines |
modified docs/backlog-overnight-2.md +2 −1
| … | 89 unchanged lines |
||
| 90 | 90 | - A fork ("oxycode") only if an extension cannot do what we need; decided with Andy. |
|
| 91 | 91 | - [ ] P5c Import microsoft/vscode's full Git history with `oxy git import` into a local scratch repository (not published): about 130 000 commits as a stress test of import, log, blame and status at scale, recorded in docs/benchmarks.md. |
|
| 92 | 92 | - [x] O10f Load file graphs lazily from a version 2 checkpoint: a state keeps unread graphs by reference and reads one when first needed. |
|
| 93 | - | - [ ] O10g Render only what a snapshot needs: the manifest records, per file, the graph blob it matched, so an unchanged file whose graph is untouched since the checkpoint is neither read nor rendered; cache materialized states. |
|
| 93 | + | - [x] O10g Render only what a snapshot needs, first part: the manifest records, per file, the key of the base graph it matched, so an unchanged file whose graph is untouched is neither read nor rendered; used by status, log, show, tag and bookmark. |
|
| 94 | + | - [ ] O10h The same for commit, diff, amend and the other full snapshots: a lazy base that builds only the graphs the change touches; cache materialized states. |
|
| 94 | 95 | - [x] O5 Concurrency: two processes committing, snapshotting and pushing on one repository; locks hold, nothing is lost. |
|
| 95 | 96 | - [x] O6 Error messages audit: every user-facing error says what to do next; `oxy doctor` checks identity, config, remotes, CA files, hub reachability. |
|
| 96 | 97 | - [x] O7 Shell completions (`oxy completions bash|zsh|fish`) and man pages generated from clap. |
|
| … | 317 unchanged lines |
modified docs/benchmarks.md +34 −0
| … | 158 unchanged lines |
||
| 159 | 159 | the manifest recorded them equal to the base. |
|
| 160 | 160 | - **O10e.** Checkpoints readable per file, so a snapshot loads only the |
|
| 161 | 161 | graphs it needs. This one changes the checkpoint format in spec §25. |
|
| 162 | + | ||
| 163 | + | ## 2026-09-28: O10g, first part: status snapshots skip unchanged files |
|
| 164 | + | ||
| 165 | + | The manifest now records, per text file equal to the base, the key of the |
|
| 166 | + | base graph it matched: the file's node, its checkpoint graph (blob, |
|
| 167 | + | inline value or none) and the set of later changes that edited it |
|
| 168 | + | (spec §12.4). When a file's stat and key both still match, the snapshot |
|
| 169 | + | neither reads nor renders it. |
|
| 170 | + | ||
| 171 | + | `status`, `log`, `show`, `tag` and `bookmark` use a partial snapshot. It |
|
| 172 | + | loads the base for its history and builds only the graphs of files that |
|
| 173 | + | are not known unchanged. `commit`, `diff` and the rest still build the |
|
| 174 | + | whole base. |
|
| 175 | + | ||
| 176 | + | 10 000 files of 40 lines, 22 changes, a version 2 checkpoint after 21 of |
|
| 177 | + | them; `OXY_TIMINGS=1`, clean working copy: |
|
| 178 | + | ||
| 179 | + | | Operation | Before | After | |
|
| 180 | + | |---|---:|---:| |
|
| 181 | + | | status | 0.55 s | 0.22 s | |
|
| 182 | + | | status peak memory | 259 MB | 51 MB | |
|
| 183 | + | | log | 0.7–0.8 s | 0.38 s | |
|
| 184 | + | | diff (full snapshot) | 0.56 s | 0.57 s | |
|
| 185 | + | ||
| 186 | + | | Phase of a clean status snapshot | Before | After | |
|
| 187 | + | |---|---:|---:| |
|
| 188 | + | | load base (whole / history only) | 317 ms | 31 ms | |
|
| 189 | + | | render base | 91 ms | 25 ms | |
|
| 190 | + | | read working copy | 68 ms | 50 ms | |
|
| 191 | + | | diff against base | 41 ms | 28 ms | |
|
| 192 | + | | write manifest | 33 ms | 35 ms | |
|
| 193 | + | ||
| 194 | + | The same for `commit` and `diff` comes next (O10h). There, only the graphs |
|
| 195 | + | the change touches are needed, and materialized states can be cached. |
modified docs/spec.md +3 −3
| … | 1 unchanged lines |
||
| 2 | 2 | ||
| 3 | 3 | | | | |
|
| 4 | 4 | |---|---| |
|
| 5 | - | | **Status** | Draft 0.8j | |
|
| 5 | + | | **Status** | Draft 0.8k | |
|
| 6 | 6 | | **Format version** | 1 | |
|
| 7 | 7 | | **Date** | 2026-09-26 | |
|
| 8 | 8 | | **Audience** | Implementers, security reviewers, and contributors | |
|
| … | 487 unchanged lines |
||
| 496 | 496 | ||
| 497 | 497 | Implementations MUST support a stat-based crawl with a persistent cache keyed by at least `(path, size, mtime)` and SHOULD support a file-system monitor (Watchman or a native inotify/FSEvents/ReadDirectoryChangesW watcher) for the latency targets in §25. |
|
| 498 | 498 | ||
| 499 | - | The reference implementation's cache is the workspace **manifest** written after every snapshot, commit, and checkout: per file its size, mtime, content hash, and, for an opaque file, the ObjectRef of its stored blob. On the next snapshot a file whose size and mtime match is not read: an opaque file contributes its recorded blob, a text file keeps its recorded hash, and when that hash equals the hash of the base state's rendering of the path, its bytes are taken from the base instead of the disk (equal hashes mean equal bytes; a stat-clean file that differs from the base, such as an older uncommitted edit, is always read). A file whose mtime lies within two seconds of the manifest's own write time is never trusted (the racy-stat window), so a modification that preserves size and lands in the same timestamp tick is still seen. An implementation MAY use a file-system monitor instead of, or in addition to, the manifest. |
|
| 499 | + | The reference implementation's cache is the workspace **manifest** written after every snapshot, commit, and checkout: per file its size, mtime, content hash, and, for an opaque file, the ObjectRef of its stored blob. A text file found byte for byte equal to the base state's unconflicted rendering also records its **graph key**: a hash of the file's node, the checkpoint graph its graph was built from (a blob ref, an inline value, or none), and the set of later changes that edited it. Equal keys mean equal graphs and so equal renderings, since a rendering depends only on the graph and the changes' immutable metadata. On the next snapshot a file whose size and mtime match is not read: an opaque file contributes its recorded blob, and a text file whose recorded graph key equals the key of its base graph now is known equal to the base, so it is neither read nor rendered (a stat-clean file that differs from the base, such as an older uncommitted edit, is always read). Attributes decide how a file compares with the base, so `.oxyattributes` files are always read, and when any of them is added, removed, edited, or changed in the base, no text file is skipped. A file whose mtime lies within two seconds of the manifest's own write time is never trusted (the racy-stat window), so a modification that preserves size and lands in the same timestamp tick is still seen. An implementation MAY use a file-system monitor instead of, or in addition to, the manifest. |
|
| 500 | 500 | ||
| 501 | 501 | ## 13. Storage engine |
|
| 502 | 502 | ||
| … | 439 unchanged lines |
||
| 942 | 942 | ||
| 943 | 943 | --- |
|
| 944 | 944 | ||
| 945 | - | *Changelog.* Draft 0.8j (2026-09-28): checkpoint payload version 2, file graphs by reference (§11). |
|
| 945 | + | *Changelog.* Draft 0.8k (2026-09-28): the manifest records graph keys, so unchanged files are neither read nor rendered (§12.4). Draft 0.8j (2026-09-28): checkpoint payload version 2, file graphs by reference (§11). |
|
| 946 | 946 | ||
| 947 | 947 | Draft 0.8i (2026-09-28): portability rules for exec bits and path names (§7.7); found by verifying a real Git import commit by commit: deletions depend on the inserters of the lines right after them, and replacements on the change that set their old blob (§8.1); replace records are superseded causally, so a reverted binary keeps its blob (§9.3); merges resolve opaque conflicts to the commit's blob, and `oxy git verify` (§23). Draft 0.8h (2026-09-28): `oxy fsck` (§21.1); durability rules and the operation head as commit point (§13.6); the operation head moves by compare-and-swap and concurrent operations are merged (§12.1); `oxy doctor` (§21.2). Draft 0.8g (2026-09-27): delegates of a rule's identities may sign reference attestations (§14.5); local-path pushes bind unbound repositories like HTTP ones. Draft 0.8f (2026-09-27): challenge signatures (§14.3) for hosts. Draft 0.8e (2026-09-27): anonymous changes (zero author) are authenticated by a seal from any key, so imported and pre-identity history can be pushed under a policy (§14.4). Draft 0.8d (2026-09-27): unbound hosted repositories adopt the first pusher's id (§16.1). Draft 0.8c (2026-09-27): bare repositories (`oxy init --bare`, §12.2: no workspace, no working copy; hosting) and the reference server as an embeddable library (§15.1). Draft 0.8b (2026-09-27): the machine interface for editors (§24): `root`, `cat`, `version`, `hunks`, `discard`, and `--json` on the reporting commands. |
|
| 948 | 948 | ||
| … | 61 unchanged lines |
modified src/cli/sync.rs +4 −1
| … | 195 unchanged lines |
||
| 196 | 196 | let r = repo.pull(&remote)?; |
|
| 197 | 197 | if as_json { |
|
| 198 | 198 | sp.clear(); |
|
| 199 | - | let conflicted = repo.snapshot("oxy pull")?.base_snapshot.is_conflicted(); |
|
| 199 | + | let conflicted = repo |
|
| 200 | + | .snapshot_status("oxy pull")? |
|
| 201 | + | .base_snapshot |
|
| 202 | + | .is_conflicted(); |
|
| 200 | 203 | println!( |
|
| 201 | 204 | "{}", |
|
| 202 | 205 | json::obj([ |
|
| … | 453 unchanged lines |
modified src/cli/workspace.rs +7 −7
| … | 58 unchanged lines |
||
| 59 | 59 | pub fn status(json: bool) -> Result<i32> { |
|
| 60 | 60 | let repo = open()?; |
|
| 61 | 61 | let sp = ui::spinner("Snapshotting the working copy…"); |
|
| 62 | - | let snap = repo.snapshot("oxy status")?; |
|
| 62 | + | let snap = repo.snapshot_status("oxy status")?; |
|
| 63 | 63 | let heads = repo.bookmark_heads()?; |
|
| 64 | 64 | sp.clear(); |
|
| 65 | 65 | if json { |
|
| … | 410 unchanged lines |
||
| 476 | 476 | ||
| 477 | 477 | pub fn log(limit: Option<usize>, verbose: bool, path: Option<&str>, as_json: bool) -> Result<i32> { |
|
| 478 | 478 | let repo = open()?; |
|
| 479 | - | let snap = repo.snapshot("oxy log")?; |
|
| 479 | + | let snap = repo.snapshot_status("oxy log")?; |
|
| 480 | 480 | let conflicted = snap.base_snapshot.is_conflicted(); |
|
| 481 | 481 | let entries = match path { |
|
| 482 | 482 | Some(p) => repo.log_for_path(p)?, |
|
| … | 290 unchanged lines |
||
| 773 | 773 | ||
| 774 | 774 | pub fn show(change: &str, as_json: bool) -> Result<i32> { |
|
| 775 | 775 | let repo = open()?; |
|
| 776 | - | repo.snapshot("oxy show")?; |
|
| 776 | + | repo.snapshot_status("oxy show")?; |
|
| 777 | 777 | let h = repo.resolve_change(change)?; |
|
| 778 | 778 | let c = repo.store().get_change(&h)?; |
|
| 779 | 779 | if as_json { |
|
| … | 392 unchanged lines |
||
| 1172 | 1172 | /// several heads beneath the working copy, the most recently made one is |
|
| 1173 | 1173 | /// chosen and announced. |
|
| 1174 | 1174 | fn current_head(repo: &Repo) -> Result<oxy_core::hash::ChangeHash> { |
|
| 1175 | - | let snap = repo.snapshot("oxy")?; |
|
| 1175 | + | let snap = repo.snapshot_status("oxy")?; |
|
| 1176 | 1176 | let base = snap.base.maximal(&snap.wc.deps); |
|
| 1177 | 1177 | match base.as_slice() { |
|
| 1178 | 1178 | [] => anyhow::bail!("the working copy sits on no change yet"), |
|
| … | 144 unchanged lines |
||
| 1323 | 1323 | ||
| 1324 | 1324 | pub fn tag_create(name: &str, changes: &[String], message: &str) -> Result<i32> { |
|
| 1325 | 1325 | let repo = open()?; |
|
| 1326 | - | repo.snapshot("oxy tag")?; |
|
| 1326 | + | repo.snapshot_status("oxy tag")?; |
|
| 1327 | 1327 | let heads = if changes.is_empty() { |
|
| 1328 | 1328 | repo.bookmark_heads()? |
|
| 1329 | 1329 | } else { |
|
| … | 190 unchanged lines |
||
| 1520 | 1520 | ||
| 1521 | 1521 | pub fn bookmark_create(name: &str, changes: &[String]) -> Result<i32> { |
|
| 1522 | 1522 | let repo = open()?; |
|
| 1523 | - | repo.snapshot("oxy bookmark create")?; |
|
| 1523 | + | repo.snapshot_status("oxy bookmark create")?; |
|
| 1524 | 1524 | let heads = if changes.is_empty() { |
|
| 1525 | 1525 | None |
|
| 1526 | 1526 | } else { |
|
| … | 17 unchanged lines |
||
| 1544 | 1544 | ||
| 1545 | 1545 | pub fn bookmark_set(name: &str, changes: &[String]) -> Result<i32> { |
|
| 1546 | 1546 | let repo = open()?; |
|
| 1547 | - | repo.snapshot("oxy bookmark set")?; |
|
| 1547 | + | repo.snapshot_status("oxy bookmark set")?; |
|
| 1548 | 1548 | let heads = heads_from(&repo, changes)?; |
|
| 1549 | 1549 | let heads = repo.bookmark_set(name, &heads)?; |
|
| 1550 | 1550 | println!( |
|
| … | 167 unchanged lines |