Status after a large commit without reading or holding its text 86e1824f2d0b
A clean status of the Linux tree after its first commit was killed at 14 GB. The profile showed a history-only load holding every inserted line (3.2 GB after 17 s) and the working copy read in full (79 s), because the manifest had no graph keys for the files the commit added. A history-only state now reads a stored insert's text blob only when it builds that file's graph, and loads add changes under the hash they were read by instead of hashing them again. A commit records the new base's graph keys in the manifest, so the next status neither reads nor renders an unchanged file. On the Linux tip a clean status now takes 3.7 s and 768 MB. refs #35
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-29 21:50 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
86e1824f2d0bd71debc5cd65f69265496b0592615a18db90e7b4bdc4bc1a0598- Depends on
- 0bdc3f91a39f 135212de5370 1d360c91a289 2951f8023f40 6f8aeb996558 75167672ebcd 7d9dab131937 957fee30826e 982864068a59 b7f102fcbb17 cc7a00565161 dfb23a3a8cfa e8d38e47e9d7 ff0af1ee299d
- Depended on by
- c2e1d09047db 648189388f1a 236f35c0baba 0389c3e82f6c e226a6cc2f71 1ec3f008e96a 77445673174b 3604179679eb 98a1620a3840 14edbaaf5291 f9466021286a 3f1b8fe25ec9 1966f5db15ee d0990c2bff74 f9575918477f e83354cdac22 8457a6cc6984 d0d52107b074 f6451fbbae3f ab4d4e31b38d 0f93f49ce059 57d8dc05c9d4 01a47cd7f0c8 69816d688300 a4fad5b085c7 c5ce0367eddf da868d5d2646
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 35 atoms, 0 tree operations, 8 files
Browse the tree at this change
modified crates/oxy-core/src/objects.rs +5 −3
| … | 539 unchanged lines |
||
| 540 | 540 | if version >= 2 && cbor::byte_strings_len(&lines) >= TEXT_BLOB_MIN { |
|
| 541 | 541 | return schema("an insert this large is stored as a text blob in a version 2 change"); |
|
| 542 | 542 | } |
|
| 543 | + | if lines.is_empty() { |
|
| 544 | + | return schema("insert with no lines"); |
|
| 545 | + | } |
|
| 543 | 546 | lines |
|
| 544 | 547 | } |
|
| 545 | 548 | (None, Some(text)) if version >= 2 => { |
|
| … | 11 unchanged lines |
||
| 557 | 560 | ) |
|
| 558 | 561 | } |
|
| 559 | 562 | }; |
|
| 560 | - | if lines.is_empty() { |
|
| 561 | - | return schema("insert with no lines"); |
|
| 562 | - | } |
|
| 563 | 563 | Edit::Insert { |
|
| 564 | 564 | node, |
|
| 565 | 565 | after: AtomRef::from_value(field(m, 2, "after")?, "after")?, |
|
| … | 208 unchanged lines |
||
| 774 | 774 | ||
| 775 | 775 | /// Reads a stored insert's lines, given the text blob and the line count |
|
| 776 | 776 | /// the insert claims; [`text_lines`] reads and checks a blob's content. |
|
| 777 | + | /// A resolver may also return no lines, leaving a placeholder for a reader |
|
| 778 | + | /// that fills it in later (history-only states, spec §11). |
|
| 777 | 779 | pub type Resolver<'a> = dyn FnMut(&ObjectRef, u32) -> Result<Vec<Vec<u8>>, ObjectError> + 'a; |
|
| 778 | 780 | ||
| 779 | 781 | /// A large insert's lines as stored (spec §6.1): the blob and its content, |
|
| … | 448 unchanged lines |
modified crates/oxy-core/src/repo/mod.rs +9 −2
| … | 846 unchanged lines |
||
| 847 | 847 | } |
|
| 848 | 848 | ||
| 849 | 849 | fn load_state_replayed(&self, heads: &[ChangeHash], mut state: State) -> Result<State> { |
|
| 850 | + | let history = state.is_history_only(); |
|
| 850 | 851 | let mut stack: Vec<ChangeHash> = heads.to_vec(); |
|
| 851 | 852 | let mut seen: BTreeSet<ChangeHash> = BTreeSet::new(); |
|
| 852 | 853 | { |
|
| … | 4 unchanged lines |
||
| 857 | 858 | continue; |
|
| 858 | 859 | } |
|
| 859 | 860 | let t0 = std::time::Instant::now(); |
|
| 860 | - | let c = self.store.get_change(&h)?; |
|
| 861 | + | // A history-only state reads text only for the graphs it |
|
| 862 | + | // builds; either way the hash is the one asked for. |
|
| 863 | + | let (c, texts) = if history { |
|
| 864 | + | self.store.get_change_lazy(&h)? |
|
| 865 | + | } else { |
|
| 866 | + | (self.store.get_change(&h)?, Vec::new()) |
|
| 867 | + | }; |
|
| 861 | 868 | read += t0.elapsed(); |
|
| 862 | 869 | stack.extend(c.deps.iter().copied()); |
|
| 863 | 870 | let t1 = std::time::Instant::now(); |
|
| 864 | - | state.add_unchecked(c); |
|
| 871 | + | state.add_known(h, c, texts); |
|
| 865 | 872 | add += t1.elapsed(); |
|
| 866 | 873 | } |
|
| 867 | 874 | crate::timing::record("load: of which reading", read); |
|
| … | 780 unchanged lines |
modified crates/oxy-core/src/repo/prompt.rs +20 −5
| … | 257 unchanged lines |
||
| 258 | 258 | write_atomic(&p, text.as_bytes()) |
|
| 259 | 259 | } |
|
| 260 | 260 | ||
| 261 | - | /// Change only the header flags of the current manifest (after a commit |
|
| 262 | - | /// the files are as the snapshot left them). |
|
| 263 | - | pub(crate) fn rewrite_manifest_flags(&self, conflicted: bool, uncommitted: bool) -> Result<()> { |
|
| 261 | + | /// Change the header flags, and set the graph key of each path in |
|
| 262 | + | /// `keys` (after a commit, the new base's keys of the files it holds as |
|
| 263 | + | /// they are on disk). |
|
| 264 | + | pub(crate) fn rewrite_manifest( |
|
| 265 | + | &self, |
|
| 266 | + | conflicted: bool, |
|
| 267 | + | uncommitted: bool, |
|
| 268 | + | keys: &HashMap<String, String>, |
|
| 269 | + | ) -> Result<()> { |
|
| 264 | 270 | let p = self.manifest_path(); |
|
| 265 | 271 | let text = match fs::read_to_string(&p) { |
|
| 266 | 272 | Ok(t) => t, |
|
| … | 6 unchanged lines |
||
| 273 | 279 | } else if line.starts_with("# uncommitted=") { |
|
| 274 | 280 | out.push_str(&format!("# uncommitted={}\n", uncommitted as u8)); |
|
| 275 | 281 | } else { |
|
| 276 | - | out.push_str(line); |
|
| 277 | - | out.push('\n'); |
|
| 282 | + | let f: Vec<&str> = line.splitn(6, '\t').collect(); |
|
| 283 | + | match (f.len() == 6).then(|| keys.get(f[5])).flatten() { |
|
| 284 | + | Some(k) => out.push_str(&format!( |
|
| 285 | + | "{}\t{}\t{}\t{}\t{k}\t{}\n", |
|
| 286 | + | f[0], f[1], f[2], f[3], f[5] |
|
| 287 | + | )), |
|
| 288 | + | None => { |
|
| 289 | + | out.push_str(line); |
|
| 290 | + | out.push('\n'); |
|
| 291 | + | } |
|
| 292 | + | } |
|
| 278 | 293 | } |
|
| 279 | 294 | } |
|
| 280 | 295 | write_atomic(&p, out.as_bytes()) |
|
| … | 203 unchanged lines |
modified crates/oxy-core/src/repo/snapshot.rs +32 −4
| … | 536 unchanged lines |
||
| 537 | 537 | /// stays on disk as uncommitted work: the next snapshot rebuilds it |
|
| 538 | 538 | /// against the new base, so the working copy always describes the |
|
| 539 | 539 | /// directory. |
|
| 540 | + | /// The graph key of each text file of the state at `heads`, by path, |
|
| 541 | + | /// from a history-only load that builds no graph. Files with an `eol` |
|
| 542 | + | /// attribute are left out: their bytes on disk are not the rendering. |
|
| 543 | + | fn committed_graph_keys(&self, heads: &[ChangeHash]) -> Result<HashMap<String, String>> { |
|
| 544 | + | let _t = crate::timing::phase("commit: graph keys"); |
|
| 545 | + | let st = self.load_state_history(heads)?; |
|
| 546 | + | let attrs = self.disk_attributes(); |
|
| 547 | + | Ok(st |
|
| 548 | + | .tree() |
|
| 549 | + | .view() |
|
| 550 | + | .entries |
|
| 551 | + | .iter() |
|
| 552 | + | .filter(|(p, e)| { |
|
| 553 | + | e.kind == NodeKind::File && !e.mode.opaque && attrs.lookup(p).eol.is_none() |
|
| 554 | + | }) |
|
| 555 | + | .filter_map(|(p, e)| { |
|
| 556 | + | let k = st.graph_key(&e.node)?; |
|
| 557 | + | Some((p.clone(), crate::hash::ObjectRef(k).to_hex())) |
|
| 558 | + | }) |
|
| 559 | + | .collect()) |
|
| 560 | + | } |
|
| 561 | + | ||
| 540 | 562 | pub fn commit_selection( |
|
| 541 | 563 | &self, |
|
| 542 | 564 | message: &str, |
|
| … | 170 unchanged lines |
||
| 713 | 735 | // The rest is rebuilt from disk by the next snapshot. |
|
| 714 | 736 | self.drop_manifest(); |
|
| 715 | 737 | } else { |
|
| 716 | - | self.rewrite_manifest_flags( |
|
| 717 | - | snap.base_snapshot.is_conflicted() && !snap.resolves, |
|
| 718 | - | false, |
|
| 719 | - | )?; |
|
| 738 | + | let conflicted = snap.base_snapshot.is_conflicted() && !snap.resolves; |
|
| 739 | + | // The files on disk are now exactly the new base's renderings: |
|
| 740 | + | // record their graph keys, so the next snapshot neither reads nor |
|
| 741 | + | // renders them (oxygen#35). |
|
| 742 | + | let keys = if conflicted { |
|
| 743 | + | HashMap::new() |
|
| 744 | + | } else { |
|
| 745 | + | self.committed_graph_keys(&wc.deps).unwrap_or_default() |
|
| 746 | + | }; |
|
| 747 | + | self.rewrite_manifest(conflicted, false, &keys)?; |
|
| 720 | 748 | } |
|
| 721 | 749 | self.maybe_autopack()?; |
|
| 722 | 750 | let _ = self.run_hook( |
|
| … | 1590 unchanged lines |
modified crates/oxy-core/src/repo/tests.rs +95 −0
| … | 5709 unchanged lines |
||
| 5710 | 5710 | .unwrap(); |
|
| 5711 | 5711 | assert_eq!(main.behind, Some(0), "{main:?}"); |
|
| 5712 | 5712 | } |
|
| 5713 | + | ||
| 5714 | + | /// oxygen#35: a history-only state keeps no inserted text. It reads a |
|
| 5715 | + | /// stored insert's text blob only when it builds that file's graph, and |
|
| 5716 | + | /// builds the same graph a full load does. |
|
| 5717 | + | #[test] |
|
| 5718 | + | fn history_only_states_read_text_only_for_the_graphs_they_build() { |
|
| 5719 | + | hermetic(); |
|
| 5720 | + | let dir = tempfile::tempdir().unwrap(); |
|
| 5721 | + | let root = dir.path(); |
|
| 5722 | + | let repo = Repo::init(root).unwrap(); |
|
| 5723 | + | let big: String = (0..3000).map(|i| format!("big line {i}\n")).collect(); |
|
| 5724 | + | let other: String = (0..3000).map(|i| format!("other line {i}\n")).collect(); |
|
| 5725 | + | write(root, "big.txt", big.as_bytes()); |
|
| 5726 | + | write(root, "other.txt", other.as_bytes()); |
|
| 5727 | + | let h = repo.commit("two large files", &[]).unwrap(); |
|
| 5728 | + | write(root, "big.txt", format!("{big}one more\n").as_bytes()); |
|
| 5729 | + | let h2 = repo.commit("append", &[]).unwrap(); |
|
| 5730 | + | ||
| 5731 | + | let full = repo.load_state_full(&[h2]).unwrap().materialize(); |
|
| 5732 | + | let text = |snap: &crate::state::Snapshot, p: &str| match &snap.files[p].content { |
|
| 5733 | + | FileContent::Text { bytes, .. } => bytes.clone(), |
|
| 5734 | + | other => panic!("{other:?}"), |
|
| 5735 | + | }; |
|
| 5736 | + | // Built on demand, every graph renders as the full load's. |
|
| 5737 | + | let mut hist = repo.load_state_history(&[h2]).unwrap(); |
|
| 5738 | + | let big_node = full.files["big.txt"].node; |
|
| 5739 | + | assert!(hist.graph_key(&big_node).is_some(), "not built yet"); |
|
| 5740 | + | repo.load_all_graphs(&mut hist).unwrap(); |
|
| 5741 | + | let rendered = hist.materialize(); |
|
| 5742 | + | for p in ["big.txt", "other.txt"] { |
|
| 5743 | + | assert_eq!(text(&rendered, p), text(&full, p), "{p}"); |
|
| 5744 | + | } |
|
| 5745 | + | ||
| 5746 | + | // Without other.txt's text blob, history still loads and big.txt still |
|
| 5747 | + | // builds: nothing read the other text. |
|
| 5748 | + | let (_, texts) = repo.store().get_change_lazy(&h).unwrap(); |
|
| 5749 | + | assert_eq!(texts.len(), 2); |
|
| 5750 | + | let content = |r: &ObjectRef| -> Vec<u8> { |
|
| 5751 | + | let blob = crate::objects::Blob::decode(&repo.store().get(r).unwrap().1).unwrap(); |
|
| 5752 | + | blob.chunks |
|
| 5753 | + | .iter() |
|
| 5754 | + | .flat_map(|c| repo.store().get(c).unwrap().1) |
|
| 5755 | + | .collect() |
|
| 5756 | + | }; |
|
| 5757 | + | let other_text = texts |
|
| 5758 | + | .iter() |
|
| 5759 | + | .map(|(r, _)| *r) |
|
| 5760 | + | .find(|r| content(r).windows(12).any(|w| w == b"other line 1")) |
|
| 5761 | + | .unwrap(); |
|
| 5762 | + | fs::remove_file(repo.store().path_for(&other_text)).unwrap(); |
|
| 5763 | + | let mut hist = repo.load_state_history(&[h2]).unwrap(); |
|
| 5764 | + | repo.load_graphs_of(&mut hist, &[big_node]).unwrap(); |
|
| 5765 | + | assert!(repo |
|
| 5766 | + | .load_graphs_of(&mut hist, &[full.files["other.txt"].node]) |
|
| 5767 | + | .is_err()); |
|
| 5768 | + | } |
|
| 5769 | + | ||
| 5770 | + | /// oxygen#35: the first status after a commit read and rendered every |
|
| 5771 | + | /// file (79 s and more than 14 GB on the Linux tree), because the |
|
| 5772 | + | /// manifest had no graph keys for files the commit added. |
|
| 5773 | + | #[test] |
|
| 5774 | + | fn a_commit_leaves_a_manifest_the_next_status_trusts() { |
|
| 5775 | + | use std::os::unix::fs::PermissionsExt; |
|
| 5776 | + | hermetic(); |
|
| 5777 | + | let dir = tempfile::tempdir().unwrap(); |
|
| 5778 | + | let root = dir.path(); |
|
| 5779 | + | let repo = Repo::init(root).unwrap(); |
|
| 5780 | + | let files = ["a.txt", "b.txt", "d/c.txt"]; |
|
| 5781 | + | for p in files { |
|
| 5782 | + | write(root, p, format!("{p}\n").as_bytes()); |
|
| 5783 | + | let old = std::time::SystemTime::now() - std::time::Duration::from_secs(30); |
|
| 5784 | + | fs::File::options() |
|
| 5785 | + | .write(true) |
|
| 5786 | + | .open(root.join(p)) |
|
| 5787 | + | .unwrap() |
|
| 5788 | + | .set_modified(old) |
|
| 5789 | + | .unwrap(); |
|
| 5790 | + | } |
|
| 5791 | + | repo.commit("added", &[]).unwrap(); |
|
| 5792 | + | let (_, _, m) = repo.read_manifest().unwrap().unwrap(); |
|
| 5793 | + | assert!( |
|
| 5794 | + | m.entries.values().all(|e| e.graph.is_some()), |
|
| 5795 | + | "keys recorded at commit" |
|
| 5796 | + | ); |
|
| 5797 | + | // Unreadable files prove the next status reads none of them. |
|
| 5798 | + | for p in files { |
|
| 5799 | + | fs::set_permissions(root.join(p), fs::Permissions::from_mode(0o000)).unwrap(); |
|
| 5800 | + | } |
|
| 5801 | + | let snap = repo.snapshot_status("status").unwrap(); |
|
| 5802 | + | for p in files { |
|
| 5803 | + | fs::set_permissions(root.join(p), fs::Permissions::from_mode(0o644)).unwrap(); |
|
| 5804 | + | } |
|
| 5805 | + | assert!(!snap.changed && snap.statuses.is_empty()); |
|
| 5806 | + | assert_eq!(snap.base_snapshot.unrendered.len(), files.len()); |
|
| 5807 | + | } |
modified crates/oxy-core/src/state.rs +85 −13
| … | 70 unchanged lines |
||
| 71 | 71 | deferred: Option<HashMap<NodeId, Deferred>>, |
|
| 72 | 72 | } |
|
| 73 | 73 | ||
| 74 | + | /// A stored insert's text blob and line count. |
|
| 75 | + | type TextRef = (ObjectRef, u32); |
|
| 76 | + | ||
| 74 | 77 | /// A file graph a history-only state has not built: its base from a |
|
| 75 | 78 | /// version 2 checkpoint, if any, and the edits of later changes in the |
|
| 76 | 79 | /// order they were applied (spec §11). Building it later gives the same |
|
| … | 2 unchanged lines |
||
| 79 | 82 | struct Deferred { |
|
| 80 | 83 | base: Option<DeferredBase>, |
|
| 81 | 84 | edits: Vec<(ChangeHash, Vec<Edit>)>, |
|
| 85 | + | /// The text blob and line count of each insert in `edits` left without |
|
| 86 | + | /// its lines, in order: read when the graph is built. |
|
| 87 | + | texts: Vec<(ObjectRef, u32)>, |
|
| 82 | 88 | } |
|
| 83 | 89 | ||
| 84 | 90 | #[derive(Clone)] |
|
| … | 153 unchanged lines |
||
| 238 | 244 | } |
|
| 239 | 245 | } |
|
| 240 | 246 | ||
| 247 | + | /// Whether this state builds file graphs only when asked |
|
| 248 | + | /// ([`State::new_history_only`]), even if it has none waiting. |
|
| 249 | + | pub fn is_history_only(&self) -> bool { |
|
| 250 | + | self.deferred.is_some() |
|
| 251 | + | } |
|
| 252 | + | ||
| 241 | 253 | /// Whether this state holds every file graph. |
|
| 242 | 254 | pub fn is_full(&self) -> bool { |
|
| 243 | 255 | self.deferred.as_ref().is_none_or(|d| d.is_empty()) |
|
| … | 35 unchanged lines |
||
| 279 | 291 | .iter() |
|
| 280 | 292 | .filter_map(|n| deferred.remove(n).map(|d| (*n, d))) |
|
| 281 | 293 | .collect(); |
|
| 282 | - | let refs: Vec<ObjectRef> = taken |
|
| 294 | + | let mut refs: Vec<ObjectRef> = taken |
|
| 283 | 295 | .iter() |
|
| 284 | 296 | .filter_map(|(_, d)| match &d.base { |
|
| 285 | 297 | Some(DeferredBase::Blob(r)) => Some(*r), |
|
| 286 | 298 | _ => None, |
|
| 287 | 299 | }) |
|
| 288 | 300 | .collect(); |
|
| 301 | + | // Text blobs of inserts read without their lines, after the bases. |
|
| 302 | + | let bases = refs.len(); |
|
| 303 | + | refs.extend( |
|
| 304 | + | taken |
|
| 305 | + | .iter() |
|
| 306 | + | .flat_map(|(_, d)| d.texts.iter().map(|(r, _)| *r)), |
|
| 307 | + | ); |
|
| 289 | 308 | let mut blobs = if refs.is_empty() { |
|
| 290 | 309 | Vec::new() |
|
| 291 | 310 | } else { |
|
| 292 | 311 | load(&refs)? |
|
| 312 | + | }; |
|
| 313 | + | if blobs.len() != refs.len() { |
|
| 314 | + | return Err(ObjectError::Schema("graph blobs: wrong count".into())); |
|
| 293 | 315 | } |
|
| 294 | - | .into_iter(); |
|
| 316 | + | let mut texts = blobs.split_off(bases).into_iter(); |
|
| 317 | + | let mut blobs = blobs.into_iter(); |
|
| 295 | 318 | // Pair each graph with its blob, then build them all in parallel: |
|
| 296 | 319 | // graphs are independent. |
|
| 297 | 320 | let mut work: Vec<(NodeId, Deferred, Option<Vec<u8>>)> = Vec::with_capacity(taken.len()); |
|
| 298 | - | for (node, d) in taken.drain(..) { |
|
| 321 | + | for (node, mut d) in taken.drain(..) { |
|
| 322 | + | // Fill in the lines of inserts read without them, in order. |
|
| 323 | + | let mut want = std::mem::take(&mut d.texts).into_iter(); |
|
| 324 | + | for (_, edits) in d.edits.iter_mut() { |
|
| 325 | + | for e in edits.iter_mut() { |
|
| 326 | + | if let Edit::Insert { lines, .. } = e { |
|
| 327 | + | if lines.is_empty() { |
|
| 328 | + | let (_, count) = want.next().ok_or_else(|| { |
|
| 329 | + | ObjectError::Schema("an insert's text is missing".into()) |
|
| 330 | + | })?; |
|
| 331 | + | let content = texts.next().ok_or_else(|| { |
|
| 332 | + | ObjectError::Schema("text blobs: wrong count".into()) |
|
| 333 | + | })?; |
|
| 334 | + | *lines = crate::objects::text_lines(&content, count)?; |
|
| 335 | + | } |
|
| 336 | + | } |
|
| 337 | + | } |
|
| 338 | + | } |
|
| 299 | 339 | let bytes = match &d.base { |
|
| 300 | 340 | Some(DeferredBase::Blob(_)) => Some( |
|
| 301 | 341 | blobs |
|
| … | 115 unchanged lines |
||
| 417 | 457 | /// Add a change in any order; call `validate` once all are in. |
|
| 418 | 458 | pub fn add_unchecked(&mut self, change: Change) -> ChangeHash { |
|
| 419 | 459 | let h = change.hash(); |
|
| 460 | + | self.add_known(h, change, Vec::new()); |
|
| 461 | + | h |
|
| 462 | + | } |
|
| 463 | + | ||
| 464 | + | /// Add a change read from the store under hash `h`, without hashing it |
|
| 465 | + | /// again (for a version 2 change that means chunking all its text). |
|
| 466 | + | /// `texts` names, in order, the inserts left without their lines (see |
|
| 467 | + | /// [`crate::store::Store::get_change_lazy`]); only a history-only state |
|
| 468 | + | /// takes them, and only for graphs it has not built. |
|
| 469 | + | pub fn add_known(&mut self, h: ChangeHash, change: Change, texts: Vec<(ObjectRef, u32)>) { |
|
| 420 | 470 | if self.changes.contains_key(&h) { |
|
| 421 | - | return h; |
|
| 471 | + | return; |
|
| 422 | 472 | } |
|
| 423 | 473 | let meta = ChangeMeta::of(&change); |
|
| 424 | - | self.apply(h, change); |
|
| 474 | + | self.apply_with(h, change, texts); |
|
| 425 | 475 | self.changes.insert(h, meta); |
|
| 426 | 476 | self.closure.lock().unwrap().clear(); |
|
| 427 | - | h |
|
| 428 | 477 | } |
|
| 429 | 478 | ||
| 430 | 479 | // ------------------------------------------------------------ checkpoints (spec §11) |
|
| … | 225 unchanged lines |
||
| 656 | 705 | Deferred { |
|
| 657 | 706 | base: Some(base), |
|
| 658 | 707 | edits: Vec::new(), |
|
| 708 | + | texts: Vec::new(), |
|
| 659 | 709 | }, |
|
| 660 | 710 | ); |
|
| 661 | 711 | } |
|
| … | 68 unchanged lines |
||
| 730 | 780 | Ok(()) |
|
| 731 | 781 | } |
|
| 732 | 782 | ||
| 733 | - | fn apply(&mut self, h: ChangeHash, c: Change) { |
|
| 783 | + | fn apply_with(&mut self, h: ChangeHash, c: Change, texts: Vec<(ObjectRef, u32)>) { |
|
| 734 | 784 | for (i, op) in c.tree_ops.iter().enumerate() { |
|
| 735 | 785 | self.tree.apply(h, i as u32, &resolve_tree_op(op, h)); |
|
| 736 | 786 | } |
|
| 737 | 787 | if let Some(deferred) = self.deferred.as_mut() { |
|
| 738 | - | // History only: edits of files not built wait, grouped per file. |
|
| 739 | - | let mut per_node: HashMap<NodeId, Vec<Edit>> = HashMap::new(); |
|
| 788 | + | // History only: edits of files not built wait, grouped per file, |
|
| 789 | + | // with the texts of inserts read without their lines. |
|
| 790 | + | let mut texts = texts.into_iter(); |
|
| 791 | + | let mut per_node: HashMap<NodeId, (Vec<Edit>, Vec<TextRef>)> = HashMap::new(); |
|
| 740 | 792 | for e in c.edits { |
|
| 741 | 793 | match e { |
|
| 742 | 794 | Edit::Replace { node, old, new } => { |
|
| 743 | 795 | self.replaces.entry(node).or_default().push((h, old, new)) |
|
| 744 | 796 | } |
|
| 745 | - | e => per_node.entry(e.node()).or_default().push(e), |
|
| 797 | + | e => { |
|
| 798 | + | let slot = per_node.entry(e.node()).or_default(); |
|
| 799 | + | if matches!(&e, Edit::Insert { lines, .. } if lines.is_empty()) { |
|
| 800 | + | slot.1.extend(texts.next()); |
|
| 801 | + | } |
|
| 802 | + | slot.0.push(e); |
|
| 803 | + | } |
|
| 746 | 804 | } |
|
| 747 | 805 | } |
|
| 748 | - | for (node, edits) in per_node { |
|
| 806 | + | for (node, (edits, node_texts)) in per_node { |
|
| 749 | 807 | match self.files.get_mut(&node) { |
|
| 750 | - | Some(g) => apply_file_edits(Arc::make_mut(g), h, edits), |
|
| 751 | - | None => deferred.entry(node).or_default().edits.push((h, edits)), |
|
| 808 | + | Some(g) => { |
|
| 809 | + | assert!( |
|
| 810 | + | node_texts.is_empty(), |
|
| 811 | + | "a change read without its text applied to a built graph" |
|
| 812 | + | ); |
|
| 813 | + | apply_file_edits(Arc::make_mut(g), h, edits) |
|
| 814 | + | } |
|
| 815 | + | None => { |
|
| 816 | + | let d = deferred.entry(node).or_default(); |
|
| 817 | + | d.edits.push((h, edits)); |
|
| 818 | + | d.texts.extend(node_texts); |
|
| 819 | + | } |
|
| 752 | 820 | } |
|
| 753 | 821 | } |
|
| 754 | 822 | return; |
|
| 755 | 823 | } |
|
| 824 | + | assert!( |
|
| 825 | + | texts.is_empty(), |
|
| 826 | + | "a change read without its text applied to a full state" |
|
| 827 | + | ); |
|
| 756 | 828 | let mut counters: HashMap<NodeId, u32> = HashMap::new(); |
|
| 757 | 829 | for e in c.edits { |
|
| 758 | 830 | match e { |
|
| … | 1027 unchanged lines |
modified crates/oxy-core/src/store.rs +15 −0
| … | 928 unchanged lines |
||
| 929 | 929 | } |
|
| 930 | 930 | } |
|
| 931 | 931 | ||
| 932 | + | /// A change whose stored inserts are left without their lines (empty |
|
| 933 | + | /// placeholders), with each one's text blob and line count in order: |
|
| 934 | + | /// for history-only states, which read text only for the graphs they |
|
| 935 | + | /// build. |
|
| 936 | + | pub fn get_change_lazy(&self, r: &ObjectRef) -> Result<(Change, Vec<(ObjectRef, u32)>)> { |
|
| 937 | + | let bytes = self.get_typed(r, ObjectType::Change)?; |
|
| 938 | + | let mut texts = Vec::new(); |
|
| 939 | + | let c = Change::decode_with(&bytes, &mut |text, count| { |
|
| 940 | + | texts.push((*text, count)); |
|
| 941 | + | Ok(Vec::new()) |
|
| 942 | + | }) |
|
| 943 | + | .map_err(|e| StoreError::Corrupt(*r, e.to_string()))?; |
|
| 944 | + | Ok((c, texts)) |
|
| 945 | + | } |
|
| 946 | + | ||
| 932 | 947 | fn read_text(&self, r: &ObjectRef) -> Result<(Blob, Vec<u8>)> { |
|
| 933 | 948 | let blob = Blob::decode(&self.get_typed(r, ObjectType::Blob)?)?; |
|
| 934 | 949 | let mut content = Vec::with_capacity(blob.length as usize); |
|
| … | 751 unchanged lines |
modified docs/benchmarks.md +28 −0
| … | 273 unchanged lines |
||
| 274 | 274 | once in the change and again in the file graph, and `status` builds |
|
| 275 | 275 | every graph (L1b). |
|
| 276 | 276 | ||
| 277 | + | ### Status after the commit (2026-09-29, oxygen#35) |
|
| 278 | + | ||
| 279 | + | A profile with the counting allocator (`--features alloc-stats`, |
|
| 280 | + | `OXY_TIMINGS=1`) of the first clean `status`, before it was killed: |
|
| 281 | + | ||
| 282 | + | | Phase | Time | Live after | Allocated | |
|
| 283 | + | |---|---:|---:|---:| |
|
| 284 | + | | Load base history | 17.5 s | 3.2 GB | 16 GB in 96 M allocations | |
|
| 285 | + | | Read working copy | 79.5 s | 4.9 GB | 1.8 GB | |
|
| 286 | + | | Load and render base | killed at 14 GB | | | |
|
| 287 | + | ||
| 288 | + | It had two causes, and both are fixed: |
|
| 289 | + | - **History held every line.** A history-only state now reads a stored |
|
| 290 | + | insert's text blob only when it builds that file's graph, and a load no |
|
| 291 | + | longer hashes each change again, which for format 2 means chunking all |
|
| 292 | + | its text. |
|
| 293 | + | - **The manifest had no graph keys.** They are now recorded at commit, so |
|
| 294 | + | the next status neither reads nor renders an unchanged file. |
|
| 295 | + | ||
| 296 | + | | Clean `status` on the Linux tip | Time | Peak RSS | |
|
| 297 | + | |---|---:|---:| |
|
| 298 | + | | Before | killed at 14 GB | | |
|
| 299 | + | | After | 3.7 s | 768 MB | |
|
| 300 | + | | Git | 0.3 s | 31 MB | |
|
| 301 | + | ||
| 302 | + | Still to do: status to the 0.4 s and 300 MB target; `fsck` is killed at |
|
| 303 | + | 14 GB; the first commit takes 57 s and 10.2 GB. |
|
| 304 | + |