Checkpoint payload version 2: large file graphs as shared blobs named from an index, read in parallel (loose and packed) and decoded in parallel; version 1 still read; gc, fsck and shallow-clone bundles carry the graph blobs (spec 11, Draft 0.8j) afaba5f1306d
closes #4
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-28 17:07 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
afaba5f1306de61a8f9ea16f5555dd532da1e5b55ddfbd99cdb45dfc28acdea3- Depends on
- 0bdc3f91a39f 252e8cb23fbc 2863456a0f6d 387335ff7f4d 62794ecc7f74 696bbf29d3eb 75167672ebcd 8f10a07a1dbe 982864068a59 b3c5a59a875a ba160f60219e c863c7a7a2d1 cfb7d414eae6 d02e545dc265 e7ddd552b619 ff0af1ee299d
- Depended on by
- 795b8b502ded 4476cdbb0443 f3811eb20bea 767ac6cd0545 092457020692 648189388f1a 9ae4d15a0e4b 01a47cd7f0c8 a9f686edf60b dfb23a3a8cfa cc7a00565161 e8d38e47e9d7 e3d071cecfe2
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 35 atoms, 0 tree operations, 8 files
Browse the tree at this change
modified crates/oxy-core/src/repo/checkpoint.rs +39 −6
| … | 54 unchanged lines |
||
| 55 | 55 | state: &State, |
|
| 56 | 56 | heads: &[ChangeHash], |
|
| 57 | 57 | ) -> Result<(ObjectRef, usize)> { |
|
| 58 | - | let payload = state.checkpoint_payload(); |
|
| 59 | 58 | let batch = self.store.begin_batch()?; |
|
| 60 | - | let blob = self.store_blob(&payload)?; |
|
| 59 | + | let (blob, payload_len) = self.store_checkpoint_payload(state)?; |
|
| 61 | 60 | let cp = Checkpoint { |
|
| 62 | 61 | heads: StateObject::new(heads.to_vec()).heads, |
|
| 63 | 62 | count: state.len() as u64, |
|
| … | 3 unchanged lines |
||
| 67 | 66 | batch.finish()?; |
|
| 68 | 67 | fs::create_dir_all(self.oxy.join("checkpoints"))?; |
|
| 69 | 68 | write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())?; |
|
| 70 | - | Ok((r, payload.len())) |
|
| 69 | + | Ok((r, payload_len)) |
|
| 71 | 70 | } |
|
| 72 | 71 | ||
| 72 | + | /// Store `state` as a version 2 payload (spec §11): the graphs of large |
|
| 73 | + | /// files as blobs of their own, shared with earlier checkpoints when |
|
| 74 | + | /// unchanged, and the index naming them. Returns the index blob and its size. |
|
| 75 | + | pub(crate) fn store_checkpoint_payload(&self, state: &State) -> Result<(ObjectRef, usize)> { |
|
| 76 | + | let graphs = state.checkpoint_file_graphs(); |
|
| 77 | + | for (_, enc) in &graphs { |
|
| 78 | + | if enc.len() >= crate::state::INLINE_GRAPH_LIMIT { |
|
| 79 | + | self.store_blob(enc)?; |
|
| 80 | + | } |
|
| 81 | + | } |
|
| 82 | + | let index = state.checkpoint_index(&graphs, Repo::blob_ref); |
|
| 83 | + | Ok((self.store_blob(&index)?, index.len())) |
|
| 84 | + | } |
|
| 85 | + | ||
| 86 | + | /// The graph blobs a checkpoint's payload names (none for version 1). |
|
| 87 | + | pub(crate) fn checkpoint_graph_blobs(&self, cp: &Checkpoint) -> Result<Vec<ObjectRef>> { |
|
| 88 | + | let payload = self.read_blob(&cp.payload)?; |
|
| 89 | + | Ok(State::checkpoint_graph_refs(&payload)?) |
|
| 90 | + | } |
|
| 91 | + | ||
| 92 | + | /// Decode a checkpoint's payload, reading its graph blobs from the store. |
|
| 93 | + | pub(crate) fn state_from_checkpoint(&self, payload: &[u8]) -> Result<State> { |
|
| 94 | + | Ok(State::from_checkpoint(payload, |refs| { |
|
| 95 | + | self.read_blobs(refs).map_err(|e| { |
|
| 96 | + | crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}")) |
|
| 97 | + | }) |
|
| 98 | + | })?) |
|
| 99 | + | } |
|
| 100 | + | ||
| 73 | 101 | /// Checkpoint the current bookmark's state now. |
|
| 74 | 102 | pub fn checkpoint(&self) -> Result<(ObjectRef, usize, usize)> { |
|
| 75 | 103 | let heads = self.bookmark_heads()?; |
|
| … | 24 unchanged lines |
||
| 100 | 128 | if current == wanted { |
|
| 101 | 129 | self.write_checkpoint(&bookmark, &state, &wanted)?.0 |
|
| 102 | 130 | } else { |
|
| 103 | - | let payload = state.checkpoint_payload(); |
|
| 104 | - | let blob = self.store_blob(&payload)?; |
|
| 131 | + | let (blob, _) = self.store_checkpoint_payload(&state)?; |
|
| 105 | 132 | let cp = Checkpoint { |
|
| 106 | 133 | heads: wanted.clone(), |
|
| 107 | 134 | count: state.len() as u64, |
|
| … | 9 unchanged lines |
||
| 117 | 144 | bundle.objects.push(self.store.get(&c)?); |
|
| 118 | 145 | } |
|
| 119 | 146 | bundle.objects.push(self.store.get(&cp.payload)?); |
|
| 147 | + | for g in self.checkpoint_graph_blobs(&cp)? { |
|
| 148 | + | for c in Blob::decode(&self.store.get_typed(&g, ObjectType::Blob)?)?.chunks { |
|
| 149 | + | bundle.objects.push(self.store.get(&c)?); |
|
| 150 | + | } |
|
| 151 | + | bundle.objects.push(self.store.get(&g)?); |
|
| 152 | + | } |
|
| 120 | 153 | bundle.objects.push((ObjectType::Checkpoint, cp.encode())); |
|
| 121 | 154 | Ok(bundle) |
|
| 122 | 155 | } |
|
| … | 19 unchanged lines |
||
| 142 | 175 | }; |
|
| 143 | 176 | let base = { |
|
| 144 | 177 | let _t = crate::timing::phase("load: decode checkpoint"); |
|
| 145 | - | State::from_checkpoint_payload(&payload)? |
|
| 178 | + | self.state_from_checkpoint(&payload)? |
|
| 146 | 179 | }; |
|
| 147 | 180 | let cp_heads: BTreeSet<ChangeHash> = cp.heads.iter().copied().collect(); |
|
| 148 | 181 | let mut found: BTreeSet<ChangeHash> = BTreeSet::new(); |
|
| … | 31 unchanged lines |
modified crates/oxy-core/src/repo/fsck.rs +17 −3
| … | 392 unchanged lines |
||
| 393 | 393 | continue; |
|
| 394 | 394 | } |
|
| 395 | 395 | }; |
|
| 396 | - | if State::from_checkpoint_payload(&payload).is_err() { |
|
| 397 | - | rep.error(Some(r), "checkpoint payload does not decode"); |
|
| 396 | + | if let Err(e) = self.state_from_checkpoint(&payload) { |
|
| 397 | + | rep.error(Some(r), format!("checkpoint payload does not decode: {e}")); |
|
| 398 | 398 | continue; |
|
| 399 | 399 | } |
|
| 400 | + | // The payload must be what the named changes produce, in its own version. |
|
| 401 | + | let version = crate::cbor::decode(&payload).ok().and_then(|v| { |
|
| 402 | + | crate::objects::map(&v, "payload").ok().and_then(|m| { |
|
| 403 | + | m.get(&0) |
|
| 404 | + | .and_then(|x| crate::objects::uint(x, "version").ok()) |
|
| 405 | + | }) |
|
| 406 | + | }); |
|
| 407 | + | let expected = |full: &State| { |
|
| 408 | + | if version == Some(crate::state::CHECKPOINT_INDEX_VERSION) { |
|
| 409 | + | full.checkpoint_index(&full.checkpoint_file_graphs(), Repo::blob_ref) |
|
| 410 | + | } else { |
|
| 411 | + | full.checkpoint_payload() |
|
| 412 | + | } |
|
| 413 | + | }; |
|
| 400 | 414 | if !shallow { |
|
| 401 | 415 | match self.load_state_full(&cp.heads) { |
|
| 402 | - | Ok(full) if full.checkpoint_payload() != payload => { |
|
| 416 | + | Ok(full) if expected(&full) != payload => { |
|
| 403 | 417 | rep.error(Some(r), "checkpoint disagrees with the changes it names") |
|
| 404 | 418 | } |
|
| 405 | 419 | Ok(_) => {} |
|
| … | 47 unchanged lines |
modified crates/oxy-core/src/repo/gc.rs +7 −0
| … | 70 unchanged lines |
||
| 71 | 71 | if let Ok(b) = self.store.get_typed(&cp.payload, ObjectType::Blob) { |
|
| 72 | 72 | keep.extend(Blob::decode(&b)?.chunks); |
|
| 73 | 73 | } |
|
| 74 | + | // A version 2 payload's graph blobs and their chunks. |
|
| 75 | + | for g in self.checkpoint_graph_blobs(&cp).unwrap_or_default() { |
|
| 76 | + | keep.insert(g); |
|
| 77 | + | if let Ok(b) = self.store.get_typed(&g, ObjectType::Blob) { |
|
| 78 | + | keep.extend(Blob::decode(&b)?.chunks); |
|
| 79 | + | } |
|
| 80 | + | } |
|
| 74 | 81 | } |
|
| 75 | 82 | } |
|
| 76 | 83 | // Tags: their states and attestations. |
|
| … | 71 unchanged lines |
modified crates/oxy-core/src/repo/mod.rs +40 −0
| … | 854 unchanged lines |
||
| 855 | 855 | .put(ObjectType::Blob, &Self::blob_for(content).encode())?) |
|
| 856 | 856 | } |
|
| 857 | 857 | ||
| 858 | + | /// Read many blobs at once (checkpoint graphs): the blob objects, then |
|
| 859 | + | /// all their chunks, are read in parallel; a blob the store lacks goes |
|
| 860 | + | /// through [`Repo::read_blob`], which may fetch it lazily. |
|
| 861 | + | pub fn read_blobs(&self, refs: &[ObjectRef]) -> Result<Vec<Vec<u8>>> { |
|
| 862 | + | let mut blobs: Vec<Option<Blob>> = Vec::with_capacity(refs.len()); |
|
| 863 | + | for (r, got) in refs.iter().zip(self.store.get_many(refs)) { |
|
| 864 | + | blobs.push(match got { |
|
| 865 | + | Ok((ObjectType::Blob, body)) => Some(Blob::decode(&body)?), |
|
| 866 | + | Ok((ty, _)) => return fail(format!("{} is a {ty:?}, not a blob", r.short(12))), |
|
| 867 | + | Err(crate::store::StoreError::NotFound(_)) => None, |
|
| 868 | + | Err(e) => return Err(e.into()), |
|
| 869 | + | }); |
|
| 870 | + | } |
|
| 871 | + | let chunk_refs: Vec<ObjectRef> = blobs |
|
| 872 | + | .iter() |
|
| 873 | + | .flatten() |
|
| 874 | + | .flat_map(|b| b.chunks.iter().copied()) |
|
| 875 | + | .collect(); |
|
| 876 | + | let mut chunks = self.store.get_many(&chunk_refs).into_iter(); |
|
| 877 | + | let mut out = Vec::with_capacity(refs.len()); |
|
| 878 | + | for (r, blob) in refs.iter().zip(blobs) { |
|
| 879 | + | match blob { |
|
| 880 | + | None => out.push(self.read_blob(r)?), |
|
| 881 | + | Some(b) => { |
|
| 882 | + | let mut content = Vec::with_capacity(b.length as usize); |
|
| 883 | + | for c in &b.chunks { |
|
| 884 | + | match chunks.next() { |
|
| 885 | + | Some(Ok((ObjectType::Chunk, body))) => content.extend(body), |
|
| 886 | + | Some(Ok(_)) => return fail(format!("{} is not a chunk", c.short(12))), |
|
| 887 | + | Some(Err(e)) => return Err(e.into()), |
|
| 888 | + | None => return fail("chunk list ended early"), |
|
| 889 | + | } |
|
| 890 | + | } |
|
| 891 | + | out.push(content); |
|
| 892 | + | } |
|
| 893 | + | } |
|
| 894 | + | } |
|
| 895 | + | Ok(out) |
|
| 896 | + | } |
|
| 897 | + | ||
| 858 | 898 | /// Read a blob's content, fetching it from the remote that withheld it |
|
| 859 | 899 | /// when it is lazily absent (spec §13.5). |
|
| 860 | 900 | pub fn read_blob(&self, r: &ObjectRef) -> Result<Vec<u8>> { |
|
| … | 602 unchanged lines |
modified crates/oxy-core/src/repo/tests.rs +87 −0
| … | 4681 unchanged lines |
||
| 4682 | 4682 | fs::set_permissions(root.join("a.txt"), fs::Permissions::from_mode(0o644)).unwrap(); |
|
| 4683 | 4683 | fs::set_permissions(root.join("b.txt"), fs::Permissions::from_mode(0o644)).unwrap(); |
|
| 4684 | 4684 | } |
|
| 4685 | + | ||
| 4686 | + | /// Version 2 checkpoints (spec §11): a large file's graph is a blob of its |
|
| 4687 | + | /// own, kept by gc, checked by fsck, shared by the next checkpoint when the |
|
| 4688 | + | /// file is unchanged, and loading through the checkpoint equals a full load. |
|
| 4689 | + | #[test] |
|
| 4690 | + | fn checkpoints_keep_large_graphs_as_blobs_of_their_own() { |
|
| 4691 | + | hermetic(); |
|
| 4692 | + | let dir = tempfile::tempdir().unwrap(); |
|
| 4693 | + | let root = dir.path(); |
|
| 4694 | + | let repo = Repo::init(root).unwrap().with_home(root.join("home")); |
|
| 4695 | + | let long: String = (0..300) |
|
| 4696 | + | .map(|i| format!("line {i} of a long file\n")) |
|
| 4697 | + | .collect(); |
|
| 4698 | + | write(root, "long.txt", long.as_bytes()); |
|
| 4699 | + | write(root, "short.txt", b"short\n"); |
|
| 4700 | + | repo.commit("base", &[]).unwrap(); |
|
| 4701 | + | let (r1, _, _) = repo.checkpoint().unwrap(); |
|
| 4702 | + | let cp1 = crate::objects::Checkpoint::decode( |
|
| 4703 | + | &repo.store().get_typed(&r1, ObjectType::Checkpoint).unwrap(), |
|
| 4704 | + | ) |
|
| 4705 | + | .unwrap(); |
|
| 4706 | + | let graphs1 = repo.checkpoint_graph_blobs(&cp1).unwrap(); |
|
| 4707 | + | assert_eq!(graphs1.len(), 1, "only the long file's graph is a blob"); |
|
| 4708 | + | ||
| 4709 | + | // gc keeps the graph blob; fsck finds the checkpoint sound. |
|
| 4710 | + | repo.gc(true).unwrap(); |
|
| 4711 | + | assert!(repo.store().has(&graphs1[0])); |
|
| 4712 | + | let report = repo.fsck().unwrap(); |
|
| 4713 | + | assert_eq!(report.errors(), 0, "{:?}", report.problems); |
|
| 4714 | + | ||
| 4715 | + | // Loading through the checkpoint equals a full load. |
|
| 4716 | + | write(root, "short.txt", b"short, edited\n"); |
|
| 4717 | + | repo.commit("edit the short file", &[]).unwrap(); |
|
| 4718 | + | let heads = repo.bookmark_heads().unwrap(); |
|
| 4719 | + | let fast = repo.load_state(&heads).unwrap(); |
|
| 4720 | + | let full = repo.load_state_full(&heads).unwrap(); |
|
| 4721 | + | assert_eq!(fast.checkpoint_payload(), full.checkpoint_payload()); |
|
| 4722 | + | ||
| 4723 | + | // The next checkpoint shares the unchanged long file's graph. |
|
| 4724 | + | let (r2, _, _) = repo.checkpoint().unwrap(); |
|
| 4725 | + | let cp2 = crate::objects::Checkpoint::decode( |
|
| 4726 | + | &repo.store().get_typed(&r2, ObjectType::Checkpoint).unwrap(), |
|
| 4727 | + | ) |
|
| 4728 | + | .unwrap(); |
|
| 4729 | + | assert_eq!(repo.checkpoint_graph_blobs(&cp2).unwrap(), graphs1); |
|
| 4730 | + | ||
| 4731 | + | // A shallow clone's bundle carries the graph blob with the checkpoint. |
|
| 4732 | + | let bundle = repo.checkpoint_bundle_for(&heads).unwrap(); |
|
| 4733 | + | assert!(bundle.objects.iter().any(|(t, body)| *t == ObjectType::Blob |
|
| 4734 | + | && crate::hash::hash_object(ObjectType::Blob, body) == graphs1[0])); |
|
| 4735 | + | ||
| 4736 | + | // A version 1 checkpoint written by an older oxy still loads. |
|
| 4737 | + | let state = repo.load_state_full(&heads).unwrap(); |
|
| 4738 | + | let old = repo.store_blob(&state.checkpoint_payload()).unwrap(); |
|
| 4739 | + | let cp_old = crate::objects::Checkpoint { |
|
| 4740 | + | heads: cp2.heads.clone(), |
|
| 4741 | + | count: cp2.count, |
|
| 4742 | + | payload: old, |
|
| 4743 | + | }; |
|
| 4744 | + | let r_old = repo |
|
| 4745 | + | .store() |
|
| 4746 | + | .put(ObjectType::Checkpoint, &cp_old.encode()) |
|
| 4747 | + | .unwrap(); |
|
| 4748 | + | repo.install_checkpoint("main", &r_old).unwrap(); |
|
| 4749 | + | assert_eq!( |
|
| 4750 | + | repo.load_state(&heads).unwrap().checkpoint_payload(), |
|
| 4751 | + | full.checkpoint_payload() |
|
| 4752 | + | ); |
|
| 4753 | + | assert_eq!(repo.fsck().unwrap().errors(), 0); |
|
| 4754 | + | ||
| 4755 | + | // A lost graph blob fails the load loudly rather than dropping the |
|
| 4756 | + | // file, and fsck reports it (a fresh repository, objects still loose). |
|
| 4757 | + | let dir2 = tempfile::tempdir().unwrap(); |
|
| 4758 | + | let root2 = dir2.path(); |
|
| 4759 | + | let repo2 = Repo::init(root2).unwrap().with_home(root2.join("home")); |
|
| 4760 | + | write(root2, "long.txt", long.as_bytes()); |
|
| 4761 | + | repo2.commit("base", &[]).unwrap(); |
|
| 4762 | + | let (r, _, _) = repo2.checkpoint().unwrap(); |
|
| 4763 | + | let cp = crate::objects::Checkpoint::decode( |
|
| 4764 | + | &repo2.store().get_typed(&r, ObjectType::Checkpoint).unwrap(), |
|
| 4765 | + | ) |
|
| 4766 | + | .unwrap(); |
|
| 4767 | + | let graph = repo2.checkpoint_graph_blobs(&cp).unwrap()[0]; |
|
| 4768 | + | fs::remove_file(repo2.store().path_for(&graph)).unwrap(); |
|
| 4769 | + | assert!(repo2.load_state(&repo2.bookmark_heads().unwrap()).is_err()); |
|
| 4770 | + | assert!(repo2.fsck().unwrap().errors() > 0, "fsck reports it"); |
|
| 4771 | + | } |
modified crates/oxy-core/src/state.rs +265 −30
| … | 12 unchanged lines |
||
| 13 | 13 | use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; |
|
| 14 | 14 | use std::sync::{Arc, Mutex}; |
|
| 15 | 15 | ||
| 16 | + | /// Checkpoint payload version 2 (spec §11): file graphs by reference. |
|
| 17 | + | pub const CHECKPOINT_INDEX_VERSION: u64 = 2; |
|
| 18 | + | ||
| 19 | + | /// A file graph encoded in fewer bytes stays inline in a version 2 index. |
|
| 20 | + | pub const INLINE_GRAPH_LIMIT: usize = 2048; |
|
| 21 | + | ||
| 16 | 22 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] |
|
| 17 | 23 | pub enum StateError { |
|
| 18 | 24 | #[error("change {1} depends on {0}, which is not in the state")] |
|
| … | 194 unchanged lines |
||
| 213 | 219 | ||
| 214 | 220 | // ------------------------------------------------------------ checkpoints (spec §11) |
|
| 215 | 221 | ||
| 216 | - | /// Serialize the whole state: change metadata, tree, file graphs, replaces. |
|
| 222 | + | /// Serialize the whole state in payload version 1: change metadata, |
|
| 223 | + | /// tree, file graphs, replaces, all in one value (spec §11). |
|
| 217 | 224 | pub fn checkpoint_payload(&self) -> Vec<u8> { |
|
| 225 | + | let mut nodes: Vec<&NodeId> = self.files.keys().collect(); |
|
| 226 | + | nodes.sort(); |
|
| 227 | + | let files = Value::Array( |
|
| 228 | + | nodes |
|
| 229 | + | .into_iter() |
|
| 230 | + | .map(|n| Value::Array(vec![Value::Bytes(n.0.to_vec()), self.files[n].to_value()])) |
|
| 231 | + | .collect(), |
|
| 232 | + | ); |
|
| 233 | + | cbor::encode(&self.payload_value(o::FORMAT_VERSION, files)) |
|
| 234 | + | } |
|
| 235 | + | ||
| 236 | + | /// Each file graph encoded on its own, by node, for a version 2 payload |
|
| 237 | + | /// (spec §11): the graphs of large files become blobs of their own. |
|
| 238 | + | pub fn checkpoint_file_graphs(&self) -> Vec<(NodeId, Vec<u8>)> { |
|
| 239 | + | use rayon::prelude::*; |
|
| 240 | + | let mut nodes: Vec<(&NodeId, &FileGraph)> = self.files.iter().collect(); |
|
| 241 | + | nodes.sort_by_key(|(n, _)| **n); |
|
| 242 | + | nodes |
|
| 243 | + | .into_par_iter() |
|
| 244 | + | .map(|(n, g)| (*n, cbor::encode(&g.to_value()))) |
|
| 245 | + | .collect() |
|
| 246 | + | } |
|
| 247 | + | ||
| 248 | + | /// The payload version 2 index for the graphs of |
|
| 249 | + | /// [`State::checkpoint_file_graphs`]: a graph encoded in fewer than |
|
| 250 | + | /// [`INLINE_GRAPH_LIMIT`] bytes is kept inline, a larger one is named by |
|
| 251 | + | /// `blob_of` (the ObjectRef of the blob holding its encoding). |
|
| 252 | + | pub fn checkpoint_index( |
|
| 253 | + | &self, |
|
| 254 | + | graphs: &[(NodeId, Vec<u8>)], |
|
| 255 | + | mut blob_of: impl FnMut(&[u8]) -> ObjectRef, |
|
| 256 | + | ) -> Vec<u8> { |
|
| 257 | + | let files = Value::Array( |
|
| 258 | + | graphs |
|
| 259 | + | .iter() |
|
| 260 | + | .map(|(n, enc)| { |
|
| 261 | + | let entry = if enc.len() < INLINE_GRAPH_LIMIT { |
|
| 262 | + | cbor::decode(enc).expect("an encoded graph decodes") |
|
| 263 | + | } else { |
|
| 264 | + | Value::Bytes(blob_of(enc).0.to_vec()) |
|
| 265 | + | }; |
|
| 266 | + | Value::Array(vec![Value::Bytes(n.0.to_vec()), entry]) |
|
| 267 | + | }) |
|
| 268 | + | .collect(), |
|
| 269 | + | ); |
|
| 270 | + | cbor::encode(&self.payload_value(CHECKPOINT_INDEX_VERSION, files)) |
|
| 271 | + | } |
|
| 272 | + | ||
| 273 | + | /// The payload map around `files`: changes, tree and replaces as in version 1. |
|
| 274 | + | fn payload_value(&self, version: u64, files: Value) -> Value { |
|
| 218 | 275 | let mut hashes: Vec<&ChangeHash> = self.changes.keys().collect(); |
|
| 219 | 276 | hashes.sort(); |
|
| 220 | 277 | let changes = Value::Array( |
|
| … | 12 unchanged lines |
||
| 233 | 290 | Value::Bytes(m.author.0.to_vec()), |
|
| 234 | 291 | ]) |
|
| 235 | 292 | }) |
|
| 236 | - | .collect(), |
|
| 237 | - | ); |
|
| 238 | - | let mut nodes: Vec<&NodeId> = self.files.keys().collect(); |
|
| 239 | - | nodes.sort(); |
|
| 240 | - | let files = Value::Array( |
|
| 241 | - | nodes |
|
| 242 | - | .into_iter() |
|
| 243 | - | .map(|n| Value::Array(vec![Value::Bytes(n.0.to_vec()), self.files[n].to_value()])) |
|
| 244 | 293 | .collect(), |
|
| 245 | 294 | ); |
|
| 246 | 295 | let mut rnodes: Vec<&NodeId> = self.replaces.keys().collect(); |
|
| … | 18 unchanged lines |
||
| 265 | 314 | }) |
|
| 266 | 315 | .collect(), |
|
| 267 | 316 | ); |
|
| 268 | - | cbor::encode(&Value::map([ |
|
| 269 | - | (0, Value::Uint(o::FORMAT_VERSION)), |
|
| 317 | + | Value::map([ |
|
| 318 | + | (0, Value::Uint(version)), |
|
| 270 | 319 | (1, changes), |
|
| 271 | 320 | (2, self.tree.to_value()), |
|
| 272 | 321 | (3, files), |
|
| 273 | 322 | (4, replaces), |
|
| 274 | - | ])) |
|
| 323 | + | ]) |
|
| 275 | 324 | } |
|
| 276 | 325 | ||
| 326 | + | /// Decode a version 1 payload (the whole state in one value). A |
|
| 327 | + | /// version 2 payload needs its graph blobs: use [`State::from_checkpoint`]. |
|
| 277 | 328 | pub fn from_checkpoint_payload(bytes: &[u8]) -> Result<State, ObjectError> { |
|
| 329 | + | State::from_checkpoint(bytes, |_| { |
|
| 330 | + | Err(ObjectError::Schema( |
|
| 331 | + | "a version 2 checkpoint needs its graph blobs".into(), |
|
| 332 | + | )) |
|
| 333 | + | }) |
|
| 334 | + | } |
|
| 335 | + | ||
| 336 | + | /// Decode a checkpoint payload of either version (spec §11). For |
|
| 337 | + | /// version 2, `load` returns the contents of the named graph blobs, in |
|
| 338 | + | /// order; the graphs are then decoded in parallel. |
|
| 339 | + | pub fn from_checkpoint( |
|
| 340 | + | bytes: &[u8], |
|
| 341 | + | load: impl FnOnce(&[ObjectRef]) -> Result<Vec<Vec<u8>>, ObjectError>, |
|
| 342 | + | ) -> Result<State, ObjectError> { |
|
| 343 | + | use rayon::prelude::*; |
|
| 278 | 344 | let v = cbor::decode(bytes)?; |
|
| 279 | 345 | let m = o::map(&v, "checkpoint payload")?; |
|
| 280 | - | o::check_version(m)?; |
|
| 346 | + | let version = o::uint(o::field(m, 0, "version")?, "version")?; |
|
| 347 | + | if version != o::FORMAT_VERSION && version != CHECKPOINT_INDEX_VERSION { |
|
| 348 | + | return Err(ObjectError::Schema(format!( |
|
| 349 | + | "checkpoint payload version {version} is not supported" |
|
| 350 | + | ))); |
|
| 351 | + | } |
|
| 352 | + | let mut st = State::payload_common(m)?; |
|
| 353 | + | let entries = o::array(o::field(m, 3, "files")?, "files")?; |
|
| 354 | + | let mut parsed: Vec<(NodeId, &Value)> = Vec::with_capacity(entries.len()); |
|
| 355 | + | for f in entries { |
|
| 356 | + | let e = o::array(f, "file")?; |
|
| 357 | + | if e.len() != 2 { |
|
| 358 | + | return Err(ObjectError::Schema("file: expected [node, graph]".into())); |
|
| 359 | + | } |
|
| 360 | + | let node = NodeId::from_slice(o::bytes(&e[0], "node")?) |
|
| 361 | + | .ok_or_else(|| ObjectError::Schema("node: expected 16 bytes".into()))?; |
|
| 362 | + | parsed.push((node, &e[1])); |
|
| 363 | + | } |
|
| 364 | + | // Version 2 names large graphs by blob: fetch those in one batch. |
|
| 365 | + | let mut fetched: HashMap<NodeId, Vec<u8>> = HashMap::new(); |
|
| 366 | + | if version == CHECKPOINT_INDEX_VERSION { |
|
| 367 | + | let wanted: Vec<(NodeId, ObjectRef)> = parsed |
|
| 368 | + | .iter() |
|
| 369 | + | .filter(|(_, e)| matches!(e, Value::Bytes(_))) |
|
| 370 | + | .map(|(n, e)| Ok((*n, o::oref(e, "graph blob")?))) |
|
| 371 | + | .collect::<Result<_, ObjectError>>()?; |
|
| 372 | + | if !wanted.is_empty() { |
|
| 373 | + | let refs: Vec<ObjectRef> = wanted.iter().map(|(_, r)| *r).collect(); |
|
| 374 | + | let contents = { |
|
| 375 | + | let _t = crate::timing::phase("load: graph blobs read"); |
|
| 376 | + | load(&refs)? |
|
| 377 | + | }; |
|
| 378 | + | if contents.len() != refs.len() { |
|
| 379 | + | return Err(ObjectError::Schema("graph blobs: wrong count".into())); |
|
| 380 | + | } |
|
| 381 | + | fetched.extend(wanted.iter().map(|(n, _)| *n).zip(contents)); |
|
| 382 | + | } |
|
| 383 | + | } else if parsed.iter().any(|(_, e)| matches!(e, Value::Bytes(_))) { |
|
| 384 | + | return Err(ObjectError::Schema( |
|
| 385 | + | "a version 1 payload holds graphs inline".into(), |
|
| 386 | + | )); |
|
| 387 | + | } |
|
| 388 | + | // File graphs are independent; decode and build them in parallel. |
|
| 389 | + | let graphs = parsed |
|
| 390 | + | .par_iter() |
|
| 391 | + | .map(|(node, entry)| { |
|
| 392 | + | let graph = match fetched.get(node) { |
|
| 393 | + | Some(bytes) => FileGraph::from_value(&cbor::decode(bytes)?)?, |
|
| 394 | + | None => FileGraph::from_value(entry)?, |
|
| 395 | + | }; |
|
| 396 | + | Ok((*node, graph)) |
|
| 397 | + | }) |
|
| 398 | + | .collect::<Result<Vec<_>, ObjectError>>()?; |
|
| 399 | + | st.files.extend(graphs); |
|
| 400 | + | Ok(st) |
|
| 401 | + | } |
|
| 402 | + | ||
| 403 | + | /// The graph blobs a payload names: empty for version 1. |
|
| 404 | + | pub fn checkpoint_graph_refs(bytes: &[u8]) -> Result<Vec<ObjectRef>, ObjectError> { |
|
| 405 | + | let v = cbor::decode(bytes)?; |
|
| 406 | + | let m = o::map(&v, "checkpoint payload")?; |
|
| 407 | + | let mut out = Vec::new(); |
|
| 408 | + | for f in o::array(o::field(m, 3, "files")?, "files")? { |
|
| 409 | + | if let Some(entry @ Value::Bytes(_)) = o::array(f, "file")?.get(1) { |
|
| 410 | + | out.push(o::oref(entry, "graph blob")?); |
|
| 411 | + | } |
|
| 412 | + | } |
|
| 413 | + | Ok(out) |
|
| 414 | + | } |
|
| 415 | + | ||
| 416 | + | /// Changes, tree and replaces of a payload of either version. |
|
| 417 | + | fn payload_common(m: &BTreeMap<u64, Value>) -> Result<State, ObjectError> { |
|
| 281 | 418 | let mut st = State::new(); |
|
| 282 | 419 | for c in o::array(o::field(m, 1, "changes")?, "changes")? { |
|
| 283 | 420 | let e = o::array(c, "change meta")?; |
|
| … | 19 unchanged lines |
||
| 303 | 440 | st.changes.insert(hash, meta); |
|
| 304 | 441 | } |
|
| 305 | 442 | st.tree = TreeGraph::from_value(o::field(m, 2, "tree")?)?; |
|
| 306 | - | // File graphs are independent; build them in parallel. |
|
| 307 | - | { |
|
| 308 | - | use rayon::prelude::*; |
|
| 309 | - | let graphs = o::array(o::field(m, 3, "files")?, "files")? |
|
| 310 | - | .par_iter() |
|
| 311 | - | .map(|f| { |
|
| 312 | - | let e = o::array(f, "file")?; |
|
| 313 | - | if e.len() != 2 { |
|
| 314 | - | return Err(ObjectError::Schema("file: expected [node, graph]".into())); |
|
| 315 | - | } |
|
| 316 | - | let node = NodeId::from_slice(o::bytes(&e[0], "node")?) |
|
| 317 | - | .ok_or_else(|| ObjectError::Schema("node: expected 16 bytes".into()))?; |
|
| 318 | - | Ok((node, FileGraph::from_value(&e[1])?)) |
|
| 319 | - | }) |
|
| 320 | - | .collect::<Result<Vec<_>, ObjectError>>()?; |
|
| 321 | - | st.files.extend(graphs); |
|
| 322 | - | } |
|
| 323 | 443 | let opt = |v: &Value| -> Result<Option<ObjectRef>, ObjectError> { |
|
| 324 | 444 | if v.is_null() { |
|
| 325 | 445 | Ok(None) |
|
| … | 586 unchanged lines |
||
| 912 | 1032 | matches!(&b.files["a"].content, FileContent::Text { bytes, .. } if bytes == b"2\n3\n") |
|
| 913 | 1033 | ); |
|
| 914 | 1034 | assert!(State::from_checkpoint_payload(b"nonsense").is_err()); |
|
| 1035 | + | } |
|
| 1036 | + | ||
| 1037 | + | /// Version 2 payloads (spec §11): small graphs inline, large ones by |
|
| 1038 | + | /// blob; the state read back is the same as from version 1. |
|
| 1039 | + | #[test] |
|
| 1040 | + | fn checkpoint_index_round_trip_matches_version_1() { |
|
| 1041 | + | let mut st = State::new(); |
|
| 1042 | + | let small = NodeId::random(); |
|
| 1043 | + | let big = NodeId::random(); |
|
| 1044 | + | let add = |node: NodeId, name: &str| TreeOp::AddNode { |
|
| 1045 | + | node, |
|
| 1046 | + | kind: NodeKind::File, |
|
| 1047 | + | mode: Mode::TEXT, |
|
| 1048 | + | parent: NodeId::ROOT, |
|
| 1049 | + | name: name.into(), |
|
| 1050 | + | }; |
|
| 1051 | + | let big_lines: Vec<Vec<u8>> = (0..400) |
|
| 1052 | + | .map(|i| format!("line {i} of a long file\n").into_bytes()) |
|
| 1053 | + | .collect(); |
|
| 1054 | + | let h1 = st |
|
| 1055 | + | .add(change( |
|
| 1056 | + | vec![], |
|
| 1057 | + | vec![add(small, "small"), add(big, "big")], |
|
| 1058 | + | vec![ |
|
| 1059 | + | Edit::Insert { |
|
| 1060 | + | node: small, |
|
| 1061 | + | after: AtomRef::Start, |
|
| 1062 | + | before: AtomRef::End, |
|
| 1063 | + | lines: vec![b"x\n".to_vec()], |
|
| 1064 | + | }, |
|
| 1065 | + | Edit::Insert { |
|
| 1066 | + | node: big, |
|
| 1067 | + | after: AtomRef::Start, |
|
| 1068 | + | before: AtomRef::End, |
|
| 1069 | + | lines: big_lines, |
|
| 1070 | + | }, |
|
| 1071 | + | ], |
|
| 1072 | + | )) |
|
| 1073 | + | .unwrap(); |
|
| 1074 | + | st.add(change( |
|
| 1075 | + | vec![h1], |
|
| 1076 | + | vec![], |
|
| 1077 | + | vec![Edit::Delete { |
|
| 1078 | + | node: big, |
|
| 1079 | + | atoms: vec![AtomRange { |
|
| 1080 | + | change: h1, |
|
| 1081 | + | from: 3, |
|
| 1082 | + | to: 7, |
|
| 1083 | + | }], |
|
| 1084 | + | }], |
|
| 1085 | + | )) |
|
| 1086 | + | .unwrap(); |
|
| 1087 | + | ||
| 1088 | + | let graphs = st.checkpoint_file_graphs(); |
|
| 1089 | + | let mut blobs: HashMap<ObjectRef, Vec<u8>> = HashMap::new(); |
|
| 1090 | + | let index = st.checkpoint_index(&graphs, |enc| { |
|
| 1091 | + | let r = crate::hash::hash_object(crate::hash::ObjectType::Blob, enc); |
|
| 1092 | + | blobs.insert(r, enc.to_vec()); |
|
| 1093 | + | r |
|
| 1094 | + | }); |
|
| 1095 | + | assert_eq!(blobs.len(), 1, "only the large graph is a blob of its own"); |
|
| 1096 | + | assert_eq!( |
|
| 1097 | + | State::checkpoint_graph_refs(&index).unwrap(), |
|
| 1098 | + | blobs.keys().copied().collect::<Vec<_>>() |
|
| 1099 | + | ); |
|
| 1100 | + | assert!( |
|
| 1101 | + | State::checkpoint_graph_refs(&st.checkpoint_payload()) |
|
| 1102 | + | .unwrap() |
|
| 1103 | + | .is_empty(), |
|
| 1104 | + | "version 1 names none" |
|
| 1105 | + | ); |
|
| 1106 | + | assert!(index.len() < st.checkpoint_payload().len()); |
|
| 1107 | + | ||
| 1108 | + | let back = State::from_checkpoint(&index, |refs| { |
|
| 1109 | + | refs.iter() |
|
| 1110 | + | .map(|r| { |
|
| 1111 | + | blobs |
|
| 1112 | + | .get(r) |
|
| 1113 | + | .cloned() |
|
| 1114 | + | .ok_or_else(|| ObjectError::Schema("missing".into())) |
|
| 1115 | + | }) |
|
| 1116 | + | .collect() |
|
| 1117 | + | }) |
|
| 1118 | + | .unwrap(); |
|
| 1119 | + | assert_eq!( |
|
| 1120 | + | back.checkpoint_payload(), |
|
| 1121 | + | st.checkpoint_payload(), |
|
| 1122 | + | "the same state as version 1" |
|
| 1123 | + | ); |
|
| 1124 | + | assert_eq!( |
|
| 1125 | + | back.materialize().files["big"].content, |
|
| 1126 | + | st.materialize().files["big"].content |
|
| 1127 | + | ); |
|
| 1128 | + | // Deterministic: the same state gives the same index. |
|
| 1129 | + | assert_eq!( |
|
| 1130 | + | back.checkpoint_index(&back.checkpoint_file_graphs(), |enc| { |
|
| 1131 | + | crate::hash::hash_object(crate::hash::ObjectType::Blob, enc) |
|
| 1132 | + | }), |
|
| 1133 | + | index |
|
| 1134 | + | ); |
|
| 1135 | + | // A missing graph blob is an error, not an empty file. |
|
| 1136 | + | assert!( |
|
| 1137 | + | State::from_checkpoint(&index, |_| Err(ObjectError::Schema("gone".into()))).is_err() |
|
| 1138 | + | ); |
|
| 1139 | + | assert!( |
|
| 1140 | + | State::from_checkpoint_payload(&index).is_err(), |
|
| 1141 | + | "version 2 needs its blobs" |
|
| 1142 | + | ); |
|
| 1143 | + | // Version 1 payloads still read, and unknown versions do not. |
|
| 1144 | + | assert!(State::from_checkpoint(&st.checkpoint_payload(), |_| unreachable!()).is_ok()); |
|
| 1145 | + | let mut v3 = cbor::decode(&index).unwrap(); |
|
| 1146 | + | if let Value::Map(m) = &mut v3 { |
|
| 1147 | + | m.insert(0, Value::Uint(3)); |
|
| 1148 | + | } |
|
| 1149 | + | assert!(State::from_checkpoint(&cbor::encode(&v3), |_| unreachable!()).is_err()); |
|
| 915 | 1150 | } |
|
| 916 | 1151 | ||
| 917 | 1152 | #[test] |
|
| … | 43 unchanged lines |
modified crates/oxy-core/src/store.rs +131 −37
| … | 26 unchanged lines |
||
| 27 | 27 | /// and refusing it keeps a bad index from driving huge allocations. |
|
| 28 | 28 | const MAX_OBJECT_LEN: u32 = 1 << 30; |
|
| 29 | 29 | ||
| 30 | + | /// An object read from the store: its type and body. |
|
| 31 | + | pub type Object = (ObjectType, Vec<u8>); |
|
| 32 | + | ||
| 30 | 33 | #[derive(Clone, Copy, Debug)] |
|
| 31 | 34 | struct PackEntry { |
|
| 32 | 35 | offset: u64, |
|
| … | 749 unchanged lines |
||
| 782 | 785 | ||
| 783 | 786 | /// Read and verify an object, loose, packed, or pending in a batch. |
|
| 784 | 787 | pub fn get(&self, r: &ObjectRef) -> Result<(ObjectType, Vec<u8>)> { |
|
| 785 | - | let path = self.path_for(r); |
|
| 786 | - | let data = match fs::read(&path) { |
|
| 787 | - | Ok(d) => d, |
|
| 788 | - | Err(e) if e.kind() == io::ErrorKind::NotFound => { |
|
| 789 | - | return match self.find_packed(r) { |
|
| 790 | - | Some((i, e)) => self.packs.borrow()[i].read(r, e), |
|
| 791 | - | None => self.pending_get(r).unwrap_or(Err(StoreError::NotFound(*r))), |
|
| 792 | - | }; |
|
| 793 | - | } |
|
| 794 | - | Err(e) => return Err(e.into()), |
|
| 795 | - | }; |
|
| 796 | - | let nul = data |
|
| 788 | + | match read_loose_at(&self.path_for(r), r) { |
|
| 789 | + | Some(found) => found, |
|
| 790 | + | None => match self.find_packed(r) { |
|
| 791 | + | Some((i, e)) => self.packs.borrow()[i].read(r, e), |
|
| 792 | + | None => self.pending_get(r).unwrap_or(Err(StoreError::NotFound(*r))), |
|
| 793 | + | }, |
|
| 794 | + | } |
|
| 795 | + | } |
|
| 796 | + | ||
| 797 | + | /// Many objects at once: loose ones are read and verified in parallel, |
|
| 798 | + | /// the rest (packed, or pending in a batch) one by one. Results are in |
|
| 799 | + | /// the order of `refs`. |
|
| 800 | + | pub fn get_many(&self, refs: &[ObjectRef]) -> Vec<Result<Object>> { |
|
| 801 | + | use rayon::prelude::*; |
|
| 802 | + | let paths: Vec<(ObjectRef, PathBuf)> = |
|
| 803 | + | refs.iter().map(|r| (*r, self.path_for(r))).collect(); |
|
| 804 | + | let mut found: Vec<Option<Result<Object>>> = |
|
| 805 | + | paths.par_iter().map(|(r, p)| read_loose_at(p, r)).collect(); |
|
| 806 | + | // Packed ones: look them up in order, then read them in parallel. |
|
| 807 | + | let packs = self.packs.borrow(); |
|
| 808 | + | let packed: Vec<(usize, usize, PackEntry)> = found |
|
| 797 | 809 | .iter() |
|
| 798 | - | .position(|&b| b == 0) |
|
| 799 | - | .ok_or_else(|| StoreError::Corrupt(*r, "no header".into()))?; |
|
| 800 | - | let head = std::str::from_utf8(&data[..nul]) |
|
| 801 | - | .map_err(|_| StoreError::Corrupt(*r, "header not UTF-8".into()))?; |
|
| 802 | - | let mut parts = head.split(' '); |
|
| 803 | - | let (Some("oxy1"), Some(ty), Some(len), None) = |
|
| 804 | - | (parts.next(), parts.next(), parts.next(), parts.next()) |
|
| 805 | - | else { |
|
| 806 | - | return Err(StoreError::Corrupt(*r, format!("bad header {head:?}"))); |
|
| 807 | - | }; |
|
| 808 | - | let ty = ObjectType::from_name(ty) |
|
| 809 | - | .ok_or_else(|| StoreError::Corrupt(*r, format!("unknown type {ty}")))?; |
|
| 810 | - | let len: usize = len |
|
| 811 | - | .parse() |
|
| 812 | - | .map_err(|_| StoreError::Corrupt(*r, "bad length".into()))?; |
|
| 813 | - | let body = &data[nul + 1..]; |
|
| 814 | - | if body.len() != len { |
|
| 815 | - | return Err(StoreError::Corrupt( |
|
| 816 | - | *r, |
|
| 817 | - | format!("length {} != header {len}", body.len()), |
|
| 818 | - | )); |
|
| 810 | + | .enumerate() |
|
| 811 | + | .filter(|(_, f)| f.is_none()) |
|
| 812 | + | .filter_map(|(i, _)| { |
|
| 813 | + | packs |
|
| 814 | + | .iter() |
|
| 815 | + | .enumerate() |
|
| 816 | + | .find_map(|(p, pack)| pack.find(&refs[i]).map(|e| (i, p, e))) |
|
| 817 | + | }) |
|
| 818 | + | .collect(); |
|
| 819 | + | let indexes: &[PackIndex] = &packs; |
|
| 820 | + | let read: Vec<(usize, Result<Object>)> = packed |
|
| 821 | + | .par_iter() |
|
| 822 | + | .map(|(i, p, e)| (*i, indexes[*p].read(&refs[*i], *e))) |
|
| 823 | + | .collect(); |
|
| 824 | + | drop(packs); |
|
| 825 | + | for (i, r) in read { |
|
| 826 | + | found[i] = Some(r); |
|
| 819 | 827 | } |
|
| 820 | - | if hash_object(ty, body) != *r { |
|
| 821 | - | return Err(StoreError::Corrupt(*r, "hash mismatch".into())); |
|
| 822 | - | } |
|
| 823 | - | Ok((ty, body.to_vec())) |
|
| 828 | + | found |
|
| 829 | + | .into_iter() |
|
| 830 | + | .zip(refs) |
|
| 831 | + | .map(|(f, r)| match f { |
|
| 832 | + | Some(f) => f, |
|
| 833 | + | None => self.get(r), |
|
| 834 | + | }) |
|
| 835 | + | .collect() |
|
| 824 | 836 | } |
|
| 825 | 837 | ||
| 826 | 838 | pub fn get_typed(&self, r: &ObjectRef, want: ObjectType) -> Result<Vec<u8>> { |
|
| … | 126 unchanged lines |
||
| 953 | 965 | out.dedup(); |
|
| 954 | 966 | Ok(out) |
|
| 955 | 967 | } |
|
| 968 | + | } |
|
| 969 | + | ||
| 970 | + | /// A loose object at `path`, verified against `r`; `None` when there is no |
|
| 971 | + | /// such file (it may be packed). |
|
| 972 | + | fn read_loose_at(path: &Path, r: &ObjectRef) -> Option<Result<Object>> { |
|
| 973 | + | let data = match fs::read(path) { |
|
| 974 | + | Ok(d) => d, |
|
| 975 | + | Err(e) if e.kind() == io::ErrorKind::NotFound => return None, |
|
| 976 | + | Err(e) => return Some(Err(e.into())), |
|
| 977 | + | }; |
|
| 978 | + | Some((|| { |
|
| 979 | + | let nul = data |
|
| 980 | + | .iter() |
|
| 981 | + | .position(|&b| b == 0) |
|
| 982 | + | .ok_or_else(|| StoreError::Corrupt(*r, "no header".into()))?; |
|
| 983 | + | let head = std::str::from_utf8(&data[..nul]) |
|
| 984 | + | .map_err(|_| StoreError::Corrupt(*r, "header not UTF-8".into()))?; |
|
| 985 | + | let mut parts = head.split(' '); |
|
| 986 | + | let (Some("oxy1"), Some(ty), Some(len), None) = |
|
| 987 | + | (parts.next(), parts.next(), parts.next(), parts.next()) |
|
| 988 | + | else { |
|
| 989 | + | return Err(StoreError::Corrupt(*r, format!("bad header {head:?}"))); |
|
| 990 | + | }; |
|
| 991 | + | let ty = ObjectType::from_name(ty) |
|
| 992 | + | .ok_or_else(|| StoreError::Corrupt(*r, format!("unknown type {ty}")))?; |
|
| 993 | + | let len: usize = len |
|
| 994 | + | .parse() |
|
| 995 | + | .map_err(|_| StoreError::Corrupt(*r, "bad length".into()))?; |
|
| 996 | + | let body = &data[nul + 1..]; |
|
| 997 | + | if body.len() != len { |
|
| 998 | + | return Err(StoreError::Corrupt( |
|
| 999 | + | *r, |
|
| 1000 | + | format!("length {} != header {len}", body.len()), |
|
| 1001 | + | )); |
|
| 1002 | + | } |
|
| 1003 | + | if hash_object(ty, body) != *r { |
|
| 1004 | + | return Err(StoreError::Corrupt(*r, "hash mismatch".into())); |
|
| 1005 | + | } |
|
| 1006 | + | Ok((ty, body.to_vec())) |
|
| 1007 | + | })()) |
|
| 956 | 1008 | } |
|
| 957 | 1009 | ||
| 958 | 1010 | #[cfg(test)] |
|
| … | 378 unchanged lines |
||
| 1337 | 1389 | .1, |
|
| 1338 | 1390 | b"dropped" |
|
| 1339 | 1391 | ); |
|
| 1392 | + | } |
|
| 1393 | + | ||
| 1394 | + | #[test] |
|
| 1395 | + | fn get_many_reads_loose_and_packed_objects_in_order() { |
|
| 1396 | + | let dir = tempfile::tempdir().unwrap(); |
|
| 1397 | + | let s = Store::open(dir.path().join("objects")).unwrap(); |
|
| 1398 | + | let packed: Vec<ObjectRef> = (0..20) |
|
| 1399 | + | .map(|i| { |
|
| 1400 | + | s.put(ObjectType::Chunk, format!("packed {i}").as_bytes()) |
|
| 1401 | + | .unwrap() |
|
| 1402 | + | }) |
|
| 1403 | + | .collect(); |
|
| 1404 | + | s.pack_loose().unwrap(); |
|
| 1405 | + | let loose: Vec<ObjectRef> = (0..20) |
|
| 1406 | + | .map(|i| { |
|
| 1407 | + | s.put(ObjectType::Chunk, format!("loose {i}").as_bytes()) |
|
| 1408 | + | .unwrap() |
|
| 1409 | + | }) |
|
| 1410 | + | .collect(); |
|
| 1411 | + | let missing = hash_object(ObjectType::Chunk, b"never stored"); |
|
| 1412 | + | let mut refs: Vec<ObjectRef> = Vec::new(); |
|
| 1413 | + | for i in 0..20 { |
|
| 1414 | + | refs.push(loose[i]); |
|
| 1415 | + | refs.push(packed[i]); |
|
| 1416 | + | } |
|
| 1417 | + | refs.push(missing); |
|
| 1418 | + | let got = s.get_many(&refs); |
|
| 1419 | + | assert_eq!(got.len(), refs.len()); |
|
| 1420 | + | for (i, r) in refs.iter().enumerate().take(40) { |
|
| 1421 | + | let want = if i % 2 == 0 { |
|
| 1422 | + | format!("loose {}", i / 2) |
|
| 1423 | + | } else { |
|
| 1424 | + | format!("packed {}", i / 2) |
|
| 1425 | + | }; |
|
| 1426 | + | let (ty, body) = got[i].as_ref().unwrap(); |
|
| 1427 | + | assert_eq!( |
|
| 1428 | + | (*ty, body.as_slice()), |
|
| 1429 | + | (ObjectType::Chunk, want.as_bytes()), |
|
| 1430 | + | "{r:?}" |
|
| 1431 | + | ); |
|
| 1432 | + | } |
|
| 1433 | + | assert!(matches!(got[40], Err(StoreError::NotFound(_)))); |
|
| 1340 | 1434 | } |
|
| 1341 | 1435 | } |
modified docs/spec.md +6 −2
| … | 1 unchanged lines |
||
| 2 | 2 | ||
| 3 | 3 | | | | |
|
| 4 | 4 | |---|---| |
|
| 5 | - | | **Status** | Draft 0.8i | |
|
| 5 | + | | **Status** | Draft 0.8j | |
|
| 6 | 6 | | **Format version** | 1 | |
|
| 7 | 7 | | **Date** | 2026-09-26 | |
|
| 8 | 8 | | **Audience** | Implementers, security reviewers, and contributors | |
|
| … | 443 unchanged lines |
||
| 452 | 452 | - `tree`: the tree graph, `{0: nodes [[node, kind, introducer]], 1: placements [[change, index, node, parent, name]], 2: retired placements [[change, index, [deleters]]], 3: modes [[change, index, node, mode]], 4: retired modes [[change, index, [deleters]]]}`. |
|
| 453 | 453 | - `files`: `[[node, file graph]]` where a file graph is `{0: atoms [[change, index, content or null, [deleters]]], 1: edges [[from, to]]}` with vertices encoded as in §5.5 (`0` start, `1` end, `[hash, index]`). Tombstones keep their content so conflicts can show deleted context. |
|
| 454 | 454 | - `replaces`: `[[node, [[change, old blob or null, new blob or null]]]]`. |
|
| 455 | + | ||
| 456 | + | **Payload version 2** has the same map with `version` 2, differing only in `files`: `[[node, graph or ref]]`, where a file graph whose encoding (as above) is shorter than 2048 bytes appears inline and a longer one is replaced by the ObjectRef (a byte string) of a blob holding exactly that encoding. Graph blobs are content addressed, so a file unchanged between two checkpoints shares its blob, and a reader can load only the graphs it needs. The rule is deterministic, so version 2 checkpoints are as reproducible as version 1. Writers produce version 2; readers MUST accept both. The checkpoint object is unchanged, and whatever carries a checkpoint (garbage-collection roots, the `checkpoint` fetch of §16.1) carries its graph blobs and their chunks with it. A version 2 payload's graph blobs are part of the checkpoint: one missing makes the checkpoint unusable, and a reader reports it rather than dropping the file. |
|
| 455 | 457 | ||
| 456 | 458 | A checkpoint is valid only if it is bit-for-bit reproducible from its heads. Implementations MUST create one at least every `core.checkpoint-interval` changes (default 256) along each bookmark and MAY discard any checkpoint at any time; keeping only the newest per bookmark is enough. To load a state from heads `H`, a client walks from `H` and stops at any change the checkpoint contains; the checkpoint applies iff every one of its heads was reached, since a state is closed downward, and the walk yields exactly the changes to replay on top. Clients fetching a state SHOULD request the nearest checkpoint and replay only the difference. |
|
| 457 | 459 | ||
| … | 481 unchanged lines |
||
| 939 | 941 | ||
| 940 | 942 | --- |
|
| 941 | 943 | ||
| 942 | - | *Changelog.* Draft 0.8i (2026-09-28): portability rules for exec bits and path names (§7.7); found by verifying a real Git import commit by commit: deletions depend on the inserters of the lines right after them, and replacements on the change that set their old blob (§8.1); replace records are superseded causally, so a reverted binary keeps its blob (§9.3); merges resolve opaque conflicts to the commit's blob, and `oxy git verify` (§23). Draft 0.8h (2026-09-28): `oxy fsck` (§21.1); durability rules and the operation head as commit point (§13.6); the operation head moves by compare-and-swap and concurrent operations are merged (§12.1); `oxy doctor` (§21.2). Draft 0.8g (2026-09-27): delegates of a rule's identities may sign reference attestations (§14.5); local-path pushes bind unbound repositories like HTTP ones. Draft 0.8f (2026-09-27): challenge signatures (§14.3) for hosts. Draft 0.8e (2026-09-27): anonymous changes (zero author) are authenticated by a seal from any key, so imported and pre-identity history can be pushed under a policy (§14.4). Draft 0.8d (2026-09-27): unbound hosted repositories adopt the first pusher's id (§16.1). Draft 0.8c (2026-09-27): bare repositories (`oxy init --bare`, §12.2: no workspace, no working copy; hosting) and the reference server as an embeddable library (§15.1). Draft 0.8b (2026-09-27): the machine interface for editors (§24): `root`, `cat`, `version`, `hunks`, `discard`, and `--json` on the reporting commands. |
|
| 944 | + | *Changelog.* Draft 0.8j (2026-09-28): checkpoint payload version 2, file graphs by reference (§11). |
|
| 945 | + | ||
| 946 | + | Draft 0.8i (2026-09-28): portability rules for exec bits and path names (§7.7); found by verifying a real Git import commit by commit: deletions depend on the inserters of the lines right after them, and replacements on the change that set their old blob (§8.1); replace records are superseded causally, so a reverted binary keeps its blob (§9.3); merges resolve opaque conflicts to the commit's blob, and `oxy git verify` (§23). Draft 0.8h (2026-09-28): `oxy fsck` (§21.1); durability rules and the operation head as commit point (§13.6); the operation head moves by compare-and-swap and concurrent operations are merged (§12.1); `oxy doctor` (§21.2). Draft 0.8g (2026-09-27): delegates of a rule's identities may sign reference attestations (§14.5); local-path pushes bind unbound repositories like HTTP ones. Draft 0.8f (2026-09-27): challenge signatures (§14.3) for hosts. Draft 0.8e (2026-09-27): anonymous changes (zero author) are authenticated by a seal from any key, so imported and pre-identity history can be pushed under a policy (§14.4). Draft 0.8d (2026-09-27): unbound hosted repositories adopt the first pusher's id (§16.1). Draft 0.8c (2026-09-27): bare repositories (`oxy init --bare`, §12.2: no workspace, no working copy; hosting) and the reference server as an embeddable library (§15.1). Draft 0.8b (2026-09-27): the machine interface for editors (§24): `root`, `cat`, `version`, `hunks`, `discard`, and `--json` on the reporting commands. |
|
| 943 | 947 | ||
| 944 | 948 | Draft 0.8a (2026-09-27): every Draft 0.8 addition is implemented in the reference implementation (§26.1). Clarifications from doing so: an unregistered nested repository is left alone and a tracked subrepo whose directory is absent keeps its pin (§7.8); Oxygen pins are omitted from Git export (§23). |
|
| 945 | 949 | ||
| … | 59 unchanged lines |