checkpoint.rs 92.4%

159 of 172 lines run, 18 of 30 functions · run #257 on main · run missed

//! Checkpoints (spec §11): a serialized state kept per bookmark so loading
//! a state costs the changes since the checkpoint rather than all of them.
//!
//! The newest checkpoint of each bookmark is pointed at by
//! `.oxy/checkpoints/<bookmark>`; older ones become unreachable and are
//! garbage collected. A checkpoint applies to a requested head set when all
//! of its heads lie in that set's closure, which the loader establishes while
//! walking from the requested heads and stopping at checkpoint members.

use super::*;
use crate::objects::Checkpoint;

impl Repo {
    fn checkpoint_pointer(&self, bookmark: &str) -> PathBuf {
        self.oxy.join("checkpoints").join(name_file(bookmark))
    }

    /// The newest checkpoint recorded for a bookmark.
    pub fn latest_checkpoint(&self, bookmark: &str) -> Result<Option<(ObjectRef, Checkpoint)>> {
        match fs::read_to_string(self.checkpoint_pointer(bookmark)) {
            Ok(h) => {
                let r = ObjectRef::from_hex(h.trim())
                    .ok_or_else(|| RepoError::Msg("corrupt checkpoint pointer".into()))?;
                match self.store.get_typed(&r, ObjectType::Checkpoint) {
                    Ok(bytes) => Ok(Some((r, Checkpoint::decode(&bytes)?))),
                    Err(StoreError::NotFound(_)) => Ok(None),
                    Err(e) => Err(e.into()),
                }
            }
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
            Err(e) => Err(e.into()),
        }
    }

    /// Every checkpoint pointer, for garbage collection roots.
    pub(crate) fn checkpoint_refs(&self) -> Result<Vec<ObjectRef>> {
        let dir = self.oxy.join("checkpoints");
        let mut out = Vec::new();
        if let Ok(rd) = fs::read_dir(&dir) {
            for e in rd.flatten() {
                if let Ok(h) = fs::read_to_string(e.path()) {
                    if let Some(r) = ObjectRef::from_hex(h.trim()) {
                        out.push(r);
                    }
                }
            }
        }
        Ok(out)
    }

    /// Write a checkpoint of `state` (whose heads are `heads`) for `bookmark`.
    pub(crate) fn write_checkpoint(
        &self,
        bookmark: &str,
        state: &State,
        heads: &[ChangeHash],
    ) -> Result<(ObjectRef, usize)> {
        let batch = self.store.begin_batch()?;
        let (blob, payload_len) = self.store_checkpoint_payload(state)?;
        let cp = Checkpoint {
            heads: StateObject::new(heads.to_vec()).heads,
            count: state.len() as u64,
            payload: blob,
        };
        let r = self.store.put(ObjectType::Checkpoint, &cp.encode())?;
        batch.finish()?;
        fs::create_dir_all(self.oxy.join("checkpoints"))?;
        write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())?;
        Ok((r, payload_len))
    }

    /// Store `state` as a version 2 payload (spec §11): the graphs of large
    /// files as blobs of their own, shared with earlier checkpoints when
    /// unchanged, and the index naming them. Returns the index blob and its size.
    pub(crate) fn store_checkpoint_payload(&self, state: &State) -> Result<(ObjectRef, usize)> {
        let graphs = state.checkpoint_file_graphs();
        for (_, enc) in &graphs {
            if enc.len() >= crate::state::INLINE_GRAPH_LIMIT {
                self.store_blob(enc)?;
            }
        }
        let index = state.checkpoint_index(&graphs, Repo::blob_ref);
        Ok((self.store_blob(&index)?, index.len()))
    }

    /// The graph blobs a checkpoint's payload names (none for version 1).
    pub(crate) fn checkpoint_graph_blobs(&self, cp: &Checkpoint) -> Result<Vec<ObjectRef>> {
        let payload = self.read_blob(&cp.payload)?;
        Ok(State::checkpoint_graph_refs(&payload)?)
    }

    /// Decode a checkpoint's payload, reading its graph blobs from the store.
    pub(crate) fn state_from_checkpoint(&self, payload: &[u8]) -> Result<State> {
        Ok(State::from_checkpoint(payload, |refs| {
            self.read_blobs(refs).map_err(|e| {
                crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}"))
            })
        })?)
    }

    /// Checkpoint the current bookmark's state now.
    pub fn checkpoint(&self) -> Result<(ObjectRef, usize, usize)> {
        let heads = self.bookmark_heads()?;
        let state = self.load_state_full(&heads)?;
        let (r, bytes) = self.write_checkpoint(&self.config.bookmark, &state, &heads)?;
        Ok((r, state.len(), bytes))
    }

    /// Point `bookmark` at an already stored checkpoint (a shallow clone's base).
    pub(crate) fn install_checkpoint(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
        Checkpoint::decode(&self.store.get_typed(r, ObjectType::Checkpoint)?)?;
        fs::create_dir_all(self.oxy.join("checkpoints"))?;
        write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())
    }

    /// A bundle holding a checkpoint of exactly `heads` with its payload
    /// (spec §16.1 `checkpoint`); the current bookmark checkpoint is reused
    /// when it matches, otherwise one is made and, when `heads` are the
    /// bookmark's, kept.
    pub fn checkpoint_bundle_for(&self, heads: &[ChangeHash]) -> Result<Bundle> {
        let wanted = StateObject::new(heads.to_vec()).heads;
        let bookmark = self.config.bookmark.clone();
        let current = self.bookmark_heads()?;
        let r = match self.latest_checkpoint(&bookmark)? {
            Some((r, cp)) if cp.heads == wanted => r,
            _ => {
                let state = self.load_state(&wanted)?;
                if current == wanted {
                    self.write_checkpoint(&bookmark, &state, &wanted)?.0
                } else {
                    let (blob, _) = self.store_checkpoint_payload(&state)?;
                    let cp = Checkpoint {
                        heads: wanted.clone(),
                        count: state.len() as u64,
                        payload: blob,
                    };
                    self.store.put(ObjectType::Checkpoint, &cp.encode())?
                }
            }
        };
        let cp = Checkpoint::decode(&self.store.get_typed(&r, ObjectType::Checkpoint)?)?;
        let mut bundle = Bundle::default();
        for c in Blob::decode(&self.store.get_typed(&cp.payload, ObjectType::Blob)?)?.chunks {
            bundle.objects.push(self.store.get(&c)?);
        }
        bundle.objects.push(self.store.get(&cp.payload)?);
        for g in self.checkpoint_graph_blobs(&cp)? {
            for c in Blob::decode(&self.store.get_typed(&g, ObjectType::Blob)?)?.chunks {
                bundle.objects.push(self.store.get(&c)?);
            }
            bundle.objects.push(self.store.get(&g)?);
        }
        bundle.objects.push((ObjectType::Checkpoint, cp.encode()));
        Ok(bundle)
    }

    /// Load the state of `heads` using a checkpoint when one applies.
    pub(crate) fn load_state_checkpointed(&self, heads: &[ChangeHash]) -> Result<Option<State>> {
        self.load_state_checkpointed_as(heads, false)
    }

    /// As [`Repo::load_state_checkpointed`]; `history` builds no file graph.
    pub(crate) fn load_state_checkpointed_as(
        &self,
        heads: &[ChangeHash],
        history: bool,
    ) -> Result<Option<State>> {
        let mut bookmarks: Vec<String> = vec![self.config.bookmark.clone()];
        if let Ok(rd) = fs::read_dir(self.oxy.join("checkpoints")) {
            for e in rd.flatten() {
                let name = file_name_to_name(&e.file_name().to_string_lossy());
                if !bookmarks.contains(&name) {
                    bookmarks.push(name);
                }
            }
        }
        for b in bookmarks {
            let Some((_, cp)) = self.latest_checkpoint(&b)? else {
                continue;
            };
            let payload = {
                let _t = crate::timing::phase("load: read checkpoint");
                self.read_blob(&cp.payload)?
            };
            let base = {
                let _t = crate::timing::phase("load: decode checkpoint");
                if history {
                    State::from_checkpoint_history(&payload)?
                } else {
                    self.state_from_checkpoint(&payload)?
                }
            };
            let cp_heads: BTreeSet<ChangeHash> = cp.heads.iter().copied().collect();
            let mut found: BTreeSet<ChangeHash> = BTreeSet::new();
            let mut delta: Vec<(ChangeHash, Change, Vec<crate::objects::TextRef>)> = Vec::new();
            let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
            let mut stack: Vec<ChangeHash> = heads.to_vec();
            while let Some(h) = stack.pop() {
                if !seen.insert(h) {
                    continue;
                }
                if base.contains(&h) {
                    if cp_heads.contains(&h) {
                        found.insert(h);
                    }
                    continue;
                }
                // For history, the skeleton: text is read only for the
                // graphs built later, and a state holding none can be
                // indexed (oxygen#93).
                let (c, texts) = if history {
                    self.store.get_change_lazy(&h)?
                } else {
                    (self.store.get_change(&h)?, Vec::new())
                };
                stack.extend(c.deps.iter().copied());
                delta.push((h, c, texts));
            }
            if found.len() != cp_heads.len() {
                continue;
            }
            let _t = crate::timing::phase("load: apply delta and validate");
            let mut state = base;
            for (h, c, texts) in delta {
                if history {
                    state.add_known(h, c, texts);
                } else {
                    state.add_unchecked(c);
                }
            }
            state.validate()?;
            return Ok(Some(state));
        }
        Ok(None)
    }
}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1		//! Checkpoints (spec §11): a serialized state kept per bookmark so loading
2		//! a state costs the changes since the checkpoint rather than all of them.
3		//!
4		//! The newest checkpoint of each bookmark is pointed at by
5		//! `.oxy/checkpoints/<bookmark>`; older ones become unreachable and are
6		//! garbage collected. A checkpoint applies to a requested head set when all
7		//! of its heads lie in that set's closure, which the loader establishes while
8		//! walking from the requested heads and stopping at checkpoint members.
9
10		use super::*;
11		use crate::objects::Checkpoint;
12
13		impl Repo {
14	2418×	fn checkpoint_pointer(&self, bookmark: &str) -> PathBuf {
15	2418×	self.oxy.join("checkpoints").join(name_file(bookmark))
16	2418×	}
17
18		/// The newest checkpoint recorded for a bookmark.
19	2407×	pub fn latest_checkpoint(&self, bookmark: &str) -> Result<Option<(ObjectRef, Checkpoint)>> {
20	2407×	match fs::read_to_string(self.checkpoint_pointer(bookmark)) {
21	59×	Ok(h) => {
22	59×	let r = ObjectRef::from_hex(h.trim())
23	59×	.ok_or_else(\|\| RepoError::Msg("corrupt checkpoint pointer".into()))?;
24	59×	match self.store.get_typed(&r, ObjectType::Checkpoint) {
25	59×	Ok(bytes) => Ok(Some((r, Checkpoint::decode(&bytes)?))),
26	0×	Err(StoreError::NotFound(_)) => Ok(None),
27	0×	Err(e) => Err(e.into()),
28		}
29		}
30	2348×	Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
31	0×	Err(e) => Err(e.into()),
32		}
33	2407×	}
34
35		/// Every checkpoint pointer, for garbage collection roots.
36	87×	pub(crate) fn checkpoint_refs(&self) -> Result<Vec<ObjectRef>> {
37	87×	let dir = self.oxy.join("checkpoints");
38	87×	let mut out = Vec::new();
39	87×	if let Ok(rd) = fs::read_dir(&dir) {
40	11×	for e in rd.flatten() {
41	11×	if let Ok(h) = fs::read_to_string(e.path()) {
42	11×	if let Some(r) = ObjectRef::from_hex(h.trim()) {
43	11×	out.push(r);
44	11×	}
45	0×	}
46		}
47	76×	}
48	87×	Ok(out)
49	87×	}
50
51		/// Write a checkpoint of `state` (whose heads are `heads`) for `bookmark`.
52	9×	pub(crate) fn write_checkpoint(
53	9×	&self,
54	9×	bookmark: &str,
55	9×	state: &State,
56	9×	heads: &[ChangeHash],
57	9×	) -> Result<(ObjectRef, usize)> {
58	9×	let batch = self.store.begin_batch()?;
59	9×	let (blob, payload_len) = self.store_checkpoint_payload(state)?;
60	9×	let cp = Checkpoint {
61	9×	heads: StateObject::new(heads.to_vec()).heads,
62	9×	count: state.len() as u64,
63	9×	payload: blob,
64	9×	};
65	9×	let r = self.store.put(ObjectType::Checkpoint, &cp.encode())?;
66	9×	batch.finish()?;
67	9×	fs::create_dir_all(self.oxy.join("checkpoints"))?;
68	9×	write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())?;
69	9×	Ok((r, payload_len))
70	9×	}
71
72		/// Store `state` as a version 2 payload (spec §11): the graphs of large
73		/// files as blobs of their own, shared with earlier checkpoints when
74		/// unchanged, and the index naming them. Returns the index blob and its size.
75	9×	pub(crate) fn store_checkpoint_payload(&self, state: &State) -> Result<(ObjectRef, usize)> {
76	9×	let graphs = state.checkpoint_file_graphs();
77	39×	for (_, enc) in &graphs {
78	39×	if enc.len() >= crate::state::INLINE_GRAPH_LIMIT {
79	3×	self.store_blob(enc)?;
80	36×	}
81		}
82	9×	let index = state.checkpoint_index(&graphs, Repo::blob_ref);
83	9×	Ok((self.store_blob(&index)?, index.len()))
84	9×	}
85
86		/// The graph blobs a checkpoint's payload names (none for version 1).
87	7×	pub(crate) fn checkpoint_graph_blobs(&self, cp: &Checkpoint) -> Result<Vec<ObjectRef>> {
88	7×	let payload = self.read_blob(&cp.payload)?;
89	7×	Ok(State::checkpoint_graph_refs(&payload)?)
90	7×	}
91
92		/// Decode a checkpoint's payload, reading its graph blobs from the store.
93	19×	pub(crate) fn state_from_checkpoint(&self, payload: &[u8]) -> Result<State> {
94	19×	Ok(State::from_checkpoint(payload, \|refs\| {
95	4×	self.read_blobs(refs).map_err(\|e\| {
96	2×	crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}"))
97	2×	})
98	4×	})?)
99	19×	}
100
101		/// Checkpoint the current bookmark's state now.
102	6×	pub fn checkpoint(&self) -> Result<(ObjectRef, usize, usize)> {
103	6×	let heads = self.bookmark_heads()?;
104	6×	let state = self.load_state_full(&heads)?;
105	6×	let (r, bytes) = self.write_checkpoint(&self.config.bookmark, &state, &heads)?;
106	6×	Ok((r, state.len(), bytes))
107	6×	}
108
109		/// Point `bookmark` at an already stored checkpoint (a shallow clone's base).
110	2×	pub(crate) fn install_checkpoint(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
111	2×	Checkpoint::decode(&self.store.get_typed(r, ObjectType::Checkpoint)?)?;
112	2×	fs::create_dir_all(self.oxy.join("checkpoints"))?;
113	2×	write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())
114	2×	}
115
116		/// A bundle holding a checkpoint of exactly `heads` with its payload
117		/// (spec §16.1 `checkpoint`); the current bookmark checkpoint is reused
118		/// when it matches, otherwise one is made and, when `heads` are the
119		/// bookmark's, kept.
120	2×	pub fn checkpoint_bundle_for(&self, heads: &[ChangeHash]) -> Result<Bundle> {
121	2×	let wanted = StateObject::new(heads.to_vec()).heads;
122	2×	let bookmark = self.config.bookmark.clone();
123	2×	let current = self.bookmark_heads()?;
124	2×	let r = match self.latest_checkpoint(&bookmark)? {
125	1×	Some((r, cp)) if cp.heads == wanted => r,
126		_ => {
127	1×	let state = self.load_state(&wanted)?;
128	1×	if current == wanted {
129	1×	self.write_checkpoint(&bookmark, &state, &wanted)?.0
130		} else {
131	0×	let (blob, _) = self.store_checkpoint_payload(&state)?;
132	0×	let cp = Checkpoint {
133	0×	heads: wanted.clone(),
134	0×	count: state.len() as u64,
135	0×	payload: blob,
136	0×	};
137	0×	self.store.put(ObjectType::Checkpoint, &cp.encode())?
138		}
139		}
140		};
141	2×	let cp = Checkpoint::decode(&self.store.get_typed(&r, ObjectType::Checkpoint)?)?;
142	2×	let mut bundle = Bundle::default();
143	2×	for c in Blob::decode(&self.store.get_typed(&cp.payload, ObjectType::Blob)?)?.chunks {
144	2×	bundle.objects.push(self.store.get(&c)?);
145		}
146	2×	bundle.objects.push(self.store.get(&cp.payload)?);
147	2×	for g in self.checkpoint_graph_blobs(&cp)? {
148	1×	for c in Blob::decode(&self.store.get_typed(&g, ObjectType::Blob)?)?.chunks {
149	1×	bundle.objects.push(self.store.get(&c)?);
150		}
151	1×	bundle.objects.push(self.store.get(&g)?);
152		}
153	2×	bundle.objects.push((ObjectType::Checkpoint, cp.encode()));
154	2×	Ok(bundle)
155	2×	}
156
157		/// Load the state of `heads` using a checkpoint when one applies.
158	194×	pub(crate) fn load_state_checkpointed(&self, heads: &[ChangeHash]) -> Result<Option<State>> {
159	194×	self.load_state_checkpointed_as(heads, false)
160	194×	}
161
162		/// As [`Repo::load_state_checkpointed`]; `history` builds no file graph.
163	1778×	pub(crate) fn load_state_checkpointed_as(
164	1778×	&self,
165	1778×	heads: &[ChangeHash],
166	1778×	history: bool,
167	1778×	) -> Result<Option<State>> {
168	1778×	let mut bookmarks: Vec<String> = vec![self.config.bookmark.clone()];
169	1778×	if let Ok(rd) = fs::read_dir(self.oxy.join("checkpoints")) {
170	43×	for e in rd.flatten() {
171	43×	let name = file_name_to_name(&e.file_name().to_string_lossy());
172	43×	if !bookmarks.contains(&name) {
173	0×	bookmarks.push(name);
174	43×	}
175		}
176	1735×	}
177	1778×	for b in bookmarks {
178	1778×	let Some((_, cp)) = self.latest_checkpoint(&b)? else {
179	1735×	continue;
180		};
181	43×	let payload = {
182	43×	let _t = crate::timing::phase("load: read checkpoint");
183	43×	self.read_blob(&cp.payload)?
184		};
185	42×	let base = {
186	43×	let _t = crate::timing::phase("load: decode checkpoint");
187	43×	if history {
188	33×	State::from_checkpoint_history(&payload)?
189		} else {
190	10×	self.state_from_checkpoint(&payload)?
191		}
192		};
193	42×	let cp_heads: BTreeSet<ChangeHash> = cp.heads.iter().copied().collect();
194	42×	let mut found: BTreeSet<ChangeHash> = BTreeSet::new();
195	42×	let mut delta: Vec<(ChangeHash, Change, Vec<crate::objects::TextRef>)> = Vec::new();
196	42×	let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
197	42×	let mut stack: Vec<ChangeHash> = heads.to_vec();
198	185×	while let Some(h) = stack.pop() {
199	143×	if !seen.insert(h) {
200	21×	continue;
201	122×	}
202	122×	if base.contains(&h) {
203	80×	if cp_heads.contains(&h) {
204	39×	found.insert(h);
205	41×	}
206	80×	continue;
207	42×	}
208		// For history, the skeleton: text is read only for the
209		// graphs built later, and a state holding none can be
210		// indexed (oxygen#93).
211	42×	let (c, texts) = if history {
212	35×	self.store.get_change_lazy(&h)?
213		} else {
214	7×	(self.store.get_change(&h)?, Vec::new())
215		};
216	42×	stack.extend(c.deps.iter().copied());
217	42×	delta.push((h, c, texts));
218		}
219	42×	if found.len() != cp_heads.len() {
220	4×	continue;
221	38×	}
222	38×	let _t = crate::timing::phase("load: apply delta and validate");
223	38×	let mut state = base;
224	38×	for (h, c, texts) in delta {
225	33×	if history {
226	26×	state.add_known(h, c, texts);
227	26×	} else {
228	7×	state.add_unchecked(c);
229	7×	}
230		}
231	38×	state.validate()?;
232	38×	return Ok(Some(state));
233		}
234	1739×	Ok(None)
235	1778×	}
236		}

Coverage / crates/oxy-core/src/repo/checkpoint.rs 92.4%

Keyboard