checkpoint.rs 92.4%

170 of 184 lines run, 18 of 30 functions · run #384 on main · run missed

//! Checkpoints (spec §11): a serialized state kept per bookmark so loading
//! a state costs the changes since the checkpoint rather than all of them.
//!
//! The newest checkpoint of each bookmark is pointed at by
//! `.oxy/checkpoints/<bookmark>`; older ones become unreachable and are
//! garbage collected. A checkpoint applies to a requested head set when all
//! of its heads lie in that set's closure, which the loader establishes while
//! walking from the requested heads and stopping at checkpoint members.

use super::*;
use crate::objects::Checkpoint;

impl Repo {
    fn checkpoint_pointer(&self, bookmark: &str) -> PathBuf {
        self.oxy.join("checkpoints").join(name_file(bookmark))
    }

    /// The newest checkpoint recorded for a bookmark.
    pub fn latest_checkpoint(&self, bookmark: &str) -> Result<Option<(ObjectRef, Checkpoint)>> {
        match fs::read_to_string(self.checkpoint_pointer(bookmark)) {
            Ok(h) => {
                let r = ObjectRef::from_hex(h.trim())
                    .ok_or_else(|| RepoError::Msg("corrupt checkpoint pointer".into()))?;
                match self.store.get_typed(&r, ObjectType::Checkpoint) {
                    Ok(bytes) => Ok(Some((r, Checkpoint::decode(&bytes)?))),
                    Err(StoreError::NotFound(_)) => Ok(None),
                    Err(e) => Err(e.into()),
                }
            }
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
            Err(e) => Err(e.into()),
        }
    }

    /// Every checkpoint pointer, for garbage collection roots.
    pub(crate) fn checkpoint_refs(&self) -> Result<Vec<ObjectRef>> {
        let dir = self.oxy.join("checkpoints");
        let mut out = Vec::new();
        if let Ok(rd) = fs::read_dir(&dir) {
            for e in rd.flatten() {
                if let Ok(h) = fs::read_to_string(e.path()) {
                    if let Some(r) = ObjectRef::from_hex(h.trim()) {
                        out.push(r);
                    }
                }
            }
        }
        Ok(out)
    }

    /// Write a checkpoint of `state` (whose heads are `heads`) for `bookmark`.
    pub(crate) fn write_checkpoint(
        &self,
        bookmark: &str,
        state: &State,
        heads: &[ChangeHash],
    ) -> Result<(ObjectRef, usize)> {
        let batch = self.store.begin_batch()?;
        let (blob, payload_len) = self.store_checkpoint_payload(state)?;
        let cp = Checkpoint {
            heads: StateObject::new(heads.to_vec()).heads,
            count: state.len() as u64,
            payload: blob,
        };
        let r = self.store.put(ObjectType::Checkpoint, &cp.encode())?;
        batch.finish()?;
        fs::create_dir_all(self.oxy.join("checkpoints"))?;
        write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())?;
        Ok((r, payload_len))
    }

    /// Store `state` as a version 2 payload (spec §11): the graphs of large
    /// files as blobs of their own, shared with earlier checkpoints when
    /// unchanged, and the index naming them. Returns the index blob and its size.
    pub(crate) fn store_checkpoint_payload(&self, state: &State) -> Result<(ObjectRef, usize)> {
        let graphs = state.checkpoint_file_graphs();
        for (_, enc) in &graphs {
            if enc.len() >= crate::state::INLINE_GRAPH_LIMIT {
                self.store_blob(enc)?;
            }
        }
        let index = state.checkpoint_index(&graphs, Repo::blob_ref);
        Ok((self.store_blob(&index)?, index.len()))
    }

    /// The graph blobs a checkpoint's payload names (none for version 1).
    pub(crate) fn checkpoint_graph_blobs(&self, cp: &Checkpoint) -> Result<Vec<ObjectRef>> {
        let payload = self.read_blob(&cp.payload)?;
        Ok(State::checkpoint_graph_refs(&payload)?)
    }

    /// Decode a checkpoint's payload, reading its graph blobs from the store.
    pub(crate) fn state_from_checkpoint(&self, payload: &[u8]) -> Result<State> {
        Ok(State::from_checkpoint(payload, |refs| {
            self.read_blobs(refs).map_err(|e| {
                crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}"))
            })
        })?)
    }

    /// Checkpoint the current bookmark's state now.
    pub fn checkpoint(&self) -> Result<(ObjectRef, usize, usize)> {
        let heads = self.bookmark_heads()?;
        let state = self.load_state_full(&heads)?;
        let (r, bytes) = self.write_checkpoint(&self.config.bookmark, &state, &heads)?;
        Ok((r, state.len(), bytes))
    }

    /// Point `bookmark` at an already stored checkpoint (a shallow clone's base).
    pub(crate) fn install_checkpoint(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
        Checkpoint::decode(&self.store.get_typed(r, ObjectType::Checkpoint)?)?;
        fs::create_dir_all(self.oxy.join("checkpoints"))?;
        write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())
    }

    /// A bundle holding a checkpoint of exactly `heads` with its payload
    /// (spec §16.1 `checkpoint`); the current bookmark checkpoint is reused
    /// when it matches, otherwise one is made and, when `heads` are the
    /// bookmark's, kept.
    pub fn checkpoint_bundle_for(&self, heads: &[ChangeHash]) -> Result<Bundle> {
        let t_find = crate::timing::phase("checkpoint: find or make");
        let wanted = StateObject::new(heads.to_vec()).heads;
        let bookmark = self.config.bookmark.clone();
        let current = self.bookmark_heads()?;
        let r = match self.latest_checkpoint(&bookmark)? {
            Some((r, cp)) if cp.heads == wanted => r,
            _ => {
                let state = self.load_state(&wanted)?;
                if current == wanted {
                    self.write_checkpoint(&bookmark, &state, &wanted)?.0
                } else {
                    let (blob, _) = self.store_checkpoint_payload(&state)?;
                    let cp = Checkpoint {
                        heads: wanted.clone(),
                        count: state.len() as u64,
                        payload: blob,
                    };
                    self.store.put(ObjectType::Checkpoint, &cp.encode())?
                }
            }
        };
        drop(t_find);
        let _t = crate::timing::phase("checkpoint: gather its objects");
        let cp = Checkpoint::decode(&self.store.get_typed(&r, ObjectType::Checkpoint)?)?;
        let mut bundle = Bundle::default();
        for c in Blob::decode(&self.store.get_typed(&cp.payload, ObjectType::Blob)?)?.chunks {
            bundle.objects.push(self.store.get(&c)?);
        }
        bundle.objects.push(self.store.get(&cp.payload)?);
        // The graph blobs and their chunks in two parallel reads (#137): a
        // big state's are tens of thousands, and one by one they took six
        // minutes for VS Code.
        let graphs = self.checkpoint_graph_blobs(&cp)?;
        let mut blobs = Vec::with_capacity(graphs.len());
        for got in self.store.get_many(&graphs) {
            blobs.push(got?);
        }
        let mut chunks = Vec::new();
        for (ty, body) in &blobs {
            if *ty != ObjectType::Blob {
                return fail("a checkpoint graph is not a blob");
            }
            chunks.extend(Blob::decode(body)?.chunks);
        }
        for got in self.store.get_many(&chunks) {
            bundle.objects.push(got?);
        }
        bundle.objects.extend(blobs);
        bundle.objects.push((ObjectType::Checkpoint, cp.encode()));
        Ok(bundle)
    }

    /// Load the state of `heads` using a checkpoint when one applies.
    pub(crate) fn load_state_checkpointed(&self, heads: &[ChangeHash]) -> Result<Option<State>> {
        self.load_state_checkpointed_as(heads, false)
    }

    /// As [`Repo::load_state_checkpointed`]; `history` builds no file graph.
    pub(crate) fn load_state_checkpointed_as(
        &self,
        heads: &[ChangeHash],
        history: bool,
    ) -> Result<Option<State>> {
        let mut bookmarks: Vec<String> = vec![self.config.bookmark.clone()];
        if let Ok(rd) = fs::read_dir(self.oxy.join("checkpoints")) {
            for e in rd.flatten() {
                let name = file_name_to_name(&e.file_name().to_string_lossy());
                if !bookmarks.contains(&name) {
                    bookmarks.push(name);
                }
            }
        }
        for b in bookmarks {
            let Some((_, cp)) = self.latest_checkpoint(&b)? else {
                continue;
            };
            let payload = {
                let _t = crate::timing::phase("load: read checkpoint");
                self.read_blob(&cp.payload)?
            };
            let base = {
                let _t = crate::timing::phase("load: decode checkpoint");
                if history {
                    State::from_checkpoint_history(&payload)?
                } else {
                    self.state_from_checkpoint(&payload)?
                }
            };
            let cp_heads: BTreeSet<ChangeHash> = cp.heads.iter().copied().collect();
            let mut found: BTreeSet<ChangeHash> = BTreeSet::new();
            let mut delta: Vec<(ChangeHash, Change, Vec<crate::objects::TextRef>)> = Vec::new();
            let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
            let mut stack: Vec<ChangeHash> = heads.to_vec();
            while let Some(h) = stack.pop() {
                if !seen.insert(h) {
                    continue;
                }
                if base.contains(&h) {
                    if cp_heads.contains(&h) {
                        found.insert(h);
                    }
                    continue;
                }
                // For history, the skeleton: text is read only for the
                // graphs built later, and a state holding none can be
                // indexed (oxygen#93).
                let (c, texts) = if history {
                    self.store.get_change_lazy(&h)?
                } else {
                    (self.store.get_change(&h)?, Vec::new())
                };
                stack.extend(c.deps.iter().copied());
                delta.push((h, c, texts));
            }
            if found.len() != cp_heads.len() {
                continue;
            }
            let _t = crate::timing::phase("load: apply delta and validate");
            let mut state = base;
            for (h, c, texts) in delta {
                if history {
                    state.add_known(h, c, texts);
                } else {
                    state.add_unchecked(c);
                }
            }
            state.validate()?;
            return Ok(Some(state));
        }
        Ok(None)
    }
}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1		//! Checkpoints (spec §11): a serialized state kept per bookmark so loading
2		//! a state costs the changes since the checkpoint rather than all of them.
3		//!
4		//! The newest checkpoint of each bookmark is pointed at by
5		//! `.oxy/checkpoints/<bookmark>`; older ones become unreachable and are
6		//! garbage collected. A checkpoint applies to a requested head set when all
7		//! of its heads lie in that set's closure, which the loader establishes while
8		//! walking from the requested heads and stopping at checkpoint members.
9
10		use super::*;
11		use crate::objects::Checkpoint;
12
13		impl Repo {
14	3022×	fn checkpoint_pointer(&self, bookmark: &str) -> PathBuf {
15	3022×	self.oxy.join("checkpoints").join(name_file(bookmark))
16	3022×	}
17
18		/// The newest checkpoint recorded for a bookmark.
19	3008×	pub fn latest_checkpoint(&self, bookmark: &str) -> Result<Option<(ObjectRef, Checkpoint)>> {
20	3008×	match fs::read_to_string(self.checkpoint_pointer(bookmark)) {
21	77×	Ok(h) => {
22	77×	let r = ObjectRef::from_hex(h.trim())
23	77×	.ok_or_else(\|\| RepoError::Msg("corrupt checkpoint pointer".into()))?;
24	77×	match self.store.get_typed(&r, ObjectType::Checkpoint) {
25	77×	Ok(bytes) => Ok(Some((r, Checkpoint::decode(&bytes)?))),
26	0×	Err(StoreError::NotFound(_)) => Ok(None),
27	0×	Err(e) => Err(e.into()),
28		}
29		}
30	2931×	Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
31	0×	Err(e) => Err(e.into()),
32		}
33	3008×	}
34
35		/// Every checkpoint pointer, for garbage collection roots.
36	102×	pub(crate) fn checkpoint_refs(&self) -> Result<Vec<ObjectRef>> {
37	102×	let dir = self.oxy.join("checkpoints");
38	102×	let mut out = Vec::new();
39	102×	if let Ok(rd) = fs::read_dir(&dir) {
40	11×	for e in rd.flatten() {
41	11×	if let Ok(h) = fs::read_to_string(e.path()) {
42	11×	if let Some(r) = ObjectRef::from_hex(h.trim()) {
43	11×	out.push(r);
44	11×	}
45	0×	}
46		}
47	91×	}
48	102×	Ok(out)
49	102×	}
50
51		/// Write a checkpoint of `state` (whose heads are `heads`) for `bookmark`.
52	10×	pub(crate) fn write_checkpoint(
53	10×	&self,
54	10×	bookmark: &str,
55	10×	state: &State,
56	10×	heads: &[ChangeHash],
57	10×	) -> Result<(ObjectRef, usize)> {
58	10×	let batch = self.store.begin_batch()?;
59	10×	let (blob, payload_len) = self.store_checkpoint_payload(state)?;
60	10×	let cp = Checkpoint {
61	10×	heads: StateObject::new(heads.to_vec()).heads,
62	10×	count: state.len() as u64,
63	10×	payload: blob,
64	10×	};
65	10×	let r = self.store.put(ObjectType::Checkpoint, &cp.encode())?;
66	10×	batch.finish()?;
67	10×	fs::create_dir_all(self.oxy.join("checkpoints"))?;
68	10×	write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())?;
69	10×	Ok((r, payload_len))
70	10×	}
71
72		/// Store `state` as a version 2 payload (spec §11): the graphs of large
73		/// files as blobs of their own, shared with earlier checkpoints when
74		/// unchanged, and the index naming them. Returns the index blob and its size.
75	10×	pub(crate) fn store_checkpoint_payload(&self, state: &State) -> Result<(ObjectRef, usize)> {
76	10×	let graphs = state.checkpoint_file_graphs();
77	40×	for (_, enc) in &graphs {
78	40×	if enc.len() >= crate::state::INLINE_GRAPH_LIMIT {
79	3×	self.store_blob(enc)?;
80	37×	}
81		}
82	10×	let index = state.checkpoint_index(&graphs, Repo::blob_ref);
83	10×	Ok((self.store_blob(&index)?, index.len()))
84	10×	}
85
86		/// The graph blobs a checkpoint's payload names (none for version 1).
87	9×	pub(crate) fn checkpoint_graph_blobs(&self, cp: &Checkpoint) -> Result<Vec<ObjectRef>> {
88	9×	let payload = self.read_blob(&cp.payload)?;
89	9×	Ok(State::checkpoint_graph_refs(&payload)?)
90	9×	}
91
92		/// Decode a checkpoint's payload, reading its graph blobs from the store.
93	19×	pub(crate) fn state_from_checkpoint(&self, payload: &[u8]) -> Result<State> {
94	19×	Ok(State::from_checkpoint(payload, \|refs\| {
95	4×	self.read_blobs(refs).map_err(\|e\| {
96	2×	crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}"))
97	2×	})
98	4×	})?)
99	19×	}
100
101		/// Checkpoint the current bookmark's state now.
102	6×	pub fn checkpoint(&self) -> Result<(ObjectRef, usize, usize)> {
103	6×	let heads = self.bookmark_heads()?;
104	6×	let state = self.load_state_full(&heads)?;
105	6×	let (r, bytes) = self.write_checkpoint(&self.config.bookmark, &state, &heads)?;
106	6×	Ok((r, state.len(), bytes))
107	6×	}
108
109		/// Point `bookmark` at an already stored checkpoint (a shallow clone's base).
110	4×	pub(crate) fn install_checkpoint(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
111	4×	Checkpoint::decode(&self.store.get_typed(r, ObjectType::Checkpoint)?)?;
112	4×	fs::create_dir_all(self.oxy.join("checkpoints"))?;
113	4×	write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())
114	4×	}
115
116		/// A bundle holding a checkpoint of exactly `heads` with its payload
117		/// (spec §16.1 `checkpoint`); the current bookmark checkpoint is reused
118		/// when it matches, otherwise one is made and, when `heads` are the
119		/// bookmark's, kept.
120	4×	pub fn checkpoint_bundle_for(&self, heads: &[ChangeHash]) -> Result<Bundle> {
121	4×	let t_find = crate::timing::phase("checkpoint: find or make");
122	4×	let wanted = StateObject::new(heads.to_vec()).heads;
123	4×	let bookmark = self.config.bookmark.clone();
124	4×	let current = self.bookmark_heads()?;
125	4×	let r = match self.latest_checkpoint(&bookmark)? {
126	2×	Some((r, cp)) if cp.heads == wanted => r,
127		_ => {
128	2×	let state = self.load_state(&wanted)?;
129	2×	if current == wanted {
130	2×	self.write_checkpoint(&bookmark, &state, &wanted)?.0
131		} else {
132	0×	let (blob, _) = self.store_checkpoint_payload(&state)?;
133	0×	let cp = Checkpoint {
134	0×	heads: wanted.clone(),
135	0×	count: state.len() as u64,
136	0×	payload: blob,
137	0×	};
138	0×	self.store.put(ObjectType::Checkpoint, &cp.encode())?
139		}
140		}
141		};
142	4×	drop(t_find);
143	4×	let _t = crate::timing::phase("checkpoint: gather its objects");
144	4×	let cp = Checkpoint::decode(&self.store.get_typed(&r, ObjectType::Checkpoint)?)?;
145	4×	let mut bundle = Bundle::default();
146	4×	for c in Blob::decode(&self.store.get_typed(&cp.payload, ObjectType::Blob)?)?.chunks {
147	4×	bundle.objects.push(self.store.get(&c)?);
148		}
149	4×	bundle.objects.push(self.store.get(&cp.payload)?);
150		// The graph blobs and their chunks in two parallel reads (#137): a
151		// big state's are tens of thousands, and one by one they took six
152		// minutes for VS Code.
153	4×	let graphs = self.checkpoint_graph_blobs(&cp)?;
154	4×	let mut blobs = Vec::with_capacity(graphs.len());
155	4×	for got in self.store.get_many(&graphs) {
156	1×	blobs.push(got?);
157		}
158	4×	let mut chunks = Vec::new();
159	4×	for (ty, body) in &blobs {
160	1×	if *ty != ObjectType::Blob {
161	0×	return fail("a checkpoint graph is not a blob");
162	1×	}
163	1×	chunks.extend(Blob::decode(body)?.chunks);
164		}
165	4×	for got in self.store.get_many(&chunks) {
166	1×	bundle.objects.push(got?);
167		}
168	4×	bundle.objects.extend(blobs);
169	4×	bundle.objects.push((ObjectType::Checkpoint, cp.encode()));
170	4×	Ok(bundle)
171	4×	}
172
173		/// Load the state of `heads` using a checkpoint when one applies.
174	210×	pub(crate) fn load_state_checkpointed(&self, heads: &[ChangeHash]) -> Result<Option<State>> {
175	210×	self.load_state_checkpointed_as(heads, false)
176	210×	}
177
178		/// As [`Repo::load_state_checkpointed`]; `history` builds no file graph.
179	2258×	pub(crate) fn load_state_checkpointed_as(
180	2258×	&self,
181	2258×	heads: &[ChangeHash],
182	2258×	history: bool,
183	2258×	) -> Result<Option<State>> {
184	2258×	let mut bookmarks: Vec<String> = vec![self.config.bookmark.clone()];
185	2258×	if let Ok(rd) = fs::read_dir(self.oxy.join("checkpoints")) {
186	58×	for e in rd.flatten() {
187	58×	let name = file_name_to_name(&e.file_name().to_string_lossy());
188	58×	if !bookmarks.contains(&name) {
189	0×	bookmarks.push(name);
190	58×	}
191		}
192	2200×	}
193	2258×	for b in bookmarks {
194	2258×	let Some((_, cp)) = self.latest_checkpoint(&b)? else {
195	2200×	continue;
196		};
197	58×	let payload = {
198	58×	let _t = crate::timing::phase("load: read checkpoint");
199	58×	self.read_blob(&cp.payload)?
200		};
201	57×	let base = {
202	58×	let _t = crate::timing::phase("load: decode checkpoint");
203	58×	if history {
204	48×	State::from_checkpoint_history(&payload)?
205		} else {
206	10×	self.state_from_checkpoint(&payload)?
207		}
208		};
209	57×	let cp_heads: BTreeSet<ChangeHash> = cp.heads.iter().copied().collect();
210	57×	let mut found: BTreeSet<ChangeHash> = BTreeSet::new();
211	57×	let mut delta: Vec<(ChangeHash, Change, Vec<crate::objects::TextRef>)> = Vec::new();
212	57×	let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
213	57×	let mut stack: Vec<ChangeHash> = heads.to_vec();
214	240×	while let Some(h) = stack.pop() {
215	183×	if !seen.insert(h) {
216	28×	continue;
217	155×	}
218	155×	if base.contains(&h) {
219	101×	if cp_heads.contains(&h) {
220	52×	found.insert(h);
221	52×	}
222	101×	continue;
223	54×	}
224		// For history, the skeleton: text is read only for the
225		// graphs built later, and a state holding none can be
226		// indexed (oxygen#93).
227	54×	let (c, texts) = if history {
228	47×	self.store.get_change_lazy(&h)?
229		} else {
230	7×	(self.store.get_change(&h)?, Vec::new())
231		};
232	54×	stack.extend(c.deps.iter().copied());
233	54×	delta.push((h, c, texts));
234		}
235	57×	if found.len() != cp_heads.len() {
236	6×	continue;
237	51×	}
238	51×	let _t = crate::timing::phase("load: apply delta and validate");
239	51×	let mut state = base;
240	51×	for (h, c, texts) in delta {
241	45×	if history {
242	38×	state.add_known(h, c, texts);
243	38×	} else {
244	7×	state.add_unchecked(c);
245	7×	}
246		}
247	51×	state.validate()?;
248	51×	return Ok(Some(state));
249		}
250	2206×	Ok(None)
251	2258×	}
252		}

Coverage / crates/oxy-core/src/repo/checkpoint.rs 92.4%

Keyboard