crates/oxy-core/src/repo/checkpoint.rs at f7510b63cceb · andymccall/oxygen

# f7510b63cceb oxygen / crates / oxy-core / src / repo / checkpoint.rs

9.3 KiB · Rust · last changed Bookmarks with / in their names push and checkpoint 1 month ago source blame history raw

//! Checkpoints (spec §11): a serialized state kept per bookmark so loading
//! a state costs the changes since the checkpoint rather than all of them.
//!
//! The newest checkpoint of each bookmark is pointed at by
//! `.oxy/checkpoints/<bookmark>`; older ones become unreachable and are
//! garbage collected. A checkpoint applies to a requested head set when all
//! of its heads lie in that set's closure, which the loader establishes while
//! walking from the requested heads and stopping at checkpoint members.

use super::*;
use crate::objects::Checkpoint;

impl Repo {
    fn checkpoint_pointer(&self, bookmark: &str) -> PathBuf {
        self.oxy.join("checkpoints").join(name_file(bookmark))
    }

    /// The newest checkpoint recorded for a bookmark.
    pub fn latest_checkpoint(&self, bookmark: &str) -> Result<Option<(ObjectRef, Checkpoint)>> {
        match fs::read_to_string(self.checkpoint_pointer(bookmark)) {
            Ok(h) => {
                let r = ObjectRef::from_hex(h.trim())
                    .ok_or_else(|| RepoError::Msg("corrupt checkpoint pointer".into()))?;
                match self.store.get_typed(&r, ObjectType::Checkpoint) {
                    Ok(bytes) => Ok(Some((r, Checkpoint::decode(&bytes)?))),
                    Err(StoreError::NotFound(_)) => Ok(None),
                    Err(e) => Err(e.into()),
                }
            }
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
            Err(e) => Err(e.into()),
        }
    }

    /// Every checkpoint pointer, for garbage collection roots.
    pub(crate) fn checkpoint_refs(&self) -> Result<Vec<ObjectRef>> {
        let dir = self.oxy.join("checkpoints");
        let mut out = Vec::new();
        if let Ok(rd) = fs::read_dir(&dir) {
            for e in rd.flatten() {
                if let Ok(h) = fs::read_to_string(e.path()) {
                    if let Some(r) = ObjectRef::from_hex(h.trim()) {
                        out.push(r);
                    }
                }
            }
        }
        Ok(out)
    }

    /// Write a checkpoint of `state` (whose heads are `heads`) for `bookmark`.
    pub(crate) fn write_checkpoint(
        &self,
        bookmark: &str,
        state: &State,
        heads: &[ChangeHash],
    ) -> Result<(ObjectRef, usize)> {
        let batch = self.store.begin_batch()?;
        let (blob, payload_len) = self.store_checkpoint_payload(state)?;
        let cp = Checkpoint {
            heads: StateObject::new(heads.to_vec()).heads,
            count: state.len() as u64,
            payload: blob,
        };
        let r = self.store.put(ObjectType::Checkpoint, &cp.encode())?;
        batch.finish()?;
        fs::create_dir_all(self.oxy.join("checkpoints"))?;
        write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())?;
        Ok((r, payload_len))
    }

    /// Store `state` as a version 2 payload (spec §11): the graphs of large
    /// files as blobs of their own, shared with earlier checkpoints when
    /// unchanged, and the index naming them. Returns the index blob and its size.
    pub(crate) fn store_checkpoint_payload(&self, state: &State) -> Result<(ObjectRef, usize)> {
        let graphs = state.checkpoint_file_graphs();
        for (_, enc) in &graphs {
            if enc.len() >= crate::state::INLINE_GRAPH_LIMIT {
                self.store_blob(enc)?;
            }
        }
        let index = state.checkpoint_index(&graphs, Repo::blob_ref);
        Ok((self.store_blob(&index)?, index.len()))
    }

    /// The graph blobs a checkpoint's payload names (none for version 1).
    pub(crate) fn checkpoint_graph_blobs(&self, cp: &Checkpoint) -> Result<Vec<ObjectRef>> {
        let payload = self.read_blob(&cp.payload)?;
        Ok(State::checkpoint_graph_refs(&payload)?)
    }

    /// Decode a checkpoint's payload, reading its graph blobs from the store.
    pub(crate) fn state_from_checkpoint(&self, payload: &[u8]) -> Result<State> {
        Ok(State::from_checkpoint(payload, |refs| {
            self.read_blobs(refs).map_err(|e| {
                crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}"))
            })
        })?)
    }

    /// Checkpoint the current bookmark's state now.
    pub fn checkpoint(&self) -> Result<(ObjectRef, usize, usize)> {
        let heads = self.bookmark_heads()?;
        let state = self.load_state_full(&heads)?;
        let (r, bytes) = self.write_checkpoint(&self.config.bookmark, &state, &heads)?;
        Ok((r, state.len(), bytes))
    }

    /// Point `bookmark` at an already stored checkpoint (a shallow clone's base).
    pub(crate) fn install_checkpoint(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
        Checkpoint::decode(&self.store.get_typed(r, ObjectType::Checkpoint)?)?;
        fs::create_dir_all(self.oxy.join("checkpoints"))?;
        write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())
    }

    /// A bundle holding a checkpoint of exactly `heads` with its payload
    /// (spec §16.1 `checkpoint`); the current bookmark checkpoint is reused
    /// when it matches, otherwise one is made and, when `heads` are the
    /// bookmark's, kept.
    pub fn checkpoint_bundle_for(&self, heads: &[ChangeHash]) -> Result<Bundle> {
        let wanted = StateObject::new(heads.to_vec()).heads;
        let bookmark = self.config.bookmark.clone();
        let current = self.bookmark_heads()?;
        let r = match self.latest_checkpoint(&bookmark)? {
            Some((r, cp)) if cp.heads == wanted => r,
            _ => {
                let state = self.load_state(&wanted)?;
                if current == wanted {
                    self.write_checkpoint(&bookmark, &state, &wanted)?.0
                } else {
                    let (blob, _) = self.store_checkpoint_payload(&state)?;
                    let cp = Checkpoint {
                        heads: wanted.clone(),
                        count: state.len() as u64,
                        payload: blob,
                    };
                    self.store.put(ObjectType::Checkpoint, &cp.encode())?
                }
            }
        };
        let cp = Checkpoint::decode(&self.store.get_typed(&r, ObjectType::Checkpoint)?)?;
        let mut bundle = Bundle::default();
        for c in Blob::decode(&self.store.get_typed(&cp.payload, ObjectType::Blob)?)?.chunks {
            bundle.objects.push(self.store.get(&c)?);
        }
        bundle.objects.push(self.store.get(&cp.payload)?);
        for g in self.checkpoint_graph_blobs(&cp)? {
            for c in Blob::decode(&self.store.get_typed(&g, ObjectType::Blob)?)?.chunks {
                bundle.objects.push(self.store.get(&c)?);
            }
            bundle.objects.push(self.store.get(&g)?);
        }
        bundle.objects.push((ObjectType::Checkpoint, cp.encode()));
        Ok(bundle)
    }

    /// Load the state of `heads` using a checkpoint when one applies.
    pub(crate) fn load_state_checkpointed(&self, heads: &[ChangeHash]) -> Result<Option<State>> {
        self.load_state_checkpointed_as(heads, false)
    }

    /// As [`Repo::load_state_checkpointed`]; `history` builds no file graph.
    pub(crate) fn load_state_checkpointed_as(
        &self,
        heads: &[ChangeHash],
        history: bool,
    ) -> Result<Option<State>> {
        let mut bookmarks: Vec<String> = vec![self.config.bookmark.clone()];
        if let Ok(rd) = fs::read_dir(self.oxy.join("checkpoints")) {
            for e in rd.flatten() {
                let name = file_name_to_name(&e.file_name().to_string_lossy());
                if !bookmarks.contains(&name) {
                    bookmarks.push(name);
                }
            }
        }
        for b in bookmarks {
            let Some((_, cp)) = self.latest_checkpoint(&b)? else {
                continue;
            };
            let payload = {
                let _t = crate::timing::phase("load: read checkpoint");
                self.read_blob(&cp.payload)?
            };
            let base = {
                let _t = crate::timing::phase("load: decode checkpoint");
                if history {
                    State::from_checkpoint_history(&payload)?
                } else {
                    self.state_from_checkpoint(&payload)?
                }
            };
            let cp_heads: BTreeSet<ChangeHash> = cp.heads.iter().copied().collect();
            let mut found: BTreeSet<ChangeHash> = BTreeSet::new();
            let mut delta: Vec<Change> = Vec::new();
            let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
            let mut stack: Vec<ChangeHash> = heads.to_vec();
            while let Some(h) = stack.pop() {
                if !seen.insert(h) {
                    continue;
                }
                if base.contains(&h) {
                    if cp_heads.contains(&h) {
                        found.insert(h);
                    }
                    continue;
                }
                let c = self.store.get_change(&h)?;
                stack.extend(c.deps.iter().copied());
                delta.push(c);
            }
            if found.len() != cp_heads.len() {
                continue;
            }
            let _t = crate::timing::phase("load: apply delta and validate");
            let mut state = base;
            for c in delta {
                state.add_unchecked(c);
            }
            state.validate()?;
            return Ok(Some(state));
        }
        Ok(None)
    }
}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1	//! Checkpoints (spec §11): a serialized state kept per bookmark so loading
2	//! a state costs the changes since the checkpoint rather than all of them.
3	//!
4	//! The newest checkpoint of each bookmark is pointed at by
5	//! `.oxy/checkpoints/<bookmark>`; older ones become unreachable and are
6	//! garbage collected. A checkpoint applies to a requested head set when all
7	//! of its heads lie in that set's closure, which the loader establishes while
8	//! walking from the requested heads and stopping at checkpoint members.
9
10	use super::*;
11	use crate::objects::Checkpoint;
12
13	impl Repo {
14	fn checkpoint_pointer(&self, bookmark: &str) -> PathBuf {
15	self.oxy.join("checkpoints").join(name_file(bookmark))
16	}
17
18	/// The newest checkpoint recorded for a bookmark.
19	pub fn latest_checkpoint(&self, bookmark: &str) -> Result<Option<(ObjectRef, Checkpoint)>> {
20	match fs::read_to_string(self.checkpoint_pointer(bookmark)) {
21	Ok(h) => {
22	let r = ObjectRef::from_hex(h.trim())
23	.ok_or_else(\|\| RepoError::Msg("corrupt checkpoint pointer".into()))?;
24	match self.store.get_typed(&r, ObjectType::Checkpoint) {
25	Ok(bytes) => Ok(Some((r, Checkpoint::decode(&bytes)?))),
26	Err(StoreError::NotFound(_)) => Ok(None),
27	Err(e) => Err(e.into()),
28	}
29	}
30	Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
31	Err(e) => Err(e.into()),
32	}
33	}
34
35	/// Every checkpoint pointer, for garbage collection roots.
36	pub(crate) fn checkpoint_refs(&self) -> Result<Vec<ObjectRef>> {
37	let dir = self.oxy.join("checkpoints");
38	let mut out = Vec::new();
39	if let Ok(rd) = fs::read_dir(&dir) {
40	for e in rd.flatten() {
41	if let Ok(h) = fs::read_to_string(e.path()) {
42	if let Some(r) = ObjectRef::from_hex(h.trim()) {
43	out.push(r);
44	}
45	}
46	}
47	}
48	Ok(out)
49	}
50
51	/// Write a checkpoint of `state` (whose heads are `heads`) for `bookmark`.
52	pub(crate) fn write_checkpoint(
53	&self,
54	bookmark: &str,
55	state: &State,
56	heads: &[ChangeHash],
57	) -> Result<(ObjectRef, usize)> {
58	let batch = self.store.begin_batch()?;
59	let (blob, payload_len) = self.store_checkpoint_payload(state)?;
60	let cp = Checkpoint {
61	heads: StateObject::new(heads.to_vec()).heads,
62	count: state.len() as u64,
63	payload: blob,
64	};
65	let r = self.store.put(ObjectType::Checkpoint, &cp.encode())?;
66	batch.finish()?;
67	fs::create_dir_all(self.oxy.join("checkpoints"))?;
68	write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())?;
69	Ok((r, payload_len))
70	}
71
72	/// Store `state` as a version 2 payload (spec §11): the graphs of large
73	/// files as blobs of their own, shared with earlier checkpoints when
74	/// unchanged, and the index naming them. Returns the index blob and its size.
75	pub(crate) fn store_checkpoint_payload(&self, state: &State) -> Result<(ObjectRef, usize)> {
76	let graphs = state.checkpoint_file_graphs();
77	for (_, enc) in &graphs {
78	if enc.len() >= crate::state::INLINE_GRAPH_LIMIT {
79	self.store_blob(enc)?;
80	}
81	}
82	let index = state.checkpoint_index(&graphs, Repo::blob_ref);
83	Ok((self.store_blob(&index)?, index.len()))
84	}
85
86	/// The graph blobs a checkpoint's payload names (none for version 1).
87	pub(crate) fn checkpoint_graph_blobs(&self, cp: &Checkpoint) -> Result<Vec<ObjectRef>> {
88	let payload = self.read_blob(&cp.payload)?;
89	Ok(State::checkpoint_graph_refs(&payload)?)
90	}
91
92	/// Decode a checkpoint's payload, reading its graph blobs from the store.
93	pub(crate) fn state_from_checkpoint(&self, payload: &[u8]) -> Result<State> {
94	Ok(State::from_checkpoint(payload, \|refs\| {
95	self.read_blobs(refs).map_err(\|e\| {
96	crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}"))
97	})
98	})?)
99	}
100
101	/// Checkpoint the current bookmark's state now.
102	pub fn checkpoint(&self) -> Result<(ObjectRef, usize, usize)> {
103	let heads = self.bookmark_heads()?;
104	let state = self.load_state_full(&heads)?;
105	let (r, bytes) = self.write_checkpoint(&self.config.bookmark, &state, &heads)?;
106	Ok((r, state.len(), bytes))
107	}
108
109	/// Point `bookmark` at an already stored checkpoint (a shallow clone's base).
110	pub(crate) fn install_checkpoint(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
111	Checkpoint::decode(&self.store.get_typed(r, ObjectType::Checkpoint)?)?;
112	fs::create_dir_all(self.oxy.join("checkpoints"))?;
113	write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())
114	}
115
116	/// A bundle holding a checkpoint of exactly `heads` with its payload
117	/// (spec §16.1 `checkpoint`); the current bookmark checkpoint is reused
118	/// when it matches, otherwise one is made and, when `heads` are the
119	/// bookmark's, kept.
120	pub fn checkpoint_bundle_for(&self, heads: &[ChangeHash]) -> Result<Bundle> {
121	let wanted = StateObject::new(heads.to_vec()).heads;
122	let bookmark = self.config.bookmark.clone();
123	let current = self.bookmark_heads()?;
124	let r = match self.latest_checkpoint(&bookmark)? {
125	Some((r, cp)) if cp.heads == wanted => r,
126	_ => {
127	let state = self.load_state(&wanted)?;
128	if current == wanted {
129	self.write_checkpoint(&bookmark, &state, &wanted)?.0
130	} else {
131	let (blob, _) = self.store_checkpoint_payload(&state)?;
132	let cp = Checkpoint {
133	heads: wanted.clone(),
134	count: state.len() as u64,
135	payload: blob,
136	};
137	self.store.put(ObjectType::Checkpoint, &cp.encode())?
138	}
139	}
140	};
141	let cp = Checkpoint::decode(&self.store.get_typed(&r, ObjectType::Checkpoint)?)?;
142	let mut bundle = Bundle::default();
143	for c in Blob::decode(&self.store.get_typed(&cp.payload, ObjectType::Blob)?)?.chunks {
144	bundle.objects.push(self.store.get(&c)?);
145	}
146	bundle.objects.push(self.store.get(&cp.payload)?);
147	for g in self.checkpoint_graph_blobs(&cp)? {
148	for c in Blob::decode(&self.store.get_typed(&g, ObjectType::Blob)?)?.chunks {
149	bundle.objects.push(self.store.get(&c)?);
150	}
151	bundle.objects.push(self.store.get(&g)?);
152	}
153	bundle.objects.push((ObjectType::Checkpoint, cp.encode()));
154	Ok(bundle)
155	}
156
157	/// Load the state of `heads` using a checkpoint when one applies.
158	pub(crate) fn load_state_checkpointed(&self, heads: &[ChangeHash]) -> Result<Option<State>> {
159	self.load_state_checkpointed_as(heads, false)
160	}
161
162	/// As [`Repo::load_state_checkpointed`]; `history` builds no file graph.
163	pub(crate) fn load_state_checkpointed_as(
164	&self,
165	heads: &[ChangeHash],
166	history: bool,
167	) -> Result<Option<State>> {
168	let mut bookmarks: Vec<String> = vec![self.config.bookmark.clone()];
169	if let Ok(rd) = fs::read_dir(self.oxy.join("checkpoints")) {
170	for e in rd.flatten() {
171	let name = file_name_to_name(&e.file_name().to_string_lossy());
172	if !bookmarks.contains(&name) {
173	bookmarks.push(name);
174	}
175	}
176	}
177	for b in bookmarks {
178	let Some((_, cp)) = self.latest_checkpoint(&b)? else {
179	continue;
180	};
181	let payload = {
182	let _t = crate::timing::phase("load: read checkpoint");
183	self.read_blob(&cp.payload)?
184	};
185	let base = {
186	let _t = crate::timing::phase("load: decode checkpoint");
187	if history {
188	State::from_checkpoint_history(&payload)?
189	} else {
190	self.state_from_checkpoint(&payload)?
191	}
192	};
193	let cp_heads: BTreeSet<ChangeHash> = cp.heads.iter().copied().collect();
194	let mut found: BTreeSet<ChangeHash> = BTreeSet::new();
195	let mut delta: Vec<Change> = Vec::new();
196	let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
197	let mut stack: Vec<ChangeHash> = heads.to_vec();
198	while let Some(h) = stack.pop() {
199	if !seen.insert(h) {
200	continue;
201	}
202	if base.contains(&h) {
203	if cp_heads.contains(&h) {
204	found.insert(h);
205	}
206	continue;
207	}
208	let c = self.store.get_change(&h)?;
209	stack.extend(c.deps.iter().copied());
210	delta.push(c);
211	}
212	if found.len() != cp_heads.len() {
213	continue;
214	}
215	let _t = crate::timing::phase("load: apply delta and validate");
216	let mut state = base;
217	for c in delta {
218	state.add_unchecked(c);
219	}
220	state.validate()?;
221	return Ok(Some(state));
222	}
223	Ok(None)
224	}
225	}

Keyboard