OXYGENHUB

Coverage / crates/oxy-core/src/repo/checkpoint.rs 92.3%

181 of 196 lines run, 21 of 34 functions · run #417 on main · run missed

1
//! Checkpoints (spec §11): a serialized state kept per bookmark so loading
2
//! a state costs the changes since the checkpoint rather than all of them.
3
//!
4
//! The newest checkpoint of each bookmark is pointed at by
5
//! `.oxy/checkpoints/<bookmark>`; older ones become unreachable and are
6
//! garbage collected. A checkpoint applies to a requested head set when all
7
//! of its heads lie in that set's closure, which the loader establishes while
8
//! walking from the requested heads and stopping at checkpoint members.
9
10
use super::*;
11
use crate::objects::Checkpoint;
12
13
impl Repo {
143172×
    fn checkpoint_pointer(&self, bookmark: &str) -> PathBuf {
153172×
        self.oxy.join("checkpoints").join(name_file(bookmark))
163172×
    }
17
18
    /// The newest checkpoint recorded for a bookmark.
193158×
    pub fn latest_checkpoint(&self, bookmark: &str) -> Result<Option<(ObjectRef, Checkpoint)>> {
203158×
        match fs::read_to_string(self.checkpoint_pointer(bookmark)) {
2177×
            Ok(h) => {
2277×
                let r = ObjectRef::from_hex(h.trim())
2377×
                    .ok_or_else(|| RepoError::Msg("corrupt checkpoint pointer".into()))?;
2477×
                match self.store.get_typed(&r, ObjectType::Checkpoint) {
2577×
                    Ok(bytes) => Ok(Some((r, Checkpoint::decode(&bytes)?))),
260×
                    Err(StoreError::NotFound(_)) => Ok(None),
270×
                    Err(e) => Err(e.into()),
28
                }
29
            }
303081×
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
310×
            Err(e) => Err(e.into()),
32
        }
333158×
    }
34
35
    /// Every checkpoint pointer, for garbage collection roots.
36105×
    pub(crate) fn checkpoint_refs(&self) -> Result<Vec<ObjectRef>> {
37105×
        let dir = self.oxy.join("checkpoints");
38105×
        let mut out = Vec::new();
39105×
        if let Ok(rd) = fs::read_dir(&dir) {
4011×
            for e in rd.flatten() {
4111×
                if let Ok(h) = fs::read_to_string(e.path()) {
4211×
                    if let Some(r) = ObjectRef::from_hex(h.trim()) {
4311×
                        out.push(r);
4411×
                    }
450×
                }
46
            }
4794×
        }
48105×
        Ok(out)
49105×
    }
50
51
    /// Write a checkpoint of `state` (whose heads are `heads`) for `bookmark`.
5210×
    pub(crate) fn write_checkpoint(
5310×
        &self,
5410×
        bookmark: &str,
5510×
        state: &State,
5610×
        heads: &[ChangeHash],
5710×
    ) -> Result<(ObjectRef, usize)> {
5810×
        let batch = self.store.begin_batch()?;
5910×
        let (blob, payload_len) = self.store_checkpoint_payload(state)?;
6010×
        let cp = Checkpoint {
6110×
            heads: StateObject::new(heads.to_vec()).heads,
6210×
            count: state.len() as u64,
6310×
            payload: blob,
6410×
        };
6510×
        let r = self.store.put(ObjectType::Checkpoint, &cp.encode())?;
6610×
        batch.finish()?;
6710×
        fs::create_dir_all(self.oxy.join("checkpoints"))?;
6810×
        write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())?;
6910×
        Ok((r, payload_len))
7010×
    }
71
72
    /// Store `state` as a version 2 payload (spec §11): the graphs of large
73
    /// files as blobs of their own, shared with earlier checkpoints when
74
    /// unchanged, and the index naming them. Returns the index blob and its size.
7510×
    pub(crate) fn store_checkpoint_payload(&self, state: &State) -> Result<(ObjectRef, usize)> {
7610×
        let graphs = state.checkpoint_file_graphs();
7740×
        for (_, enc) in &graphs {
7840×
            if enc.len() >= crate::state::INLINE_GRAPH_LIMIT {
793×
                self.store_blob(enc)?;
8037×
            }
81
        }
8210×
        let index = state.checkpoint_index(&graphs, Repo::blob_ref);
8310×
        Ok((self.store_blob(&index)?, index.len()))
8410×
    }
85
86
    /// The graph blobs a checkpoint's payload names (none for version 1).
879×
    pub(crate) fn checkpoint_graph_blobs(&self, cp: &Checkpoint) -> Result<Vec<ObjectRef>> {
889×
        let payload = self.read_blob(&cp.payload)?;
899×
        Ok(State::checkpoint_graph_refs(&payload)?)
909×
    }
91
92
    /// Decode a checkpoint's payload, reading its graph blobs from the store.
9319×
    pub(crate) fn state_from_checkpoint(&self, payload: &[u8]) -> Result<State> {
9419×
        Ok(State::from_checkpoint(payload, |refs| {
954×
            self.read_blobs(refs).map_err(|e| {
962×
                crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}"))
972×
            })
984×
        })?)
9919×
    }
100
101
    /// Checkpoint the current bookmark's state now.
1026×
    pub fn checkpoint(&self) -> Result<(ObjectRef, usize, usize)> {
1036×
        let heads = self.bookmark_heads()?;
1046×
        let state = self.load_state_full(&heads)?;
1056×
        let (r, bytes) = self.write_checkpoint(&self.config.bookmark, &state, &heads)?;
1066×
        Ok((r, state.len(), bytes))
1076×
    }
108
109
    /// Point `bookmark` at an already stored checkpoint (a shallow clone's base).
1104×
    pub(crate) fn install_checkpoint(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
1114×
        Checkpoint::decode(&self.store.get_typed(r, ObjectType::Checkpoint)?)?;
1124×
        fs::create_dir_all(self.oxy.join("checkpoints"))?;
1134×
        write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())
1144×
    }
115
116
    /// The checkpoint of exactly `heads`: the bookmark's latest when it
117
    /// is that state, else one made now (kept as the bookmark's when the
118
    /// bookmark is at `heads`).
1194×
    fn checkpoint_ref_for(&self, heads: &[ChangeHash]) -> Result<ObjectRef> {
1204×
        let _t = crate::timing::phase("checkpoint: find or make");
1214×
        let wanted = StateObject::new(heads.to_vec()).heads;
1224×
        let bookmark = self.config.bookmark.clone();
1234×
        let current = self.bookmark_heads()?;
1244×
        Ok(match self.latest_checkpoint(&bookmark)? {
1252×
            Some((r, cp)) if cp.heads == wanted => r,
126
            _ => {
1272×
                let state = self.load_state(&wanted)?;
1282×
                if current == wanted {
1292×
                    self.write_checkpoint(&bookmark, &state, &wanted)?.0
130
                } else {
1310×
                    let (blob, _) = self.store_checkpoint_payload(&state)?;
1320×
                    let cp = Checkpoint {
1330×
                        heads: wanted.clone(),
1340×
                        count: state.len() as u64,
1350×
                        payload: blob,
1360×
                    };
1370×
                    self.store.put(ObjectType::Checkpoint, &cp.encode())?
138
                }
139
            }
140
        })
1414×
    }
142
143
    /// A checkpoint of exactly `heads` without its graph blobs (spec
144
    /// §16.1 `checkpoint-index`, oxygen#137): the checkpoint object, its
145
    /// payload blob and the blob's chunks. The client asks for the graph
146
    /// blobs the payload names in parts, with `objects`, so neither side
147
    /// holds a big state's whole answer at once.
1484×
    pub fn checkpoint_index_for(&self, heads: &[ChangeHash]) -> Result<Bundle> {
1494×
        let r = self.checkpoint_ref_for(heads)?;
1504×
        let cp = Checkpoint::decode(&self.store.get_typed(&r, ObjectType::Checkpoint)?)?;
1514×
        let mut bundle = Bundle::default();
1524×
        let chunks = Blob::decode(&self.store.get_typed(&cp.payload, ObjectType::Blob)?)?.chunks;
1534×
        for got in self.store.get_many(&chunks) {
1544×
            bundle.objects.push(got?);
155
        }
1564×
        bundle.objects.push(self.store.get(&cp.payload)?);
1574×
        bundle.objects.push((ObjectType::Checkpoint, cp.encode()));
1584×
        Ok(bundle)
1594×
    }
160
161
    /// A checkpoint of exactly `heads` with its graph blobs and their
162
    /// chunks, in one answer (spec §16.1 `checkpoint`): what a client
163
    /// older than `checkpoint-index` asks for.
1641×
    pub fn checkpoint_bundle_for(&self, heads: &[ChangeHash]) -> Result<Bundle> {
1651×
        let mut bundle = self.checkpoint_index_for(heads)?;
1661×
        let _t = crate::timing::phase("checkpoint: gather its objects");
1671×
        let cp = match bundle.objects.last() {
1681×
            Some((ObjectType::Checkpoint, body)) => Checkpoint::decode(body)?,
1690×
            _ => return fail("a checkpoint index without its checkpoint"),
170
        };
171
        // The graph blobs and their chunks in two parallel reads (#137): a
172
        // big state's are tens of thousands, and one by one they took six
173
        // minutes for VS Code.
1741×
        let graphs = self.checkpoint_graph_blobs(&cp)?;
1751×
        let mut blobs = Vec::with_capacity(graphs.len());
1761×
        for got in self.store.get_many(&graphs) {
1771×
            blobs.push(got?);
178
        }
1791×
        let mut chunks = Vec::new();
1801×
        for (ty, body) in &blobs {
1811×
            if *ty != ObjectType::Blob {
1820×
                return fail("a checkpoint graph is not a blob");
1831×
            }
1841×
            chunks.extend(Blob::decode(body)?.chunks);
185
        }
1861×
        let last = bundle.objects.pop();
1871×
        for got in self.store.get_many(&chunks) {
1881×
            bundle.objects.push(got?);
189
        }
1901×
        bundle.objects.extend(blobs);
1911×
        bundle.objects.extend(last);
1921×
        Ok(bundle)
1931×
    }
194
195
    /// Load the state of `heads` using a checkpoint when one applies.
196228×
    pub(crate) fn load_state_checkpointed(&self, heads: &[ChangeHash]) -> Result<Option<State>> {
197228×
        self.load_state_checkpointed_as(heads, false)
198228×
    }
199
200
    /// As [`Repo::load_state_checkpointed`]; `history` builds no file graph.
2012372×
    pub(crate) fn load_state_checkpointed_as(
2022372×
        &self,
2032372×
        heads: &[ChangeHash],
2042372×
        history: bool,
2052372×
    ) -> Result<Option<State>> {
2062372×
        let mut bookmarks: Vec<String> = vec![self.config.bookmark.clone()];
2072372×
        if let Ok(rd) = fs::read_dir(self.oxy.join("checkpoints")) {
20858×
            for e in rd.flatten() {
20958×
                let name = file_name_to_name(&e.file_name().to_string_lossy());
21058×
                if !bookmarks.contains(&name) {
2110×
                    bookmarks.push(name);
21258×
                }
213
            }
2142314×
        }
2152372×
        for b in bookmarks {
2162372×
            let Some((_, cp)) = self.latest_checkpoint(&b)? else {
2172314×
                continue;
218
            };
21958×
            let payload = {
22058×
                let _t = crate::timing::phase("load: read checkpoint");
22158×
                self.read_blob(&cp.payload)?
222
            };
22357×
            let base = {
22458×
                let _t = crate::timing::phase("load: decode checkpoint");
22558×
                if history {
22648×
                    State::from_checkpoint_history(&payload)?
227
                } else {
22810×
                    self.state_from_checkpoint(&payload)?
229
                }
230
            };
23157×
            let cp_heads: BTreeSet<ChangeHash> = cp.heads.iter().copied().collect();
23257×
            let mut found: BTreeSet<ChangeHash> = BTreeSet::new();
23357×
            let mut delta: Vec<(ChangeHash, Change, Vec<crate::objects::TextRef>)> = Vec::new();
23457×
            let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
23557×
            let mut stack: Vec<ChangeHash> = heads.to_vec();
236240×
            while let Some(h) = stack.pop() {
237183×
                if !seen.insert(h) {
23828×
                    continue;
239155×
                }
240155×
                if base.contains(&h) {
241101×
                    if cp_heads.contains(&h) {
24252×
                        found.insert(h);
24352×
                    }
244101×
                    continue;
24554×
                }
246
                // For history, the skeleton: text is read only for the
247
                // graphs built later, and a state holding none can be
248
                // indexed (oxygen#93).
24954×
                let (c, texts) = if history {
25047×
                    self.store.get_change_lazy(&h)?
251
                } else {
2527×
                    (self.store.get_change(&h)?, Vec::new())
253
                };
25454×
                stack.extend(c.deps.iter().copied());
25554×
                delta.push((h, c, texts));
256
            }
25757×
            if found.len() != cp_heads.len() {
2586×
                continue;
25951×
            }
26051×
            let _t = crate::timing::phase("load: apply delta and validate");
26151×
            let mut state = base;
26251×
            for (h, c, texts) in delta {
26345×
                if history {
26438×
                    state.add_known(h, c, texts);
26538×
                } else {
2667×
                    state.add_unchecked(c);
2677×
                }
268
            }
26951×
            state.validate()?;
27051×
            return Ok(Some(state));
271
        }
2722320×
        Ok(None)
2732372×
    }
274
}