OXYGENHUB
# afaba5f1306d oxygen / crates / oxy-core / src / repo / checkpoint.rs
8.9 KiB · Rust · last changed Checkpoint payload version 2: large file graphs as shared blobs named from an index, read in parallel (loose and packed) and decoded in parallel; version 1 still read; gc, fsck and shallow-clone bundles carry the graph blobs (spec 11, Draft 0.8j) 1 month ago source blame history raw
1 
//! Checkpoints (spec §11): a serialized state kept per bookmark so loading
2 
//! a state costs the changes since the checkpoint rather than all of them.
3 
//!
4 
//! The newest checkpoint of each bookmark is pointed at by
5 
//! `.oxy/checkpoints/<bookmark>`; older ones become unreachable and are
6 
//! garbage collected. A checkpoint applies to a requested head set when all
7 
//! of its heads lie in that set's closure, which the loader establishes while
8 
//! walking from the requested heads and stopping at checkpoint members.
9
10 
use super::*;
11 
use crate::objects::Checkpoint;
12
13 
impl Repo {
14 
    fn checkpoint_pointer(&self, bookmark: &str) -> PathBuf {
15 
        self.oxy.join("checkpoints").join(bookmark)
16 
    }
17
18 
    /// The newest checkpoint recorded for a bookmark.
19 
    pub fn latest_checkpoint(&self, bookmark: &str) -> Result<Option<(ObjectRef, Checkpoint)>> {
20 
        match fs::read_to_string(self.checkpoint_pointer(bookmark)) {
21 
            Ok(h) => {
22 
                let r = ObjectRef::from_hex(h.trim())
23 
                    .ok_or_else(|| RepoError::Msg("corrupt checkpoint pointer".into()))?;
24 
                match self.store.get_typed(&r, ObjectType::Checkpoint) {
25 
                    Ok(bytes) => Ok(Some((r, Checkpoint::decode(&bytes)?))),
26 
                    Err(StoreError::NotFound(_)) => Ok(None),
27 
                    Err(e) => Err(e.into()),
28 
                }
29 
            }
30 
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
31 
            Err(e) => Err(e.into()),
32 
        }
33 
    }
34
35 
    /// Every checkpoint pointer, for garbage collection roots.
36 
    pub(crate) fn checkpoint_refs(&self) -> Result<Vec<ObjectRef>> {
37 
        let dir = self.oxy.join("checkpoints");
38 
        let mut out = Vec::new();
39 
        if let Ok(rd) = fs::read_dir(&dir) {
40 
            for e in rd.flatten() {
41 
                if let Ok(h) = fs::read_to_string(e.path()) {
42 
                    if let Some(r) = ObjectRef::from_hex(h.trim()) {
43 
                        out.push(r);
44 
                    }
45 
                }
46 
            }
47 
        }
48 
        Ok(out)
49 
    }
50
51 
    /// Write a checkpoint of `state` (whose heads are `heads`) for `bookmark`.
52 
    pub(crate) fn write_checkpoint(
53 
        &self,
54 
        bookmark: &str,
55 
        state: &State,
56 
        heads: &[ChangeHash],
57 
    ) -> Result<(ObjectRef, usize)> {
58 
        let batch = self.store.begin_batch()?;
59 
        let (blob, payload_len) = self.store_checkpoint_payload(state)?;
60 
        let cp = Checkpoint {
61 
            heads: StateObject::new(heads.to_vec()).heads,
62 
            count: state.len() as u64,
63 
            payload: blob,
64 
        };
65 
        let r = self.store.put(ObjectType::Checkpoint, &cp.encode())?;
66 
        batch.finish()?;
67 
        fs::create_dir_all(self.oxy.join("checkpoints"))?;
68 
        write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())?;
69 
        Ok((r, payload_len))
70 
    }
71
72 
    /// Store `state` as a version 2 payload (spec §11): the graphs of large
73 
    /// files as blobs of their own, shared with earlier checkpoints when
74 
    /// unchanged, and the index naming them. Returns the index blob and its size.
75 
    pub(crate) fn store_checkpoint_payload(&self, state: &State) -> Result<(ObjectRef, usize)> {
76 
        let graphs = state.checkpoint_file_graphs();
77 
        for (_, enc) in &graphs {
78 
            if enc.len() >= crate::state::INLINE_GRAPH_LIMIT {
79 
                self.store_blob(enc)?;
80 
            }
81 
        }
82 
        let index = state.checkpoint_index(&graphs, Repo::blob_ref);
83 
        Ok((self.store_blob(&index)?, index.len()))
84 
    }
85
86 
    /// The graph blobs a checkpoint's payload names (none for version 1).
87 
    pub(crate) fn checkpoint_graph_blobs(&self, cp: &Checkpoint) -> Result<Vec<ObjectRef>> {
88 
        let payload = self.read_blob(&cp.payload)?;
89 
        Ok(State::checkpoint_graph_refs(&payload)?)
90 
    }
91
92 
    /// Decode a checkpoint's payload, reading its graph blobs from the store.
93 
    pub(crate) fn state_from_checkpoint(&self, payload: &[u8]) -> Result<State> {
94 
        Ok(State::from_checkpoint(payload, |refs| {
95 
            self.read_blobs(refs).map_err(|e| {
96 
                crate::objects::ObjectError::Schema(format!("checkpoint graph blobs: {e}"))
97 
            })
98 
        })?)
99 
    }
100
101 
    /// Checkpoint the current bookmark's state now.
102 
    pub fn checkpoint(&self) -> Result<(ObjectRef, usize, usize)> {
103 
        let heads = self.bookmark_heads()?;
104 
        let state = self.load_state_full(&heads)?;
105 
        let (r, bytes) = self.write_checkpoint(&self.config.bookmark, &state, &heads)?;
106 
        Ok((r, state.len(), bytes))
107 
    }
108
109 
    /// Point `bookmark` at an already stored checkpoint (a shallow clone's base).
110 
    pub(crate) fn install_checkpoint(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
111 
        Checkpoint::decode(&self.store.get_typed(r, ObjectType::Checkpoint)?)?;
112 
        fs::create_dir_all(self.oxy.join("checkpoints"))?;
113 
        write_atomic(&self.checkpoint_pointer(bookmark), r.to_hex().as_bytes())
114 
    }
115
116 
    /// A bundle holding a checkpoint of exactly `heads` with its payload
117 
    /// (spec §16.1 `checkpoint`); the current bookmark checkpoint is reused
118 
    /// when it matches, otherwise one is made and, when `heads` are the
119 
    /// bookmark's, kept.
120 
    pub fn checkpoint_bundle_for(&self, heads: &[ChangeHash]) -> Result<Bundle> {
121 
        let wanted = StateObject::new(heads.to_vec()).heads;
122 
        let bookmark = self.config.bookmark.clone();
123 
        let current = self.bookmark_heads()?;
124 
        let r = match self.latest_checkpoint(&bookmark)? {
125 
            Some((r, cp)) if cp.heads == wanted => r,
126 
            _ => {
127 
                let state = self.load_state(&wanted)?;
128 
                if current == wanted {
129 
                    self.write_checkpoint(&bookmark, &state, &wanted)?.0
130 
                } else {
131 
                    let (blob, _) = self.store_checkpoint_payload(&state)?;
132 
                    let cp = Checkpoint {
133 
                        heads: wanted.clone(),
134 
                        count: state.len() as u64,
135 
                        payload: blob,
136 
                    };
137 
                    self.store.put(ObjectType::Checkpoint, &cp.encode())?
138 
                }
139 
            }
140 
        };
141 
        let cp = Checkpoint::decode(&self.store.get_typed(&r, ObjectType::Checkpoint)?)?;
142 
        let mut bundle = Bundle::default();
143 
        for c in Blob::decode(&self.store.get_typed(&cp.payload, ObjectType::Blob)?)?.chunks {
144 
            bundle.objects.push(self.store.get(&c)?);
145 
        }
146 
        bundle.objects.push(self.store.get(&cp.payload)?);
147 
        for g in self.checkpoint_graph_blobs(&cp)? {
148 
            for c in Blob::decode(&self.store.get_typed(&g, ObjectType::Blob)?)?.chunks {
149 
                bundle.objects.push(self.store.get(&c)?);
150 
            }
151 
            bundle.objects.push(self.store.get(&g)?);
152 
        }
153 
        bundle.objects.push((ObjectType::Checkpoint, cp.encode()));
154 
        Ok(bundle)
155 
    }
156
157 
    /// Load the state of `heads` using a checkpoint when one applies.
158 
    pub(crate) fn load_state_checkpointed(&self, heads: &[ChangeHash]) -> Result<Option<State>> {
159 
        let mut bookmarks: Vec<String> = vec![self.config.bookmark.clone()];
160 
        if let Ok(rd) = fs::read_dir(self.oxy.join("checkpoints")) {
161 
            for e in rd.flatten() {
162 
                let name = e.file_name().to_string_lossy().to_string();
163 
                if !bookmarks.contains(&name) {
164 
                    bookmarks.push(name);
165 
                }
166 
            }
167 
        }
168 
        for b in bookmarks {
169 
            let Some((_, cp)) = self.latest_checkpoint(&b)? else {
170 
                continue;
171 
            };
172 
            let payload = {
173 
                let _t = crate::timing::phase("load: read checkpoint");
174 
                self.read_blob(&cp.payload)?
175 
            };
176 
            let base = {
177 
                let _t = crate::timing::phase("load: decode checkpoint");
178 
                self.state_from_checkpoint(&payload)?
179 
            };
180 
            let cp_heads: BTreeSet<ChangeHash> = cp.heads.iter().copied().collect();
181 
            let mut found: BTreeSet<ChangeHash> = BTreeSet::new();
182 
            let mut delta: Vec<Change> = Vec::new();
183 
            let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
184 
            let mut stack: Vec<ChangeHash> = heads.to_vec();
185 
            while let Some(h) = stack.pop() {
186 
                if !seen.insert(h) {
187 
                    continue;
188 
                }
189 
                if base.contains(&h) {
190 
                    if cp_heads.contains(&h) {
191 
                        found.insert(h);
192 
                    }
193 
                    continue;
194 
                }
195 
                let c = self.store.get_change(&h)?;
196 
                stack.extend(c.deps.iter().copied());
197 
                delta.push(c);
198 
            }
199 
            if found.len() != cp_heads.len() {
200 
                continue;
201 
            }
202 
            let _t = crate::timing::phase("load: apply delta and validate");
203 
            let mut state = base;
204 
            for c in delta {
205 
                state.add_unchecked(c);
206 
            }
207 
            state.validate()?;
208 
            return Ok(Some(state));
209 
        }
210 
        Ok(None)
211 
    }
212 
}