OXYGENHUB
◆ v0.4.1 oxygen / crates / oxy-core / src / repo / gc.rs
7.1 KiB · Rust · last changed oxy gc says what it is doing 1 month ago source blame history raw
da652d63 Andy McCall ↶ 1
//! Garbage collection (spec §21): delete objects
ba160f60 Andy McCall ↶ 2
//! nothing reachable refers to.
3
//!
4
//! Roots: the current view, the views of every operation newer than
fc1a466a Andy McCall ↶ 5
//! `core.op-retention`, reference attestations, and the policy; receipts
6
//! live as long as the changes of their state. Operation
ba160f60 Andy McCall ↶ 7
//! objects are always kept so the log stays walkable, but an operation older
8
//! than the retention window no longer keeps its states alive, so undoing
9
//! that far back may fail once its objects are gone.
10
11
use super::*;
12
use crate::store::{PackReport, PruneReport};
13
use std::collections::HashSet;
14
15
#[derive(Debug, Clone, Default, PartialEq, Eq)]
16
pub struct GcReport {
17
    pub packed: PackReport,
18
    pub pruned: PruneReport,
19
    pub reachable: usize,
da652d63 Andy McCall ↶ 20
    /// Packs folded into the one a full repack writes.
135212de Andy McCall ↶ 21
    pub merged_packs: usize,
ba160f60 Andy McCall ↶ 22
}
23
24
impl Repo {
da652d63 Andy McCall ↶ 25
    /// Prune unreachable objects older than the grace period (`now`
26
    /// ignores it), then repack everything into one pack (spec §13.4).
ba160f60 Andy McCall ↶ 27
    pub fn gc(&self, now: bool) -> Result<GcReport> {
9ae4d15a Andy McCall ↶ 28
        self.gc_with(now, &mut |_| {})
29
    }
30
31
    /// [`Repo::gc`], reporting each phase and the packing's progress
32
    /// (oxygen#66), for a spinner or a log.
33
    pub fn gc_with(&self, now: bool, progress: crate::store::Progress) -> Result<GcReport> {
34
        progress("finding what is reachable");
ba160f60 Andy McCall ↶ 35
        let keep = self.reachable()?;
75167672 Andy McCall 36
        let grace = if now {
37
            std::time::Duration::ZERO
38
        } else {
39
            std::time::Duration::from_secs(self.config.gc_grace_days * 86_400)
40
        };
9ae4d15a Andy McCall ↶ 41
        let pruned = self.store.prune_with(&keep, grace, progress)?;
da652d63 Andy McCall ↶ 42
        // Then everything left in one pack, compressed harder than a
43
        // commit's (spec §13.4), as `git gc` repacks.
9ae4d15a Andy McCall ↶ 44
        let (packed, merged_packs) = self
45
            .store
46
            .repack_all_with(crate::store::REPACK_ZSTD_LEVEL, progress)?;
75167672 Andy McCall 47
        Ok(GcReport {
48
            packed,
49
            pruned,
50
            reachable: keep.len(),
51
            merged_packs,
52
        })
ba160f60 Andy McCall ↶ 53
    }
54
55
    /// Every object that must survive garbage collection.
56
    pub fn reachable(&self) -> Result<HashSet<ObjectRef>> {
57
        let mut keep: HashSet<ObjectRef> = HashSet::new();
58
        let mut states: Vec<ObjectRef> = Vec::new();
59
        let mut changes: Vec<ChangeHash> = Vec::new();
60
        let mut attests: Vec<ObjectRef> = Vec::new();
61
        let mut policies: Vec<ObjectRef> = Vec::new();
62
63
        // Operations: always kept; views within retention are roots.
75167672 Andy McCall 64
        let cutoff = Hlc::now()
65
            .wall_ms
66
            .saturating_sub(self.config.op_retention_days * 86_400_000);
ba160f60 Andy McCall ↶ 67
        for (i, (r, op)) in self.op_log()?.into_iter().enumerate() {
68
            keep.insert(r);
69
            if i == 0 || op.hlc.wall_ms >= cutoff {
70
                states.extend(op.view.bookmarks.values().copied());
71
                states.extend(op.view.remotes.values().flat_map(|b| b.values().copied()));
72
                changes.extend(op.view.workspaces.values().copied());
73
                policies.extend(op.view.policy);
74
            }
75
        }
252e8cb2 Andy McCall ↶ 76
        // Checkpoints: the object, its payload blob, and the blob's chunks.
77
        for r in self.checkpoint_refs()? {
78
            if let Ok(bytes) = self.store.get_typed(&r, ObjectType::Checkpoint) {
79
                keep.insert(r);
80
                let cp = crate::objects::Checkpoint::decode(&bytes)?;
81
                keep.insert(cp.payload);
82
                if let Ok(b) = self.store.get_typed(&cp.payload, ObjectType::Blob) {
83
                    keep.extend(Blob::decode(&b)?.chunks);
84
                }
afaba5f1 Andy McCall ↶ 85
                // A version 2 payload's graph blobs and their chunks.
86
                for g in self.checkpoint_graph_blobs(&cp).unwrap_or_default() {
87
                    keep.insert(g);
88
                    if let Ok(b) = self.store.get_typed(&g, ObjectType::Blob) {
89
                        keep.extend(Blob::decode(&b)?.chunks);
90
                    }
91
                }
252e8cb2 Andy McCall ↶ 92
            }
93
        }
132b884b Andy McCall ↶ 94
        // Tags: their states and attestations.
95
        let view_now = self.view()?;
96
        states.extend(view_now.tags.values().copied());
97
        for name in view_now.tags.keys() {
98
            if let Some((r, _)) = self.tag_attest(name)? {
99
                attests.push(r);
100
            }
101
        }
ba160f60 Andy McCall ↶ 102
        // Reference attestations.
103
        for name in self.view()?.bookmarks.keys() {
104
            if let Some((r, _)) = self.latest_ref_attest(name)? {
105
                attests.push(r);
106
            }
107
        }
21b2aec2 Andy McCall ↶ 108
        // A historical view may name objects pruned under an earlier, shorter
109
        // retention; there is nothing to keep for those.
ba160f60 Andy McCall ↶ 110
        for st in states {
111
            if keep.insert(st) {
21b2aec2 Andy McCall ↶ 112
                match self.store.get_state(&st) {
113
                    Ok(s) => changes.extend(s.heads),
114
                    Err(StoreError::NotFound(_)) => continue,
115
                    Err(e) => return Err(e.into()),
116
                }
ba160f60 Andy McCall ↶ 117
            }
118
        }
119
        for p in policies {
120
            if keep.insert(p) {
21b2aec2 Andy McCall ↶ 121
                let bytes = match self.store.get_typed(&p, ObjectType::Policy) {
122
                    Ok(b) => b,
123
                    Err(StoreError::NotFound(_)) => continue,
124
                    Err(e) => return Err(e.into()),
125
                };
126
                let policy = crate::policy::Policy::decode(&bytes)?;
ba160f60 Andy McCall ↶ 127
                for id in &policy.identities {
128
                    keep.insert(id.hash());
129
                }
130
                for sig in self.store.signatures(&p)? {
131
                    keep.insert(sig.key);
132
                }
133
            }
134
        }
135
        // Changes: their closure, identities, blobs, chunks, seals.
136
        let mut stack = changes;
137
        while let Some(h) = stack.pop() {
138
            if !keep.insert(h) {
139
                continue;
140
            }
21b2aec2 Andy McCall ↶ 141
            // Beyond the shallow boundary, or pruned under an earlier
142
            // retention, there is nothing to keep.
6f8aeb99 Andy McCall ↶ 143
            let c = match self.store.get_change_parts(&h) {
144
                Ok((c, texts)) => {
145
                    keep.extend(texts);
146
                    c
147
                }
2863456a Andy McCall ↶ 148
                Err(StoreError::NotFound(_)) => continue,
149
                Err(e) => return Err(e.into()),
150
            };
ba160f60 Andy McCall ↶ 151
            stack.extend(c.deps.iter().copied());
152
            if !c.author.is_zero() {
153
                keep.insert(c.author);
154
            }
155
            for sig in self.store.signatures(&h)? {
156
                keep.insert(sig.key);
157
            }
158
            for e in &c.edits {
159
                if let Edit::Replace { old, new, .. } = e {
160
                    for b in [old, new].into_iter().flatten() {
161
                        if keep.insert(*b) {
162
                            if let Ok(bytes) = self.store.get_typed(b, ObjectType::Blob) {
163
                                keep.extend(Blob::decode(&bytes)?.chunks);
164
                            }
165
                        }
166
                    }
167
                }
168
            }
169
            attests.extend(self.store.seals_for(&h)?);
170
        }
fc1a466a Andy McCall ↶ 171
        self.reachable_receipts(&mut keep)?;
ba160f60 Andy McCall ↶ 172
        for a in attests {
173
            if keep.insert(a) {
174
                for sig in self.store.signatures(&a)? {
175
                    keep.insert(sig.key);
176
                }
177
            }
178
        }
179
        Ok(keep)
180
    }
181
}