gc.rs 91.7%

111 of 121 lines run, 5 of 10 functions · run #254 on main · run missed

//! Garbage collection (spec §21): delete objects
//! nothing reachable refers to.
//!
//! Roots: the current view, the views of every operation newer than
//! `core.op-retention`, reference attestations, and the policy; receipts
//! live as long as the changes of their state. Operation
//! objects are always kept so the log stays walkable, but an operation older
//! than the retention window no longer keeps its states alive, so undoing
//! that far back may fail once its objects are gone.

use super::*;
use crate::store::{PackReport, PruneReport};
use std::collections::HashSet;

#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct GcReport {
    pub packed: PackReport,
    pub pruned: PruneReport,
    pub reachable: usize,
    /// Packs folded into the one a full repack writes.
    pub merged_packs: usize,
}

impl Repo {
    /// Prune unreachable objects older than the grace period (`now`
    /// ignores it), then repack everything into one pack (spec §13.4).
    pub fn gc(&self, now: bool) -> Result<GcReport> {
        self.gc_with(now, &mut |_| {})
    }

    /// [`Repo::gc`], reporting each phase and the packing's progress
    /// (oxygen#66), for a spinner or a log.
    pub fn gc_with(&self, now: bool, progress: crate::store::Progress) -> Result<GcReport> {
        progress("finding what is reachable");
        let keep = self.reachable()?;
        let grace = if now {
            std::time::Duration::ZERO
        } else {
            std::time::Duration::from_secs(self.config.gc_grace_days * 86_400)
        };
        let pruned = self.store.prune_with(&keep, grace, progress)?;
        // Then everything left in one pack, compressed harder than a
        // commit's (spec §13.4), as `git gc` repacks.
        let (packed, merged_packs) = self
            .store
            .repack_all_with(crate::store::REPACK_ZSTD_LEVEL, progress)?;
        Ok(GcReport {
            packed,
            pruned,
            reachable: keep.len(),
            merged_packs,
        })
    }

    /// Every object that must survive garbage collection.
    pub fn reachable(&self) -> Result<HashSet<ObjectRef>> {
        let mut keep: HashSet<ObjectRef> = HashSet::new();
        let mut states: Vec<ObjectRef> = Vec::new();
        let mut changes: Vec<ChangeHash> = Vec::new();
        let mut attests: Vec<ObjectRef> = Vec::new();
        let mut policies: Vec<ObjectRef> = Vec::new();

        // Operations: always kept; views within retention are roots.
        let cutoff = Hlc::now()
            .wall_ms
            .saturating_sub(self.config.op_retention_days * 86_400_000);
        for (i, (r, op)) in self.op_log()?.into_iter().enumerate() {
            keep.insert(r);
            if i == 0 || op.hlc.wall_ms >= cutoff {
                states.extend(op.view.bookmarks.values().copied());
                states.extend(op.view.remotes.values().flat_map(|b| b.values().copied()));
                changes.extend(op.view.workspaces.values().copied());
                policies.extend(op.view.policy);
            }
        }
        // Checkpoints: the object, its payload blob, and the blob's chunks.
        for r in self.checkpoint_refs()? {
            if let Ok(bytes) = self.store.get_typed(&r, ObjectType::Checkpoint) {
                keep.insert(r);
                let cp = crate::objects::Checkpoint::decode(&bytes)?;
                keep.insert(cp.payload);
                if let Ok(b) = self.store.get_typed(&cp.payload, ObjectType::Blob) {
                    keep.extend(Blob::decode(&b)?.chunks);
                }
                // A version 2 payload's graph blobs and their chunks.
                for g in self.checkpoint_graph_blobs(&cp).unwrap_or_default() {
                    keep.insert(g);
                    if let Ok(b) = self.store.get_typed(&g, ObjectType::Blob) {
                        keep.extend(Blob::decode(&b)?.chunks);
                    }
                }
            }
        }
        // Tags: their states and attestations.
        let view_now = self.view()?;
        states.extend(view_now.tags.values().copied());
        for name in view_now.tags.keys() {
            if let Some((r, _)) = self.tag_attest(name)? {
                attests.push(r);
            }
        }
        // Reference attestations.
        for name in self.view()?.bookmarks.keys() {
            if let Some((r, _)) = self.latest_ref_attest(name)? {
                attests.push(r);
            }
        }
        // A historical view may name objects pruned under an earlier, shorter
        // retention; there is nothing to keep for those.
        for st in states {
            if keep.insert(st) {
                match self.store.get_state(&st) {
                    Ok(s) => changes.extend(s.heads),
                    Err(StoreError::NotFound(_)) => continue,
                    Err(e) => return Err(e.into()),
                }
            }
        }
        for p in policies {
            if keep.insert(p) {
                let bytes = match self.store.get_typed(&p, ObjectType::Policy) {
                    Ok(b) => b,
                    Err(StoreError::NotFound(_)) => continue,
                    Err(e) => return Err(e.into()),
                };
                let policy = crate::policy::Policy::decode(&bytes)?;
                for id in &policy.identities {
                    keep.insert(id.hash());
                }
                for sig in self.store.signatures(&p)? {
                    keep.insert(sig.key);
                }
            }
        }
        // Changes: their closure, identities, blobs, chunks, seals.
        let mut stack = changes;
        while let Some(h) = stack.pop() {
            if !keep.insert(h) {
                continue;
            }
            // Beyond the shallow boundary, or pruned under an earlier
            // retention, there is nothing to keep.
            let c = match self.store.get_change_parts(&h) {
                Ok((c, texts)) => {
                    keep.extend(texts);
                    c
                }
                Err(StoreError::NotFound(_)) => continue,
                Err(e) => return Err(e.into()),
            };
            stack.extend(c.deps.iter().copied());
            if !c.author.is_zero() {
                keep.insert(c.author);
            }
            for sig in self.store.signatures(&h)? {
                keep.insert(sig.key);
            }
            for e in &c.edits {
                if let Edit::Replace { old, new, .. } = e {
                    for b in [old, new].into_iter().flatten() {
                        if keep.insert(*b) {
                            if let Ok(bytes) = self.store.get_typed(b, ObjectType::Blob) {
                                keep.extend(Blob::decode(&bytes)?.chunks);
                            }
                        }
                    }
                }
            }
            attests.extend(self.store.seals_for(&h)?);
        }
        self.reachable_receipts(&mut keep)?;
        for a in attests {
            if keep.insert(a) {
                for sig in self.store.signatures(&a)? {
                    keep.insert(sig.key);
                }
            }
        }
        Ok(keep)
    }
}

1		//! Garbage collection (spec §21): delete objects
2		//! nothing reachable refers to.
3		//!
4		//! Roots: the current view, the views of every operation newer than
5		//! `core.op-retention`, reference attestations, and the policy; receipts
6		//! live as long as the changes of their state. Operation
7		//! objects are always kept so the log stays walkable, but an operation older
8		//! than the retention window no longer keeps its states alive, so undoing
9		//! that far back may fail once its objects are gone.
10
11		use super::*;
12		use crate::store::{PackReport, PruneReport};
13		use std::collections::HashSet;
14
15		#[derive(Debug, Clone, Default, PartialEq, Eq)]
16		pub struct GcReport {
17		pub packed: PackReport,
18		pub pruned: PruneReport,
19		pub reachable: usize,
20		/// Packs folded into the one a full repack writes.
21		pub merged_packs: usize,
22		}
23
24		impl Repo {
25		/// Prune unreachable objects older than the grace period (`now`
26		/// ignores it), then repack everything into one pack (spec §13.4).
27	11×	pub fn gc(&self, now: bool) -> Result<GcReport> {
28	59×	self.gc_with(now, &mut \|_\| {})
29	11×	}
30
31		/// [`Repo::gc`], reporting each phase and the packing's progress
32		/// (oxygen#66), for a spinner or a log.
33	12×	pub fn gc_with(&self, now: bool, progress: crate::store::Progress) -> Result<GcReport> {
34	12×	progress("finding what is reachable");
35	12×	let keep = self.reachable()?;
36	12×	let grace = if now {
37	10×	std::time::Duration::ZERO
38		} else {
39	2×	std::time::Duration::from_secs(self.config.gc_grace_days * 86_400)
40		};
41	12×	let pruned = self.store.prune_with(&keep, grace, progress)?;
42		// Then everything left in one pack, compressed harder than a
43		// commit's (spec §13.4), as `git gc` repacks.
44	12×	let (packed, merged_packs) = self
45	12×	.store
46	12×	.repack_all_with(crate::store::REPACK_ZSTD_LEVEL, progress)?;
47	12×	Ok(GcReport {
48	12×	packed,
49	12×	pruned,
50	12×	reachable: keep.len(),
51	12×	merged_packs,
52	12×	})
53	12×	}
54
55		/// Every object that must survive garbage collection.
56	12×	pub fn reachable(&self) -> Result<HashSet<ObjectRef>> {
57	12×	let mut keep: HashSet<ObjectRef> = HashSet::new();
58	12×	let mut states: Vec<ObjectRef> = Vec::new();
59	12×	let mut changes: Vec<ChangeHash> = Vec::new();
60	12×	let mut attests: Vec<ObjectRef> = Vec::new();
61	12×	let mut policies: Vec<ObjectRef> = Vec::new();
62
63		// Operations: always kept; views within retention are roots.
64	12×	let cutoff = Hlc::now()
65	12×	.wall_ms
66	12×	.saturating_sub(self.config.op_retention_days * 86_400_000);
67	104×	for (i, (r, op)) in self.op_log()?.into_iter().enumerate() {
68	104×	keep.insert(r);
69	104×	if i == 0 \|\| op.hlc.wall_ms >= cutoff {
70	66×	states.extend(op.view.bookmarks.values().copied());
71	66×	states.extend(op.view.remotes.values().flat_map(\|b\| b.values().copied()));
72	66×	changes.extend(op.view.workspaces.values().copied());
73	66×	policies.extend(op.view.policy);
74	38×	}
75		}
76		// Checkpoints: the object, its payload blob, and the blob's chunks.
77	12×	for r in self.checkpoint_refs()? {
78	2×	if let Ok(bytes) = self.store.get_typed(&r, ObjectType::Checkpoint) {
79	2×	keep.insert(r);
80	2×	let cp = crate::objects::Checkpoint::decode(&bytes)?;
81	2×	keep.insert(cp.payload);
82	2×	if let Ok(b) = self.store.get_typed(&cp.payload, ObjectType::Blob) {
83	2×	keep.extend(Blob::decode(&b)?.chunks);
84	0×	}
85		// A version 2 payload's graph blobs and their chunks.
86	2×	for g in self.checkpoint_graph_blobs(&cp).unwrap_or_default() {
87	1×	keep.insert(g);
88	1×	if let Ok(b) = self.store.get_typed(&g, ObjectType::Blob) {
89	1×	keep.extend(Blob::decode(&b)?.chunks);
90	0×	}
91		}
92	0×	}
93		}
94		// Tags: their states and attestations.
95	12×	let view_now = self.view()?;
96	12×	states.extend(view_now.tags.values().copied());
97	12×	for name in view_now.tags.keys() {
98	3×	if let Some((r, _)) = self.tag_attest(name)? {
99	3×	attests.push(r);
100	3×	}
101		}
102		// Reference attestations.
103	12×	for name in self.view()?.bookmarks.keys() {
104	12×	if let Some((r, _)) = self.latest_ref_attest(name)? {
105	0×	attests.push(r);
106	12×	}
107		}
108		// A historical view may name objects pruned under an earlier, shorter
109		// retention; there is nothing to keep for those.
110	85×	for st in states {
111	85×	if keep.insert(st) {
112	39×	match self.store.get_state(&st) {
113	36×	Ok(s) => changes.extend(s.heads),
114	3×	Err(StoreError::NotFound(_)) => continue,
115	0×	Err(e) => return Err(e.into()),
116		}
117	46×	}
118		}
119	12×	for p in policies {
120	4×	if keep.insert(p) {
121	1×	let bytes = match self.store.get_typed(&p, ObjectType::Policy) {
122	1×	Ok(b) => b,
123	0×	Err(StoreError::NotFound(_)) => continue,
124	0×	Err(e) => return Err(e.into()),
125		};
126	1×	let policy = crate::policy::Policy::decode(&bytes)?;
127	1×	for id in &policy.identities {
128	1×	keep.insert(id.hash());
129	1×	}
130	1×	for sig in self.store.signatures(&p)? {
131	1×	keep.insert(sig.key);
132	1×	}
133	3×	}
134		}
135		// Changes: their closure, identities, blobs, chunks, seals.
136	12×	let mut stack = changes;
137	218×	while let Some(h) = stack.pop() {
138	206×	if !keep.insert(h) {
139	107×	continue;
140	99×	}
141		// Beyond the shallow boundary, or pruned under an earlier
142		// retention, there is nothing to keep.
143	99×	let c = match self.store.get_change_parts(&h) {
144	89×	Ok((c, texts)) => {
145	89×	keep.extend(texts);
146	89×	c
147		}
148	10×	Err(StoreError::NotFound(_)) => continue,
149	0×	Err(e) => return Err(e.into()),
150		};
151	89×	stack.extend(c.deps.iter().copied());
152	89×	if !c.author.is_zero() {
153	30×	keep.insert(c.author);
154	59×	}
155	89×	for sig in self.store.signatures(&h)? {
156	13×	keep.insert(sig.key);
157	13×	}
158	106×	for e in &c.edits {
159	106×	if let Edit::Replace { old, new, .. } = e {
160	17×	for b in [old, new].into_iter().flatten() {
161	17×	if keep.insert(*b) {
162	17×	if let Ok(bytes) = self.store.get_typed(b, ObjectType::Blob) {
163	17×	keep.extend(Blob::decode(&bytes)?.chunks);
164	0×	}
165	0×	}
166		}
167	89×	}
168		}
169	89×	attests.extend(self.store.seals_for(&h)?);
170		}
171	12×	self.reachable_receipts(&mut keep)?;
172	12×	for a in attests {
173	5×	if keep.insert(a) {
174	4×	for sig in self.store.signatures(&a)? {
175	4×	keep.insert(sig.key);
176	4×	}
177	1×	}
178		}
179	12×	Ok(keep)
180	12×	}
181		}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

Coverage / crates/oxy-core/src/repo/gc.rs 91.7%

Keyboard