OXYGENHUB
◆ wiki-says-so oxygenhub / crates / hub / src / maintenance.rs
4.0 KiB · Rust · last changed Reports: keep a job's report file by file, index its declaration, metrics and files 25 days ago source blame history raw
8a3334b7 Andy McCall ↶ 1 
//! Housekeeping that runs on its own (hub spec §17.2): the nightly
2 
//! garbage collection of every repository, one at a time under its
3 
//! write lock, so a push never races a repack. `oxygenhub gc` runs the
4 
//! same pass by hand.
5
6 
use crate::db;
7 
use crate::repos::RepoName;
8 
use crate::Shared;
9
10 
/// What one repository's gc did, for the log and `oxygenhub gc`.
11 
#[derive(Clone, Debug)]
12 
pub struct Done {
13 
    pub repo: String,
14 
    pub failed: bool,
15 
    pub summary: String,
16 
}
17
18 
impl std::fmt::Display for Done {
19 
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
20 
        write!(f, "{}: {}", self.repo, self.summary)
21 
    }
22 
}
23
24 
/// Garbage-collect every repository, in turn. A repository that fails is
25 
/// reported and the rest still run.
26 
pub async fn gc_all(app: &Shared) -> Vec<Done> {
27 
    let rows = match db::repos::list(&app.db).await {
28 
        Ok(rows) => rows,
29 
        Err(e) => {
30 
            tracing::warn!("gc: listing repositories: {e}");
31 
            return Vec::new();
32 
        }
33 
    };
34 
    let mut done = Vec::with_capacity(rows.len());
35 
    for row in rows {
36 
        let slug = format!("{}/{}", row.owner, row.name);
37 
        let Some(name) = RepoName::new(&row.owner, &row.name) else { continue };
38 
        if !app.repos.exists(&name) {
39 
            continue;
40 
        }
41 
        let label = slug.clone();
a36bab6b Andy McCall ↶ 42 
        // Flow's artifacts and reports' files are named only in the
43 
        // database: kept as roots, or gc would prune them after the grace
44 
        // period (#126, #127). A listing that fails skips this repository
45 
        // rather than risk them.
46 
        let listed = match db::flow::artifact_blobs(&app.db, row.id).await {
47 
            Ok(mut blobs) => crate::reports::file_blobs(&app.db, row.id).await.map(|files| {
48 
                blobs.extend(files);
49 
                blobs
50 
            }),
51 
            Err(e) => Err(e),
52 
        };
53 
        let keep: Vec<oxy_core::hash::ObjectRef> = match listed {
45977501 Andy McCall ↶ 54 
            Ok(blobs) => blobs.iter().filter_map(|b| oxy_core::hash::ObjectRef::from_hex(b)).collect(),
55 
            Err(e) => {
56 
                tracing::warn!("gc: {slug}: listing its artifacts: {e}; skipped");
57 
                continue;
58 
            }
59 
        };
8a3334b7 Andy McCall ↶ 60 
        let started = std::time::Instant::now();
45977501 Andy McCall ↶ 61 
        let result = app
62 
            .repos
63 
            .write(&name, move |repo, _| Ok(repo.gc_keeping(false, &keep, &mut |phase| tracing::debug!("gc {label}: {phase}"))?))
64 
            .await;
8a3334b7 Andy McCall ↶ 65 
        let d = match result {
66 
            Ok(r) => Done {
67 
                summary: format!(
68 
                    "{} reachable; pruned {} ({} bytes){}; packed {} into one pack in {:.1} s",
69 
                    r.reachable,
70 
                    r.pruned.objects,
71 
                    r.pruned.bytes,
72 
                    if r.pruned.deferred > 0 { format!(", {} deferred by the grace period", r.pruned.deferred) } else { String::new() },
73 
                    r.packed.objects,
74 
                    started.elapsed().as_secs_f64()
75 
                ),
76 
                repo: slug,
77 
                failed: false,
78 
            },
79 
            Err(e) => Done { summary: format!("failed: {e}"), repo: slug, failed: true },
80 
        };
81 
        if d.failed {
82 
            tracing::warn!("gc {}", d);
83 
        } else {
84 
            tracing::info!("gc {}", d);
85 
        }
86 
        done.push(d);
87 
    }
88 
    done
89 
}
90
91 
/// Run [`gc_all`] once a day at `retention.gc_hour` UTC.
92 
pub async fn nightly(app: Shared) {
93 
    let hour = app.config.retention.gc_hour;
94 
    if hour >= 24 {
95 
        tracing::info!("gc: the nightly run is off (retention.gc_hour = {hour})");
96 
        return;
97 
    }
98 
    let mut last_day = None;
99 
    loop {
100 
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
101 
        let now = time::OffsetDateTime::now_utc();
102 
        if u32::from(now.hour()) != hour {
103 
            continue;
104 
        }
105 
        let day = now.date();
106 
        if last_day == Some(day) {
107 
            continue;
108 
        }
109 
        last_day = Some(day);
110 
        let done = gc_all(&app).await;
111 
        tracing::info!("gc: nightly run over {} repositories, {} failed", done.len(), done.iter().filter(|d| d.failed).count());
112 
    }
113 
}