OXYGENHUB
# 1e8a6e8bb91b oxygenhub / crates / hub / src / backups.rs
57.3 KiB · Rust · last changed Admin › Backups: whether each owner's backup ran, and who has none 23 days ago source blame history raw
1 
//! Backups (docs/backups.md, oxygenhub#169): each user and organisation
2 
//! copies its repositories, on a schedule, to a destination it controls.
3 
//!
4 
//! A repository is its store directory (`.oxy/`): packs and loose objects
5 
//! that never change once written, and small files that do (refs, the
6 
//! operation log, policies, signature indexes). A run copies each file
7 
//! once, named by its BLAKE3 (`blobs/`), and writes a manifest of path to
8 
//! hash per repository (`runs/<at>/repos/`), so only what changed travels
9 
//! and any kept run can be restored byte for byte, signatures and all.
10 
//! The large files are copied first; the walk is then repeated under the
11 
//! repository's read lock, which pushes wait on, so each manifest is one
12 
//! consistent moment. Beside each manifest go the hub's records for the
13 
//! repository as JSON (never a secret value), the owner's records, and
14 
//! last a `DONE` marker: a run without one was cut short and is ignored.
15
16 
use crate::db::{self, Pool};
17 
use crate::repos::RepoName;
18 
use crate::Shared;
19 
use anyhow::{bail, Context, Result};
20 
use serde::{Deserialize, Serialize};
21 
use std::collections::HashMap;
22 
use std::path::{Path, PathBuf};
23
24 
/// How often, as the settings offer it.
25 
pub const EVERY: &[(&str, &str)] = &[("off", "Off"), ("hourly", "Every hour"), ("daily", "Every day")];
26 
/// How long runs are kept, in days.
27 
pub const KEEP: &[i64] = &[7, 30, 90];
28
29 
/// What an owner backs up, where and how often.
30 
#[derive(Clone, Debug, sqlx::FromRow, Serialize)]
31 
pub struct Setting {
32 
    pub owner: String,
33 
    pub kind: String,
34 
    pub every: String,
35 
    pub destination: String,
36 
    pub target: String,
37 
    pub prefix: String,
38 
    pub provider: String,
39 
    pub keep_days: i64,
40 
    pub updated_ms: i64,
41 
}
42
43 
impl Setting {
44 
    pub fn off(owner: &str, kind: &str) -> Setting {
45 
        Setting {
46 
            owner: owner.into(),
47 
            kind: kind.into(),
48 
            every: "off".into(),
49 
            destination: "dir".into(),
50 
            target: String::new(),
51 
            prefix: String::new(),
52 
            provider: String::new(),
53 
            keep_days: 30,
54 
            updated_ms: 0,
55 
        }
56 
    }
57
58 
    /// Seconds between runs; none when off.
59 
    pub fn period_s(&self) -> Option<i64> {
60 
        match self.every.as_str() {
61 
            "hourly" => Some(3600),
62 
            "daily" => Some(86_400),
63 
            _ => None,
64 
        }
65 
    }
66 
}
67
68 
/// One run, as the settings page and the API show it.
69 
#[derive(Clone, Debug, sqlx::FromRow, Serialize)]
70 
pub struct Run {
71 
    pub id: i64,
72 
    pub owner: String,
73 
    pub started_ms: i64,
74 
    pub finished_ms: Option<i64>,
75 
    pub status: String,
76 
    pub at: String,
77 
    pub repos: i64,
78 
    pub files: i64,
79 
    pub sent: i64,
80 
    pub bytes: i64,
81 
    pub detail: String,
82 
    pub trigger: String,
83 
}
84
85 
pub async fn setting(pool: &Pool, owner: &str) -> sqlx::Result<Option<Setting>> {
86 
    sqlx::query_as("SELECT * FROM backup_settings WHERE owner = ?").bind(owner).fetch_optional(pool).await
87 
}
88
89 
pub async fn save(pool: &Pool, s: &Setting) -> sqlx::Result<()> {
90 
    sqlx::query(
91 
        "INSERT INTO backup_settings (owner, kind, every, destination, target, prefix, provider, keep_days, updated_ms) \
92 
         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(owner) DO UPDATE SET kind = excluded.kind, every = excluded.every, \
93 
         destination = excluded.destination, target = excluded.target, prefix = excluded.prefix, provider = excluded.provider, \
94 
         keep_days = excluded.keep_days, updated_ms = excluded.updated_ms",
95 
    )
96 
    .bind(&s.owner)
97 
    .bind(&s.kind)
98 
    .bind(&s.every)
99 
    .bind(&s.destination)
100 
    .bind(&s.target)
101 
    .bind(&s.prefix)
102 
    .bind(&s.provider)
103 
    .bind(s.keep_days)
104 
    .bind(db::now_ms())
105 
    .execute(pool)
106 
    .await?;
107 
    Ok(())
108 
}
109
110 
pub async fn runs(pool: &Pool, owner: &str, limit: i64) -> sqlx::Result<Vec<Run>> {
111 
    sqlx::query_as("SELECT * FROM backup_runs WHERE owner = ? ORDER BY id DESC LIMIT ?").bind(owner).bind(limit).fetch_all(pool).await
112 
}
113
114 
/// One owner's backup as Admin › Backups shows it (§5): its setting, its
115 
/// newest run, and how many runs failed in the last day. An administrator
116 
/// sees that it ran, never what it wrote.
117 
#[derive(Clone, Debug, Serialize)]
118 
pub struct Overview {
119 
    pub setting: Setting,
120 
    pub last: Option<Run>,
121 
    pub failed_day: i64,
122 
    /// The repositories it covers now.
123 
    pub repos: i64,
124 
}
125
126 
impl Overview {
127 
    /// Where its runs go, as a person reads it.
128 
    pub fn place(&self) -> String {
129 
        let s = &self.setting;
130 
        match s.destination.as_str() {
131 
            "gcs" if s.prefix.is_empty() => format!("gs://{}", s.target),
132 
            "gcs" => format!("gs://{}/{}", s.target, s.prefix),
133 
            _ => format!("{}/{}", s.target, s.owner),
134 
        }
135 
    }
136 
}
137
138 
/// Every owner's backup (Admin › Backups), and the owners of repositories
139 
/// that none covers, with how many each has.
140 
pub async fn overview(pool: &Pool) -> sqlx::Result<(Vec<Overview>, Vec<(String, i64)>)> {
141 
    let settings: Vec<Setting> = sqlx::query_as("SELECT * FROM backup_settings ORDER BY owner").fetch_all(pool).await?;
142 
    let counts: Vec<(String, i64)> =
143 
        sqlx::query_as("SELECT owner, COUNT(*) FROM repositories GROUP BY owner ORDER BY owner").fetch_all(pool).await?;
144 
    let day_ago = db::now_ms() - 86_400_000;
145 
    let mut out = Vec::new();
146 
    for s in settings {
147 
        let last = runs(pool, &s.owner, 1).await?.into_iter().next();
148 
        let failed_day: i64 =
149 
            sqlx::query_scalar("SELECT COUNT(*) FROM backup_runs WHERE owner = ? AND status = 'failed' AND started_ms >= ?")
150 
                .bind(&s.owner)
151 
                .bind(day_ago)
152 
                .fetch_one(pool)
153 
                .await?;
154 
        let repos = counts.iter().find(|(o, _)| *o == s.owner).map_or(0, |c| c.1);
155 
        out.push(Overview { setting: s, last, failed_day, repos });
156 
    }
157 
    let uncovered =
158 
        counts.into_iter().filter(|(owner, _)| !out.iter().any(|o| o.setting.owner == *owner && o.setting.every != "off")).collect();
159 
    Ok((out, uncovered))
160 
}
161
162 
// ---------------------------------------------------------------- where
163
164 
/// Where backups are written: keys are `/`-separated paths.
165 
pub trait Destination: Send + Sync {
166 
    fn exists(&self, key: &str) -> Result<bool>;
167 
    /// Write `bytes` at `key`, whole or not at all.
168 
    fn put(&self, key: &str, bytes: &[u8]) -> Result<()>;
169 
    fn get(&self, key: &str) -> Result<Vec<u8>>;
170 
    /// The names directly under `prefix` (files and "directories").
171 
    fn list(&self, prefix: &str) -> Result<Vec<String>>;
172 
    /// Remove `key` and everything under it.
173 
    fn remove(&self, key: &str) -> Result<()>;
174 
}
175
176 
/// A directory on the hub's host.
177 
pub struct Directory {
178 
    pub root: PathBuf,
179 
}
180
181 
impl Directory {
182 
    fn path(&self, key: &str) -> Result<PathBuf> {
183 
        if key.split('/').any(|p| p == ".." || p == ".") || key.starts_with('/') {
184 
            bail!("not a clean key: {key}");
185 
        }
186 
        Ok(self.root.join(key))
187 
    }
188 
}
189
190 
impl Destination for Directory {
191 
    fn exists(&self, key: &str) -> Result<bool> {
192 
        Ok(self.path(key)?.exists())
193 
    }
194 
    fn put(&self, key: &str, bytes: &[u8]) -> Result<()> {
195 
        let p = self.path(key)?;
196 
        std::fs::create_dir_all(p.parent().context("no parent")?)?;
197 
        let tmp = p.with_extension(format!("part-{}", std::process::id()));
198 
        std::fs::write(&tmp, bytes)?;
199 
        std::fs::rename(&tmp, &p)?;
200 
        Ok(())
201 
    }
202 
    fn get(&self, key: &str) -> Result<Vec<u8>> {
203 
        let p = self.path(key)?;
204 
        std::fs::read(&p).with_context(|| format!("read {}", p.display()))
205 
    }
206 
    fn list(&self, prefix: &str) -> Result<Vec<String>> {
207 
        let p = self.path(prefix)?;
208 
        let mut out: Vec<String> = match std::fs::read_dir(&p) {
209 
            Ok(d) => d.flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect(),
210 
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
211 
            Err(e) => return Err(e.into()),
212 
        };
213 
        out.sort();
214 
        Ok(out)
215 
    }
216 
    fn remove(&self, key: &str) -> Result<()> {
217 
        let p = self.path(key)?;
218 
        if p.is_dir() {
219 
            std::fs::remove_dir_all(p)?;
220 
        } else if p.exists() {
221 
            std::fs::remove_file(p)?;
222 
        }
223 
        Ok(())
224 
    }
225 
}
226
227 
/// A Cloud Storage bucket, reached keylessly (docs/backups.md §3, §6):
228 
/// the hub mints a short token as `backup:OWNER`, Google's STS exchanges
229 
/// it for an access token through the owner's workload identity
230 
/// provider, and that writes the bucket. No key exists anywhere; the
231 
/// access token lasts an hour and is fetched again before it runs out.
232 
pub struct Gcs {
233 
    bucket: String,
234 
    /// Keys go under this, when set.
235 
    prefix: String,
236 
    storage: String,
237 
    sts: String,
238 
    /// The provider's resource name, `//iam.googleapis.com/projects/…`.
239 
    provider: String,
240 
    subject: String,
241 
    app: Shared,
242 
    client: reqwest::Client,
243 
    rt: tokio::runtime::Handle,
244 
    token: std::sync::Mutex<Option<(String, i64)>>,
245 
}
246
247 
impl Gcs {
248 
    pub fn new(app: &Shared, s: &Setting) -> Gcs {
249 
        Gcs {
250 
            bucket: s.target.clone(),
251 
            prefix: s.prefix.trim_matches('/').to_string(),
252 
            storage: app.config.backups.storage_url.trim_end_matches('/').to_string(),
253 
            sts: app.config.backups.sts_url.trim_end_matches('/').to_string(),
254 
            provider: s.provider.clone(),
255 
            subject: format!("backup:{}", s.owner),
256 
            app: app.clone(),
257 
            client: reqwest::Client::builder().timeout(std::time::Duration::from_secs(300)).build().unwrap_or_default(),
258 
            rt: tokio::runtime::Handle::current(),
259 
            token: std::sync::Mutex::new(None),
260 
        }
261 
    }
262
263 
    fn name(&self, key: &str) -> String {
264 
        if self.prefix.is_empty() {
265 
            key.to_string()
266 
        } else {
267 
            format!("{}/{key}", self.prefix)
268 
        }
269 
    }
270
271 
    /// An access token, exchanged for a fresh one of the hub's when the
272 
    /// last is within five minutes of running out.
273 
    fn access(&self) -> Result<String> {
274 
        let now = db::now_ms();
275 
        if let Some((t, until)) = self.token.lock().unwrap().as_ref() {
276 
            if *until - now > 300_000 {
277 
                return Ok(t.clone());
278 
            }
279 
        }
280 
        let (token, expires_in) = self.rt.block_on(async {
281 
            let issuer = self.app.oidc().await?;
282 
            // Google's default audience for a provider: its name over https.
283 
            let audience = format!("https:{}", self.provider);
284 
            let jwt = issuer.mint_for(self.app.config.base_url(), &self.subject, &audience, db::now_ms(), 600);
285 
            let res = self
286 
                .client
287 
                .post(format!("{}/v1/token", self.sts))
288 
                .json(&serde_json::json!({
289 
                    "grantType": "urn:ietf:params:oauth:grant-type:token-exchange",
290 
                    "audience": self.provider,
291 
                    "scope": "https://www.googleapis.com/auth/devstorage.read_write",
292 
                    "requestedTokenType": "urn:ietf:params:oauth:token-type:access_token",
293 
                    "subjectToken": jwt,
294 
                    "subjectTokenType": "urn:ietf:params:oauth:token-type:jwt",
295 
                }))
296 
                .send()
297 
                .await?;
298 
            let status = res.status();
299 
            let body: serde_json::Value = res.json().await.unwrap_or_default();
300 
            if !status.is_success() {
301 
                bail!(
302 
                    "Google's token exchange refused {}: {} (check the provider's issuer is this hub's address and its attribute condition admits backup:)",
303 
                    self.subject,
304 
                    body["error_description"].as_str().or(body["error"].as_str()).unwrap_or("no reason given")
305 
                );
306 
            }
307 
            let token = body["access_token"].as_str().context("the token exchange gave no access token")?.to_string();
308 
            anyhow::Ok((token, body["expires_in"].as_i64().unwrap_or(3600)))
309 
        })?;
310 
        *self.token.lock().unwrap() = Some((token.clone(), now + expires_in * 1000));
311 
        Ok(token)
312 
    }
313
314 
    fn object_url(&self, key: &str) -> Result<reqwest::Url> {
315 
        let mut u = reqwest::Url::parse(&format!("{}/storage/v1/b/{}/o", self.storage, self.bucket))?;
316 
        u.path_segments_mut().map_err(|_| anyhow::anyhow!("bad storage URL"))?.push(&self.name(key));
317 
        Ok(u)
318 
    }
319
320 
    fn refused(&self, what: &str, status: reqwest::StatusCode, body: &str) -> anyhow::Error {
321 
        let hint = if status == reqwest::StatusCode::FORBIDDEN || status == reqwest::StatusCode::UNAUTHORIZED {
322 
            format!(" (grant roles/storage.objectAdmin on gs://{} to the principal …/subject/{})", self.bucket, self.subject)
323 
        } else {
324 
            String::new()
325 
        };
326 
        anyhow::anyhow!("GCS {what}: {status}{hint}: {}", body.chars().take(200).collect::<String>())
327 
    }
328
329 
    /// Every object name under `full` (no delimiter), for removal.
330 
    fn all_under(&self, full: &str) -> Result<Vec<String>> {
331 
        let mut out = Vec::new();
332 
        let mut page: Option<String> = None;
333 
        loop {
334 
            let token = self.access()?;
335 
            let mut u = reqwest::Url::parse(&format!("{}/storage/v1/b/{}/o", self.storage, self.bucket))?;
336 
            u.query_pairs_mut().append_pair("prefix", full).append_pair("fields", "items(name),nextPageToken");
337 
            if let Some(p) = &page {
338 
                u.query_pairs_mut().append_pair("pageToken", p);
339 
            }
340 
            let body: serde_json::Value = self.rt.block_on(async {
341 
                let res = self.client.get(u).bearer_auth(&token).send().await?;
342 
                let status = res.status();
343 
                let text = res.text().await?;
344 
                if !status.is_success() {
345 
                    return Err(self.refused("list", status, &text));
346 
                }
347 
                anyhow::Ok(serde_json::from_str(&text)?)
348 
            })?;
349 
            out.extend(body["items"].as_array().into_iter().flatten().filter_map(|i| i["name"].as_str().map(String::from)));
350 
            match body["nextPageToken"].as_str() {
351 
                Some(p) => page = Some(p.to_string()),
352 
                None => return Ok(out),
353 
            }
354 
        }
355 
    }
356 
}
357
358 
impl Destination for Gcs {
359 
    fn exists(&self, key: &str) -> Result<bool> {
360 
        let token = self.access()?;
361 
        let mut u = self.object_url(key)?;
362 
        u.query_pairs_mut().append_pair("fields", "name");
363 
        self.rt.block_on(async {
364 
            let res = self.client.get(u).bearer_auth(&token).send().await?;
365 
            match res.status() {
366 
                s if s.is_success() => Ok(true),
367 
                reqwest::StatusCode::NOT_FOUND => Ok(false),
368 
                s => Err(self.refused("read", s, &res.text().await.unwrap_or_default())),
369 
            }
370 
        })
371 
    }
372 
    fn put(&self, key: &str, bytes: &[u8]) -> Result<()> {
373 
        let token = self.access()?;
374 
        let mut u = reqwest::Url::parse(&format!("{}/upload/storage/v1/b/{}/o", self.storage, self.bucket))?;
375 
        u.query_pairs_mut().append_pair("uploadType", "media").append_pair("name", &self.name(key));
376 
        let body = bytes.to_vec();
377 
        self.rt.block_on(async {
378 
            let res = self.client.post(u).bearer_auth(&token).header("content-type", "application/octet-stream").body(body).send().await?;
379 
            let status = res.status();
380 
            if !status.is_success() {
381 
                return Err(self.refused("write", status, &res.text().await.unwrap_or_default()));
382 
            }
383 
            Ok(())
384 
        })
385 
    }
386 
    fn get(&self, key: &str) -> Result<Vec<u8>> {
387 
        let token = self.access()?;
388 
        let mut u = self.object_url(key)?;
389 
        u.query_pairs_mut().append_pair("alt", "media");
390 
        self.rt.block_on(async {
391 
            let res = self.client.get(u).bearer_auth(&token).send().await?;
392 
            let status = res.status();
393 
            if !status.is_success() {
394 
                return Err(self.refused(&format!("read {key}"), status, &res.text().await.unwrap_or_default()));
395 
            }
396 
            Ok(res.bytes().await?.to_vec())
397 
        })
398 
    }
399 
    fn list(&self, prefix: &str) -> Result<Vec<String>> {
400 
        let full = format!("{}/", self.name(prefix.trim_end_matches('/')));
401 
        let mut out = std::collections::BTreeSet::new();
402 
        let mut page: Option<String> = None;
403 
        loop {
404 
            let token = self.access()?;
405 
            let mut u = reqwest::Url::parse(&format!("{}/storage/v1/b/{}/o", self.storage, self.bucket))?;
406 
            u.query_pairs_mut()
407 
                .append_pair("prefix", &full)
408 
                .append_pair("delimiter", "/")
409 
                .append_pair("fields", "items(name),prefixes,nextPageToken");
410 
            if let Some(p) = &page {
411 
                u.query_pairs_mut().append_pair("pageToken", p);
412 
            }
413 
            let body: serde_json::Value = self.rt.block_on(async {
414 
                let res = self.client.get(u).bearer_auth(&token).send().await?;
415 
                let status = res.status();
416 
                let text = res.text().await?;
417 
                if !status.is_success() {
418 
                    return Err(self.refused("list", status, &text));
419 
                }
420 
                anyhow::Ok(serde_json::from_str(&text)?)
421 
            })?;
422 
            for n in body["items"].as_array().into_iter().flatten().filter_map(|i| i["name"].as_str()) {
423 
                out.insert(n.trim_start_matches(&full).to_string());
424 
            }
425 
            for p in body["prefixes"].as_array().into_iter().flatten().filter_map(|p| p.as_str()) {
426 
                out.insert(p.trim_start_matches(&full).trim_end_matches('/').to_string());
427 
            }
428 
            match body["nextPageToken"].as_str() {
429 
                Some(p) => page = Some(p.to_string()),
430 
                None => return Ok(out.into_iter().filter(|n| !n.is_empty()).collect()),
431 
            }
432 
        }
433 
    }
434 
    fn remove(&self, key: &str) -> Result<()> {
435 
        let mut names = self.all_under(&format!("{}/", self.name(key)))?;
436 
        names.push(self.name(key));
437 
        for n in names {
438 
            let token = self.access()?;
439 
            let mut u = reqwest::Url::parse(&format!("{}/storage/v1/b/{}/o", self.storage, self.bucket))?;
440 
            u.path_segments_mut().map_err(|_| anyhow::anyhow!("bad storage URL"))?.push(&n);
441 
            self.rt.block_on(async {
442 
                let res = self.client.delete(u).bearer_auth(&token).send().await?;
443 
                match res.status() {
444 
                    s if s.is_success() || s == reqwest::StatusCode::NOT_FOUND => Ok(()),
445 
                    s => Err(self.refused("delete", s, &res.text().await.unwrap_or_default())),
446 
                }
447 
            })?;
448 
        }
449 
        Ok(())
450 
    }
451 
}
452
453 
/// Check a destination end to end: write a probe, read it back, remove it.
454 
pub fn probe(dest: &dyn Destination) -> Result<()> {
455 
    let marker = format!("probe {}", db::now_ms());
456 
    dest.put(".oxyhub-probe", marker.as_bytes())?;
457 
    if dest.get(".oxyhub-probe")? != marker.as_bytes() {
458 
        bail!("the probe read back differently");
459 
    }
460 
    dest.remove(".oxyhub-probe")
461 
}
462
463 
/// The owner's destination, from its setting and the site's rules.
464 
pub fn destination(app: &Shared, s: &Setting) -> Result<Box<dyn Destination>> {
465 
    match s.destination.as_str() {
466 
        "dir" => {
467 
            let allowed = &app.config.backups.directories;
468 
            let target = PathBuf::from(&s.target);
469 
            if !allowed.contains(&target) {
470 
                bail!("{} is not a backup directory this hub allows (hub.toml [backups] directories)", s.target);
471 
            }
472 
            let mut root = target.join(&s.owner);
473 
            if !s.prefix.is_empty() {
474 
                root = root.join(s.prefix.trim_matches('/'));
475 
            }
476 
            Ok(Box::new(Directory { root }))
477 
        }
478 
        "gcs" => {
479 
            if s.target.is_empty() || s.provider.is_empty() {
480 
                bail!("a GCS destination needs its bucket and its workload identity provider");
481 
            }
482 
            Ok(Box::new(Gcs::new(app, s)))
483 
        }
484 
        other => bail!("unknown destination {other:?}"),
485 
    }
486 
}
487
488 
// ---------------------------------------------------------------- what
489
490 
/// One file of a repository's store, in a manifest.
491 
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
492 
pub struct File {
493 
    pub path: String,
494 
    pub size: u64,
495 
    /// Its modification time in nanoseconds: unchanged size and time mean
496 
    /// unchanged content, so it is not hashed again.
497 
    pub mtime: i128,
498 
    pub hash: String,
499 
}
500
501 
/// A repository at one run.
502 
#[derive(Clone, Debug, Serialize, Deserialize)]
503 
pub struct Manifest {
504 
    pub owner: String,
505 
    pub name: String,
506 
    /// `repos` or `wikis`.
507 
    pub kind: String,
508 
    pub files: Vec<File>,
509 
}
510
511 
fn blob_key(hash: &str) -> String {
512 
    format!("blobs/{}/{hash}", &hash[..2])
513 
}
514
515 
/// The large files that never change once written come first.
516 
fn immutable(path: &str) -> bool {
517 
    path.starts_with(".oxy/objects/") && (path.contains("/pack-") || path.contains("/loose/"))
518 
}
519
520 
#[derive(Default)]
521 
struct Sent {
522 
    files: i64,
523 
    sent: i64,
524 
    bytes: i64,
525 
}
526
527 
/// Walk `dir`, sending each file the destination lacks; `known` says which
528 
/// paths are unchanged since the last run (and their hashes).
529 
fn copy_dir(dest: &dyn Destination, dir: &Path, known: &HashMap<String, File>, only_immutable: bool, sent: &mut Sent) -> Result<Vec<File>> {
530 
    let mut out = Vec::new();
531 
    let mut stack = vec![dir.to_path_buf()];
532 
    while let Some(d) = stack.pop() {
533 
        let entries = match std::fs::read_dir(&d) {
534 
            Ok(e) => e,
535 
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
536 
            Err(e) => return Err(e.into()),
537 
        };
538 
        for e in entries.flatten() {
539 
            let p = e.path();
540 
            let ft = e.file_type()?;
541 
            if ft.is_dir() {
542 
                stack.push(p);
543 
                continue;
544 
            }
545 
            if !ft.is_file() {
546 
                continue;
547 
            }
548 
            let rel = p.strip_prefix(dir).unwrap_or(&p).to_string_lossy().replace('\\', "/");
549 
            if only_immutable && !immutable(&rel) {
550 
                continue;
551 
            }
552 
            // A file packed away while the walk ran is simply gone.
553 
            let Ok(md) = std::fs::metadata(&p) else { continue };
554 
            let mtime =
555 
                md.modified().ok().and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()).map(|d| d.as_nanos() as i128).unwrap_or(0);
556 
            let size = md.len();
557 
            sent.files += 1;
558 
            if let Some(k) = known.get(&rel).filter(|k| k.size == size && k.mtime == mtime) {
559 
                out.push(k.clone());
560 
                continue;
561 
            }
562 
            let Ok(bytes) = std::fs::read(&p) else { continue };
563 
            let hash = blake3::hash(&bytes).to_hex().to_string();
564 
            let key = blob_key(&hash);
565 
            if !dest.exists(&key)? {
566 
                dest.put(&key, &bytes)?;
567 
                sent.sent += 1;
568 
                sent.bytes += bytes.len() as i64;
569 
            }
570 
            out.push(File { path: rel, size, mtime, hash });
571 
        }
572 
    }
573 
    out.sort_by(|a, b| a.path.cmp(&b.path));
574 
    Ok(out)
575 
}
576
577 
// ---------------------------------------------------------------- records
578
579 
/// Tables and columns never written: secrets, credentials, sessions.
580 
fn private_table(t: &str) -> bool {
581 
    ["secret", "token", "session", "password", "totp", "recovery", "sqlx"].iter().any(|w| t.contains(w))
582 
}
583
584 
fn private_column(c: &str) -> bool {
585 
    ["secret", "token", "password", "salt", "totp", "sealed", "hash_key"].iter().any(|w| c.contains(w))
586 
}
587
588 
async fn tables(pool: &Pool) -> Result<Vec<String>> {
589 
    let mut t: Vec<String> =
590 
        sqlx::query_scalar("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").fetch_all(pool).await?;
591 
    t.retain(|t| !private_table(t));
592 
    t.sort();
593 
    Ok(t)
594 
}
595
596 
async fn columns(pool: &Pool, table: &str) -> Result<Vec<String>> {
597 
    Ok(sqlx::query_scalar(&format!("SELECT name FROM pragma_table_info('{table}')")).fetch_all(pool).await?)
598 
}
599
600 
/// The rows of `table` matching `cond`, as a JSON array, private columns
601 
/// left out and blobs as hex.
602 
async fn rows(pool: &Pool, table: &str, cond: &str) -> Result<serde_json::Value> {
603 
    let cols = columns(pool, table).await?;
604 
    let fields: Vec<String> = cols
605 
        .iter()
606 
        .filter(|c| !private_column(c))
607 
        .map(|c| format!("'{c}', CASE WHEN typeof(\"{c}\") = 'blob' THEN hex(\"{c}\") ELSE \"{c}\" END"))
608 
        .collect();
609 
    if fields.is_empty() {
610 
        return Ok(serde_json::json!([]));
611 
    }
612 
    let sql = format!("SELECT json_group_array(json_object({})) FROM \"{table}\" WHERE {cond}", fields.join(", "));
613 
    let text: String = sqlx::query_scalar(&sql).fetch_one(pool).await?;
614 
    Ok(serde_json::from_str(&text)?)
615 
}
616
617 
/// Every row about something: the tables with column `key` equal to `id`,
618 
/// then, three levels deep, the tables whose foreign keys point at rows
619 
/// already taken (an issue's comments, a proposal's reviews and their
620 
/// threads).
621 
async fn records(pool: &Pool, key: &str, id: i64) -> Result<serde_json::Value> {
622 
    let all = tables(pool).await?;
623 
    let mut conds: Vec<(String, String)> = Vec::new();
624 
    for t in &all {
625 
        if columns(pool, t).await?.iter().any(|c| c == key) {
626 
            conds.push((t.clone(), format!("\"{key}\" = {id}")));
627 
        }
628 
    }
629 
    for _ in 0..3 {
630 
        let mut more = Vec::new();
631 
        for t in &all {
632 
            if conds.iter().any(|(c, _)| c == t) || more.iter().any(|(c, _): &(String, String)| c == t) {
633 
                continue;
634 
            }
635 
            let fks: Vec<(String, String, String)> =
636 
                sqlx::query_as(&format!("SELECT \"table\", \"from\", \"to\" FROM pragma_foreign_key_list('{t}')")).fetch_all(pool).await?;
637 
            if let Some((parent, from, to, pcond)) = fks
638 
                .iter()
639 
                .find_map(|(p, f, to)| conds.iter().find(|(c, _)| c == p).map(|(_, pc)| (p.clone(), f.clone(), to.clone(), pc.clone())))
640 
            {
641 
                more.push((t.clone(), format!("\"{from}\" IN (SELECT \"{to}\" FROM \"{parent}\" WHERE {pcond})")));
642 
            }
643 
        }
644 
        if more.is_empty() {
645 
            break;
646 
        }
647 
        conds.extend(more);
648 
    }
649 
    let mut out = serde_json::Map::new();
650 
    for (t, cond) in conds {
651 
        let r = rows(pool, &t, &cond).await?;
652 
        if r.as_array().is_some_and(|a| !a.is_empty()) {
653 
            out.insert(t, r);
654 
        }
655 
    }
656 
    Ok(serde_json::Value::Object(out))
657 
}
658
659 
/// The key of a run's records naming the accounts, teams and organisations
660 
/// their rows point at, by id: `{"accounts": {"7": "andy"}, …}`.
661 
const NAMES: &str = "_names";
662
663 
/// Tables a restore leaves out: the search index (rebuilt from what is
664 
/// restored), and what is passing or personal rather than the
665 
/// repository's: inboxes, webhook deliveries, runners' declines.
666 
const NOT_RESTORED: [&str; 5] = ["search", "notifications", "deliveries", "flow_declines", NAMES];
667
668 
async fn fk_list(pool: &Pool, table: &str) -> Result<Vec<(String, String, String)>> {
669 
    Ok(sqlx::query_as(&format!("SELECT \"table\", \"from\", \"to\" FROM pragma_foreign_key_list('{table}')")).fetch_all(pool).await?)
670 
}
671
672 
/// The names behind the account, team and organisation ids that `rec`'s
673 
/// rows point at, so a restore on another hub finds them again by name.
674 
async fn names_of(pool: &Pool, rec: &serde_json::Value) -> Result<serde_json::Value> {
675 
    let mut ids: HashMap<String, std::collections::BTreeSet<i64>> = HashMap::new();
676 
    for (table, rows) in rec.as_object().into_iter().flatten() {
677 
        let Some(rows) = rows.as_array() else { continue };
678 
        for (parent, from, _) in fk_list(pool, table).await? {
679 
            if ["accounts", "teams", "organisations"].contains(&parent.as_str()) {
680 
                ids.entry(parent).or_default().extend(rows.iter().filter_map(|r| r[&from].as_i64()));
681 
            }
682 
        }
683 
    }
684 
    let mut out = serde_json::Map::new();
685 
    for (kind, sql) in [
686 
        ("accounts", "SELECT id, username FROM accounts WHERE id = ?"),
687 
        ("organisations", "SELECT id, name FROM organisations WHERE id = ?"),
688 
        ("teams", "SELECT t.id, o.name || '/' || t.name FROM teams t JOIN organisations o ON o.id = t.org_id WHERE t.id = ?"),
689 
    ] {
690 
        let mut m = serde_json::Map::new();
691 
        for id in ids.get(kind).into_iter().flatten() {
692 
            if let Some((id, name)) = sqlx::query_as::<_, (i64, String)>(sql).bind(id).fetch_optional(pool).await? {
693 
                m.insert(id.to_string(), name.into());
694 
            }
695 
        }
696 
        out.insert(kind.into(), m.into());
697 
    }
698 
    Ok(out.into())
699 
}
700
701 
/// The id `name` has on this hub as one of `_names`' kinds.
702 
async fn id_by_name(pool: &Pool, kind: &str, name: &str) -> Result<Option<i64>> {
703 
    Ok(match kind {
704 
        "accounts" => sqlx::query_scalar("SELECT id FROM accounts WHERE username = ?").bind(name).fetch_optional(pool).await?,
705 
        "organisations" => sqlx::query_scalar("SELECT id FROM organisations WHERE name = ?").bind(name).fetch_optional(pool).await?,
706 
        "teams" => match name.split_once('/') {
707 
            Some((org, team)) => {
708 
                sqlx::query_scalar("SELECT t.id FROM teams t JOIN organisations o ON o.id = t.org_id WHERE o.name = ? AND t.name = ?")
709 
                    .bind(org)
710 
                    .bind(team)
711 
                    .fetch_optional(pool)
712 
                    .await?
713 
            }
714 
            None => None,
715 
        },
716 
        _ => None,
717 
    })
718 
}
719
720 
/// One column of a table on this hub.
721 
struct Col {
722 
    name: String,
723 
    ty: String,
724 
    notnull: bool,
725 
    default: Option<String>,
726 
    pk: i64,
727 
}
728
729 
async fn table_cols(pool: &Pool, table: &str) -> Result<Vec<Col>> {
730 
    let rows: Vec<(String, String, i64, Option<String>, i64)> =
731 
        sqlx::query_as(&format!("SELECT name, type, \"notnull\", dflt_value, pk FROM pragma_table_info('{table}')"))
732 
            .fetch_all(pool)
733 
            .await?;
734 
    Ok(rows.into_iter().map(|(name, ty, notnull, default, pk)| Col { name, ty, notnull: notnull != 0, default, pk }).collect())
735 
}
736
737 
/// Bind one value of a backed-up row: blobs were written as hex.
738 
fn bind_json<'q>(
739 
    q: sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>>,
740 
    v: serde_json::Value,
741 
    blob: bool,
742 
) -> sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>> {
743 
    match v {
744 
        serde_json::Value::Null => q.bind(None::<i64>),
745 
        serde_json::Value::Bool(b) => q.bind(b as i64),
746 
        serde_json::Value::Number(n) => match n.as_i64() {
747 
            Some(i) => q.bind(i),
748 
            None => q.bind(n.as_f64().unwrap_or(0.0)),
749 
        },
750 
        serde_json::Value::String(s) if blob => q.bind(hex::decode(&s).unwrap_or_else(|_| s.into_bytes())),
751 
        serde_json::Value::String(s) => q.bind(s),
752 
        other => q.bind(other.to_string()),
753 
    }
754 
}
755
756 
/// Restore one repository's records from a run into this hub, the
757 
/// repository being `repo` here and `old` in the backup (docs/backups.md
758 
/// §7): its issues, proposals, labels, milestones, releases and their
759 
/// assets, checks, Flow runs and the rest, each table after the tables it
760 
/// points at, every id mapped to its new one and every account, team and
761 
/// organisation found again by name. A row pointing at an account this
762 
/// hub lacks keeps a blank reference where it may, and is left out where
763 
/// it may not. What needs re-entering (webhook secrets, never backed up)
764 
/// and what was left out come back as notes.
765 
async fn restore_records(
766 
    app: &Shared,
767 
    dest: &std::sync::Arc<dyn Destination>,
768 
    rec: &serde_json::Value,
769 
    old: i64,
770 
    repo: i64,
771 
) -> Result<Vec<String>> {
772 
    let pool = &app.db;
773 
    let obj = rec.as_object().context("the records are not an object")?;
774 
    let mut by_name: HashMap<(String, i64), Option<i64>> = HashMap::new();
775 
    for (kind, m) in obj.get(NAMES).and_then(|n| n.as_object()).into_iter().flatten() {
776 
        for (id, name) in m.as_object().into_iter().flatten() {
777 
            let (Ok(id), Some(name)) = (id.parse::<i64>(), name.as_str()) else { continue };
778 
            by_name.insert((kind.clone(), id), id_by_name(pool, kind, name).await?);
779 
        }
780 
    }
781 
    let here: std::collections::HashSet<String> = tables(pool).await?.into_iter().collect();
782 
    let mut notes = Vec::new();
783 
    let mut todo: Vec<String> = Vec::new();
784 
    for t in obj.keys().filter(|t| t.as_str() != "repositories" && !NOT_RESTORED.contains(&t.as_str())) {
785 
        if here.contains(t) {
786 
            todo.push(t.clone());
787 
        } else {
788 
            notes.push(format!("{t}: not a table on this hub, left out"));
789 
        }
790 
    }
791 
    let mut fks: HashMap<String, Vec<(String, String, String)>> = HashMap::new();
792 
    for t in &todo {
793 
        fks.insert(t.clone(), fk_list(pool, t).await?);
794 
    }
795 
    // Each table after the tables of the records it points at.
796 
    let mut order = Vec::new();
797 
    while !todo.is_empty() {
798 
        let mut ready: Vec<String> =
799 
            todo.iter().filter(|t| fks[*t].iter().all(|(p, _, _)| p == *t || !todo.contains(p))).cloned().collect();
800 
        if ready.is_empty() {
801 
            ready.push(todo[0].clone());
802 
        }
803 
        todo.retain(|t| !ready.contains(t));
804 
        order.extend(ready);
805 
    }
806 
    let mut maps: HashMap<String, HashMap<i64, i64>> = HashMap::new();
807 
    maps.insert("repositories".into(), [(old, repo)].into());
808 
    let mut left_out: std::collections::BTreeMap<String, usize> = Default::default();
809 
    let mut tx = pool.begin().await?;
810 
    for table in &order {
811 
        let cols = table_cols(pool, table).await?;
812 
        let keys: Vec<&str> = cols.iter().filter(|c| c.pk > 0).map(|c| c.name.as_str()).collect();
813 
        let by_id = keys == ["id"] && cols.iter().any(|c| c.name == "id" && c.ty.eq_ignore_ascii_case("INTEGER"));
814 
        let verb = if keys == ["repo_id"] {
815 
            "INSERT OR REPLACE"
816 
        } else if by_id {
817 
            "INSERT"
818 
        } else {
819 
            "INSERT OR IGNORE"
820 
        };
821 
        let mut rows: Vec<&serde_json::Map<String, serde_json::Value>> =
822 
            obj[table].as_array().into_iter().flatten().filter_map(|r| r.as_object()).collect();
823 
        rows.sort_by_key(|r| r.get("id").and_then(|v| v.as_i64()).unwrap_or(0));
824 
        'row: for row in rows {
825 
            let (mut names, mut values) = (Vec::new(), Vec::new());
826 
            for c in &cols {
827 
                if by_id && c.name == "id" {
828 
                    continue;
829 
                }
830 
                let mut v = row.get(&c.name).cloned().filter(|v| !v.is_null());
831 
                if let Some((parent, _, _)) = fks[table].iter().find(|(_, from, _)| *from == c.name) {
832 
                    if let Some(was) = v.as_ref().and_then(|v| v.as_i64()) {
833 
                        let now = match parent.as_str() {
834 
                            "accounts" | "teams" | "organisations" => by_name.get(&(parent.clone(), was)).copied().flatten(),
835 
                            p => maps.get(p).and_then(|m| m.get(&was)).copied(),
836 
                        };
837 
                        if now.is_none() && c.notnull {
838 
                            *left_out.entry(table.clone()).or_default() += 1;
839 
                            continue 'row;
840 
                        }
841 
                        v = now.map(serde_json::Value::from);
842 
                    }
843 
                }
844 
                let blob = c.ty.to_ascii_uppercase().contains("BLOB");
845 
                match v {
846 
                    Some(v) => {
847 
                        names.push(c.name.clone());
848 
                        values.push((v, blob));
849 
                    }
850 
                    // Never backed up (a secret): blank here, to enter again.
851 
                    None if c.notnull && c.default.is_none() => {
852 
                        names.push(c.name.clone());
853 
                        let blank = if c.ty.to_ascii_uppercase().contains("INT") { serde_json::json!(0) } else { serde_json::json!("") };
854 
                        values.push((blank, false));
855 
                    }
856 
                    None => {}
857 
                }
858 
            }
859 
            let sql = format!(
860 
                "{verb} INTO \"{table}\" ({}) VALUES ({})",
861 
                names.iter().map(|n| format!("\"{n}\"")).collect::<Vec<_>>().join(", "),
862 
                vec!["?"; names.len()].join(", ")
863 
            );
864 
            let mut q = sqlx::query(&sql);
865 
            for (v, blob) in values {
866 
                q = bind_json(q, v, blob);
867 
            }
868 
            let done = q.execute(&mut *tx).await.with_context(|| format!("restore a row of {table}"))?;
869 
            if let (true, Some(was)) = (by_id, row.get("id").and_then(|v| v.as_i64())) {
870 
                maps.entry(table.clone()).or_default().insert(was, done.last_insert_rowid());
871 
            }
872 
        }
873 
    }
874 
    tx.commit().await?;
875 
    for (table, n) in left_out {
876 
        notes.push(format!("{table}: {n} row(s) left out: they name accounts or rows this hub lacks"));
877 
    }
878 
    // Search finds what came back.
879 
    for (kind, table) in [("issue", "issues"), ("proposal", "proposals")] {
880 
        let found: Vec<(i64, i64, String, String)> =
881 
            sqlx::query_as(&format!("SELECT id, number, title, body FROM {table} WHERE repo_id = ?")).bind(repo).fetch_all(pool).await?;
882 
        for (id, number, title, body) in found {
883 
            db::notify::index(pool, kind, repo, id, number, &title, &body).await?;
884 
        }
885 
    }
886 
    // Webhook secrets were never written: they are entered again.
887 
    let hooks: Vec<String> =
888 
        sqlx::query_scalar("SELECT url FROM hooks WHERE repo_id = ? AND secret = ''").bind(repo).fetch_all(pool).await?;
889 
    for url in hooks {
890 
        notes.push(format!("webhook {url}: enter its secret again (Settings › Webhooks)"));
891 
    }
892 
    // Release assets, each checked against its name.
893 
    for hash in obj.get("release_assets").and_then(|a| a.as_array()).into_iter().flatten().filter_map(|a| a["hash"].as_str()) {
894 
        if hash.len() != 64 || !hash.bytes().all(|b| b.is_ascii_hexdigit()) {
895 
            continue;
896 
        }
897 
        let path = crate::web::releases::blob_path(app, hash);
898 
        if path.exists() {
899 
            continue;
900 
        }
901 
        let got = tokio::task::block_in_place(|| dest.get(&blob_key(hash)));
902 
        match got {
903 
            Ok(bytes) if blake3::hash(&bytes).to_hex().as_str() == hash => {
904 
                std::fs::create_dir_all(path.parent().context("no parent")?)?;
905 
                std::fs::write(&path, &bytes)?;
906 
            }
907 
            Ok(_) => notes.push(format!("release asset {}: the copy does not match its hash", &hash[..12])),
908 
            Err(_) => notes.push(format!("release asset {}: not in the backup", &hash[..12])),
909 
        }
910 
    }
911 
    Ok(notes)
912 
}
913
914 
// ---------------------------------------------------------------- a run
915
916 
fn run_name(ms: i64) -> String {
917 
    let t = time::OffsetDateTime::from_unix_timestamp_nanos(ms as i128 * 1_000_000).unwrap_or(time::OffsetDateTime::UNIX_EPOCH);
918 
    format!("{:04}{:02}{:02}T{:02}{:02}{:02}Z", t.year(), u8::from(t.month()), t.day(), t.hour(), t.minute(), t.second())
919 
}
920
921 
/// The last whole run's manifests, by `kind/name`.
922 
fn last_manifests(dest: &dyn Destination) -> HashMap<String, HashMap<String, File>> {
923 
    let mut out = HashMap::new();
924 
    let Ok(at) = dest.get("LATEST").map(|b| String::from_utf8_lossy(&b).trim().to_string()) else { return out };
925 
    for name in dest.list(&format!("runs/{at}/repos")).unwrap_or_default() {
926 
        if let Ok(m) = dest.get(&format!("runs/{at}/repos/{name}")).and_then(|b| Ok(serde_json::from_slice::<Manifest>(&b)?)) {
927 
            out.insert(format!("{}/{}", m.kind, m.name), m.files.into_iter().map(|f| (f.path.clone(), f)).collect());
928 
        }
929 
    }
930 
    out
931 
}
932
933 
/// Back up `owner` now, as its setting says. The run is recorded whatever
934 
/// happens; a failure notifies the owner once until a run succeeds.
935 
pub async fn run(app: &Shared, owner: &str, trigger: &str) -> Result<Run> {
936 
    let s = setting(&app.db, owner).await?.with_context(|| format!("{owner} has no backup set up"))?;
937 
    let started = db::now_ms();
938 
    let at = run_name(started);
939 
    let id: i64 =
940 
        sqlx::query_scalar("INSERT INTO backup_runs (owner, started_ms, status, at, trigger) VALUES (?, ?, 'running', ?, ?) RETURNING id")
941 
            .bind(owner)
942 
            .bind(started)
943 
            .bind(&at)
944 
            .bind(trigger)
945 
            .fetch_one(&app.db)
946 
            .await?;
947 
    let outcome = run_into(app, &s, &at).await;
948 
    let (status, detail, sent) = match outcome {
949 
        Ok((sent, problems)) if problems.is_empty() => ("ok", String::new(), sent),
950 
        Ok((sent, problems)) => ("partial", problems.join("; "), sent),
951 
        Err(e) => ("failed", format!("{e:#}"), (0, Sent::default())),
952 
    };
953 
    sqlx::query("UPDATE backup_runs SET finished_ms = ?, status = ?, detail = ?, repos = ?, files = ?, sent = ?, bytes = ? WHERE id = ?")
954 
        .bind(db::now_ms())
955 
        .bind(status)
956 
        .bind(&detail)
957 
        .bind(sent.0)
958 
        .bind(sent.1.files)
959 
        .bind(sent.1.sent)
960 
        .bind(sent.1.bytes)
961 
        .bind(id)
962 
        .execute(&app.db)
963 
        .await?;
964 
    if status != "ok" {
965 
        notify_failure(app, &s, status, &detail).await;
966 
    }
967 
    let r: Run = sqlx::query_as("SELECT * FROM backup_runs WHERE id = ?").bind(id).fetch_one(&app.db).await?;
968 
    Ok(r)
969 
}
970
971 
async fn run_into(app: &Shared, s: &Setting, at: &str) -> Result<((i64, Sent), Vec<String>)> {
972 
    let dest: std::sync::Arc<dyn Destination> = destination(app, s)?.into();
973 
    let known = {
974 
        let d = dest.clone();
975 
        tokio::task::spawn_blocking(move || last_manifests(d.as_ref())).await?
976 
    };
977 
    let mine: Vec<db::repos::RepoRow> = db::repos::list(&app.db).await?.into_iter().filter(|r| r.owner == s.owner).collect();
978 
    let mut problems = Vec::new();
979 
    let mut sent = Sent::default();
980 
    let mut repos = 0i64;
981 
    for row in &mine {
982 
        let Some(name) = RepoName::new(&row.owner, &row.name) else { continue };
983 
        for (kind, store) in [("repos", &app.repos), ("wikis", &app.wikis)] {
984 
            if !store.exists(&name) {
985 
                continue;
986 
            }
987 
            let dir = store.dir(&name);
988 
            let k = known.get(&format!("{kind}/{}", row.name)).cloned().unwrap_or_default();
989 
            // The large, unchanging files without the lock, then the whole
990 
            // walk again under the read lock: pushes wait for that moment.
991 
            let (d1, dir1, k1) = (dest.clone(), dir.clone(), k.clone());
992 
            let first = tokio::task::spawn_blocking(move || {
993 
                let mut s = Sent::default();
994 
                copy_dir(d1.as_ref(), &dir1, &k1, true, &mut s).map(|f| (f, s))
995 
            })
996 
            .await?;
997 
            let mut k2 = k;
998 
            if let Ok((files, _)) = &first {
999 
                for f in files {
1000 
                    k2.insert(f.path.clone(), f.clone());
1001 
                }
1002 
            }
1003 
            let guard = store.read_guard(&name).await;
1004 
            let (d2, dir2) = (dest.clone(), dir.clone());
1005 
            let second = tokio::task::spawn_blocking(move || {
1006 
                let _guard = guard;
1007 
                let mut s = Sent::default();
1008 
                copy_dir(d2.as_ref(), &dir2, &k2, false, &mut s).map(|f| (f, s))
1009 
            })
1010 
            .await?;
1011 
            match second {
1012 
                Ok((files, s2)) => {
1013 
                    let s1 = first.map(|(_, s)| s).unwrap_or_default();
1014 
                    sent.files += s2.files;
1015 
                    sent.sent += s1.sent + s2.sent;
1016 
                    sent.bytes += s1.bytes + s2.bytes;
1017 
                    let m = Manifest { owner: row.owner.clone(), name: row.name.clone(), kind: kind.into(), files };
1018 
                    let key = format!("runs/{at}/repos/{kind}-{}.json", row.name);
1019 
                    tokio::task::block_in_place(|| dest.put(&key, &serde_json::to_vec(&m)?))?;
1020 
                    if kind == "repos" {
1021 
                        repos += 1;
1022 
                    }
1023 
                }
1024 
                Err(e) => problems.push(format!("{}/{} ({kind}): {e:#}", row.owner, row.name)),
1025 
            }
1026 
        }
1027 
        let rec = async {
1028 
            let mut r = records(&app.db, "repo_id", row.id).await?;
1029 
            // The repository's own row, which restore recreates first, and
1030 
            // the names behind the accounts, teams and organisations its
1031 
            // rows point at, which another hub finds them by.
1032 
            let own = rows(&app.db, "repositories", &format!("id = {}", row.id)).await?;
1033 
            let names = names_of(&app.db, &r).await?;
1034 
            if let Some(m) = r.as_object_mut() {
1035 
                m.insert("repositories".into(), own);
1036 
                m.insert(NAMES.into(), names);
1037 
            }
1038 
            anyhow::Ok(r)
1039 
        };
1040 
        match rec.await {
1041 
            Ok(r) => {
1042 
                tokio::task::block_in_place(|| dest.put(&format!("runs/{at}/records/{}.json", row.name), &serde_json::to_vec(&r)?))?;
1043 
                // Release assets live beside the stores, named by their
1044 
                // BLAKE3: each sent once, as the stores' files are.
1045 
                for hash in r["release_assets"].as_array().into_iter().flatten().filter_map(|a| a["hash"].as_str()) {
1046 
                    if hash.len() != 64 || !hash.bytes().all(|b| b.is_ascii_hexdigit()) {
1047 
                        continue;
1048 
                    }
1049 
                    let path = crate::web::releases::blob_path(app, hash);
1050 
                    let key = blob_key(hash);
1051 
                    let put = tokio::task::block_in_place(|| -> Result<Option<i64>> {
1052 
                        if dest.exists(&key)? {
1053 
                            return Ok(None);
1054 
                        }
1055 
                        let bytes = std::fs::read(&path).with_context(|| format!("read release asset {}", &hash[..12]))?;
1056 
                        dest.put(&key, &bytes)?;
1057 
                        Ok(Some(bytes.len() as i64))
1058 
                    });
1059 
                    match put {
1060 
                        Ok(Some(n)) => {
1061 
                            sent.files += 1;
1062 
                            sent.sent += 1;
1063 
                            sent.bytes += n;
1064 
                        }
1065 
                        Ok(None) => sent.files += 1,
1066 
                        Err(e) => problems.push(format!("{}/{} release asset: {e:#}", row.owner, row.name)),
1067 
                    }
1068 
                }
1069 
            }
1070 
            Err(e) => problems.push(format!("{}/{} records: {e:#}", row.owner, row.name)),
1071 
        }
1072 
    }
1073 
    // The owner itself.
1074 
    let owner_rows = if s.kind == "org" {
1075 
        match db::accounts::org_by_name(&app.db, &s.owner).await? {
1076 
            Some(o) => records(&app.db, "org_id", o.id).await?,
1077 
            None => serde_json::json!({}),
1078 
        }
1079 
    } else {
1080 
        match db::accounts::account_by_name(&app.db, &s.owner).await? {
1081 
            Some(a) => records(&app.db, "account_id", a.id).await?,
1082 
            None => serde_json::json!({}),
1083 
        }
1084 
    };
1085 
    // The blocking calls of a destination (GCS drives its client on this
1086 
    // runtime) run where blocking is allowed.
1087 
    tokio::task::block_in_place(|| -> Result<()> {
1088 
        dest.put(&format!("runs/{at}/owner.json"), &serde_json::to_vec(&owner_rows)?)?;
1089 
        dest.put(&format!("runs/{at}/DONE"), at.as_bytes())?;
1090 
        dest.put("LATEST", at.as_bytes())?;
1091 
        prune(dest.as_ref(), s.keep_days)
1092 
    })?;
1093 
    Ok(((repos, sent), problems))
1094 
}
1095
1096 
/// Drop whole runs older than `keep_days`, and the blobs no kept run
1097 
/// names. The newest whole run is always kept.
1098 
pub fn prune(dest: &dyn Destination, keep_days: i64) -> Result<()> {
1099 
    let cutoff = run_name(db::now_ms() - keep_days.max(1) * 86_400_000);
1100 
    let runs = dest.list("runs")?;
1101 
    let whole: Vec<&String> = runs.iter().filter(|r| dest.exists(&format!("runs/{r}/DONE")).unwrap_or(false)).collect();
1102 
    let newest = whole.last().map(|s| s.to_string());
1103 
    let mut removed = false;
1104 
    for r in &runs {
1105 
        if r.as_str() < cutoff.as_str() && Some(r.to_string()) != newest {
1106 
            dest.remove(&format!("runs/{r}"))?;
1107 
            removed = true;
1108 
        }
1109 
    }
1110 
    if !removed {
1111 
        return Ok(());
1112 
    }
1113 
    let mut wanted = std::collections::HashSet::new();
1114 
    for r in dest.list("runs")? {
1115 
        for m in dest.list(&format!("runs/{r}/repos"))? {
1116 
            if let Ok(m) = dest.get(&format!("runs/{r}/repos/{m}")).and_then(|b| Ok(serde_json::from_slice::<Manifest>(&b)?)) {
1117 
                wanted.extend(m.files.into_iter().map(|f| f.hash));
1118 
            }
1119 
        }
1120 
    }
1121 
    for shard in dest.list("blobs")? {
1122 
        for b in dest.list(&format!("blobs/{shard}"))? {
1123 
            if !wanted.contains(&b) {
1124 
                dest.remove(&format!("blobs/{shard}/{b}"))?;
1125 
            }
1126 
        }
1127 
    }
1128 
    Ok(())
1129 
}
1130
1131 
async fn notify_failure(app: &Shared, s: &Setting, status: &str, detail: &str) {
1132 
    // Once per spell of failures: not when the run before failed too.
1133 
    let before: Option<String> =
1134 
        sqlx::query_scalar("SELECT status FROM backup_runs WHERE owner = ? AND finished_ms IS NOT NULL ORDER BY id DESC LIMIT 1 OFFSET 1")
1135 
            .bind(&s.owner)
1136 
            .fetch_optional(&app.db)
1137 
            .await
1138 
            .ok()
1139 
            .flatten();
1140 
    if before.as_deref().is_some_and(|b| b != "ok") {
1141 
        return;
1142 
    }
1143 
    let (who, url): (Vec<i64>, String) = if s.kind == "org" {
1144 
        match db::accounts::org_by_name(&app.db, &s.owner).await.ok().flatten() {
1145 
            Some(o) => (
1146 
                db::accounts::org_members(&app.db, o.id)
1147 
                    .await
1148 
                    .unwrap_or_default()
1149 
                    .into_iter()
1150 
                    .filter(|(_, r)| r == "owner")
1151 
                    .map(|(a, _)| a.id)
1152 
                    .collect(),
1153 
                format!("/orgs/{}/settings#backups", s.owner),
1154 
            ),
1155 
            None => return,
1156 
        }
1157 
    } else {
1158 
        match db::accounts::account_by_name(&app.db, &s.owner).await.ok().flatten() {
1159 
            Some(a) => (vec![a.id], "/settings/backups".into()),
1160 
            None => return,
1161 
        }
1162 
    };
1163 
    let title = format!("Backup of {} {status}: {}", s.owner, detail.chars().take(160).collect::<String>());
1164 
    for id in who {
1165 
        let _ = db::notify::add(
1166 
            &app.db,
1167 
            db::notify::New {
1168 
                account_id: id,
1169 
                repo_id: None,
1170 
                reason: db::notify::Reason::Participating,
1171 
                kind: "backup",
1172 
                title: &title,
1173 
                url: &url,
1174 
                actor: "",
1175 
            },
1176 
        )
1177 
        .await;
1178 
    }
1179 
}
1180
1181 
/// Run each owner's backup when it is due, one at a time. A run left
1182 
/// "running" by a restart is marked failed first.
1183 
pub async fn scheduler(app: Shared) {
1184 
    let _ = sqlx::query(
1185 
        "UPDATE backup_runs SET status = 'failed', detail = 'the hub restarted during the run', finished_ms = ? WHERE status = 'running'",
1186 
    )
1187 
    .bind(db::now_ms())
1188 
    .execute(&app.db)
1189 
    .await;
1190 
    loop {
1191 
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
1192 
        for s in due(&app.db).await.unwrap_or_default() {
1193 
            if let Err(e) = run(&app, &s.owner, "schedule").await {
1194 
                tracing::warn!("backup of {}: {e:#}", s.owner);
1195 
            }
1196 
        }
1197 
    }
1198 
}
1199
1200 
/// The owners whose next run is due now.
1201 
pub async fn due(pool: &Pool) -> sqlx::Result<Vec<Setting>> {
1202 
    let all: Vec<Setting> = sqlx::query_as("SELECT * FROM backup_settings WHERE every != 'off'").fetch_all(pool).await?;
1203 
    let now = db::now_ms();
1204 
    let mut out = Vec::new();
1205 
    for s in all {
1206 
        let Some(period) = s.period_s() else { continue };
1207 
        let last: Option<i64> =
1208 
            sqlx::query_scalar("SELECT MAX(started_ms) FROM backup_runs WHERE owner = ?").bind(&s.owner).fetch_one(pool).await?;
1209 
        if last.is_none_or(|l| now - l >= period * 1000) {
1210 
            out.push(s);
1211 
        }
1212 
    }
1213 
    Ok(out)
1214 
}
1215
1216 
/// Restore every repository of the whole run `at` (the latest when none)
1217 
/// from `dest` into `repos_root/OWNER/NAME` and `wikis_root/…`, refusing to
1218 
/// overwrite one that exists. Each file is checked against its hash.
1219 
pub fn restore(dest: &dyn Destination, at: Option<&str>, repos_root: &Path, wikis_root: &Path) -> Result<Vec<String>> {
1220 
    let at = match at {
1221 
        Some(a) => a.to_string(),
1222 
        None => String::from_utf8_lossy(&dest.get("LATEST").context("no whole run at this destination")?).trim().to_string(),
1223 
    };
1224 
    if !dest.exists(&format!("runs/{at}/DONE"))? {
1225 
        bail!("run {at} is not whole (no DONE): pick another with --run");
1226 
    }
1227 
    let mut done = Vec::new();
1228 
    for name in dest.list(&format!("runs/{at}/repos"))? {
1229 
        let m: Manifest = serde_json::from_slice(&dest.get(&format!("runs/{at}/repos/{name}"))?)?;
1230 
        let root = if m.kind == "wikis" { wikis_root } else { repos_root };
1231 
        let dir = root.join(&m.owner).join(&m.name);
1232 
        if dir.exists() {
1233 
            bail!("{} exists: restore into an empty hub, or move it aside", dir.display());
1234 
        }
1235 
        for f in &m.files {
1236 
            let bytes = dest.get(&blob_key(&f.hash))?;
1237 
            if blake3::hash(&bytes).to_hex().as_str() != f.hash {
1238 
                bail!("{}/{} {}: the copy does not match its hash", m.owner, m.name, f.path);
1239 
            }
1240 
            let p = dir.join(&f.path);
1241 
            std::fs::create_dir_all(p.parent().context("no parent")?)?;
1242 
            std::fs::write(&p, &bytes)?;
1243 
        }
1244 
        done.push(format!("{}/{} ({})", m.owner, m.name, m.kind));
1245 
    }
1246 
    Ok(done)
1247 
}
1248
1249 
/// Restore the whole run `at` (the latest when none) into this hub: each
1250 
/// repository's and wiki's store byte for byte, then its row and settings,
1251 
/// then its records (issues, proposals, releases and the rest), so it shows
1252 
/// on the hub as it was (docs/backups.md §7). What was restored, then any
1253 
/// notes (`note: …`) on what needs entering again or was left out.
1254 
pub async fn restore_into_hub(app: &Shared, dest: std::sync::Arc<dyn Destination>, at: Option<String>) -> Result<Vec<String>> {
1255 
    let (repos_root, wikis_root) = (app.repos.root().to_path_buf(), app.wikis.root().to_path_buf());
1256 
    let (d, a) = (dest.clone(), at.clone());
1257 
    let done = tokio::task::spawn_blocking(move || restore(d.as_ref(), a.as_deref(), &repos_root, &wikis_root)).await??;
1258 
    let at = match at {
1259 
        Some(a) => a,
1260 
        None => String::from_utf8_lossy(&tokio::task::block_in_place(|| dest.get("LATEST"))?).trim().to_string(),
1261 
    };
1262 
    let mut notes = Vec::new();
1263 
    for name in tokio::task::block_in_place(|| dest.list(&format!("runs/{at}/records")))? {
1264 
        let rec: serde_json::Value =
1265 
            serde_json::from_slice(&tokio::task::block_in_place(|| dest.get(&format!("runs/{at}/records/{name}")))?)?;
1266 
        let Some(row) = rec["repositories"].as_array().and_then(|a| a.first()) else { continue };
1267 
        let (owner, repo) = (row["owner"].as_str().unwrap_or_default(), row["name"].as_str().unwrap_or_default());
1268 
        if owner.is_empty() || db::repos::get(&app.db, owner, repo).await?.is_some() {
1269 
            continue;
1270 
        }
1271 
        let visibility = match row["visibility"].as_str() {
1272 
            Some("private") => hub_api::Visibility::Private,
1273 
            Some("internal") => hub_api::Visibility::Internal,
1274 
            _ => hub_api::Visibility::Public,
1275 
        };
1276 
        let made = db::repos::create(&app.db, owner, repo, row["description"].as_str().unwrap_or_default(), visibility).await?;
1277 
        // Its settings as they were: every column the row kept but its
1278 
        // name, its id and a fork's parent (another hub's id).
1279 
        let cols = table_cols(&app.db, "repositories").await?;
1280 
        let set: Vec<(&Col, serde_json::Value)> = cols
1281 
            .iter()
1282 
            .filter(|c| !["id", "owner", "name", "forked_from"].contains(&c.name.as_str()))
1283 
            .filter_map(|c| row.get(&c.name).filter(|v| !v.is_null()).map(|v| (c, v.clone())))
1284 
            .collect();
1285 
        if !set.is_empty() {
1286 
            let sql = format!(
1287 
                "UPDATE repositories SET {} WHERE id = ?",
1288 
                set.iter().map(|(c, _)| format!("\"{}\" = ?", c.name)).collect::<Vec<_>>().join(", ")
1289 
            );
1290 
            let mut q = sqlx::query(&sql);
1291 
            for (c, v) in set {
1292 
                q = bind_json(q, v, c.ty.to_ascii_uppercase().contains("BLOB"));
1293 
            }
1294 
            q.bind(made.id).execute(&app.db).await?;
1295 
        }
1296 
        let old = row["id"].as_i64().unwrap_or(-1);
1297 
        for n in restore_records(app, &dest, &rec, old, made.id).await.with_context(|| format!("restore {owner}/{repo}'s records"))? {
1298 
            notes.push(format!("note: {owner}/{repo}: {n}"));
1299 
        }
1300 
    }
1301 
    let mut done = done;
1302 
    done.extend(notes);
1303 
    Ok(done)
1304 
}
1305
1306 
#[cfg(test)]
1307 
mod tests {
1308 
    use super::*;
1309
1310 
    #[test]
1311 
    fn a_directory_destination_keeps_keys_inside_its_root() {
1312 
        let d = tempfile::tempdir().unwrap();
1313 
        let dest = Directory { root: d.path().to_path_buf() };
1314 
        dest.put("runs/a/DONE", b"a").unwrap();
1315 
        assert_eq!(dest.get("runs/a/DONE").unwrap(), b"a");
1316 
        assert_eq!(dest.list("runs").unwrap(), vec!["a"]);
1317 
        assert!(dest.put("../escape", b"x").is_err() && dest.put("/abs", b"x").is_err());
1318 
        dest.remove("runs").unwrap();
1319 
        assert!(dest.list("runs").unwrap().is_empty());
1320 
    }
1321
1322 
    #[test]
1323 
    fn files_are_sent_once_and_unchanged_ones_not_read_again() {
1324 
        let src = tempfile::tempdir().unwrap();
1325 
        let out = tempfile::tempdir().unwrap();
1326 
        let dest = Directory { root: out.path().to_path_buf() };
1327 
        std::fs::create_dir_all(src.path().join(".oxy/objects/loose")).unwrap();
1328 
        std::fs::write(src.path().join(".oxy/objects/loose/aa"), b"object").unwrap();
1329 
        std::fs::write(src.path().join(".oxy/ops-head"), b"ref one").unwrap();
1330 
        let mut s = Sent::default();
1331 
        let files = copy_dir(&dest, src.path(), &HashMap::new(), false, &mut s).unwrap();
1332 
        assert_eq!((files.len(), s.sent), (2, 2));
1333 
        let known: HashMap<String, File> = files.iter().map(|f| (f.path.clone(), f.clone())).collect();
1334 
        let mut s = Sent::default();
1335 
        let again = copy_dir(&dest, src.path(), &known, false, &mut s).unwrap();
1336 
        assert_eq!((again, s.sent), (files.clone(), 0), "nothing new travels");
1337 
        // The first pass takes only the files that never change.
1338 
        let mut s = Sent::default();
1339 
        let first = copy_dir(&dest, src.path(), &HashMap::new(), true, &mut s).unwrap();
1340 
        assert_eq!(first.iter().map(|f| f.path.as_str()).collect::<Vec<_>>(), vec![".oxy/objects/loose/aa"]);
1341 
    }
1342
1343 
    #[test]
1344 
    fn run_names_sort_by_time() {
1345 
        assert_eq!(run_name(0), "19700101T000000Z");
1346 
        assert!(run_name(1_791_900_000_000) < run_name(1_791_903_600_000));
1347 
    }
1348 
}