OXYGENHUB
# 5ef6cf72cd19 oxygenhub / crates / hub / src / backups.rs
55.3 KiB · Rust · last changed Backups: a restore brings back the records, and release assets travel 22 days ago source blame history raw
1 
//! Backups (docs/backups.md, oxygenhub#169): each user and organisation
2 
//! copies its repositories, on a schedule, to a destination it controls.
3 
//!
4 
//! A repository is its store directory (`.oxy/`): packs and loose objects
5 
//! that never change once written, and small files that do (refs, the
6 
//! operation log, policies, signature indexes). A run copies each file
7 
//! once, named by its BLAKE3 (`blobs/`), and writes a manifest of path to
8 
//! hash per repository (`runs/<at>/repos/`), so only what changed travels
9 
//! and any kept run can be restored byte for byte, signatures and all.
10 
//! The large files are copied first; the walk is then repeated under the
11 
//! repository's read lock, which pushes wait on, so each manifest is one
12 
//! consistent moment. Beside each manifest go the hub's records for the
13 
//! repository as JSON (never a secret value), the owner's records, and
14 
//! last a `DONE` marker: a run without one was cut short and is ignored.
15
16 
use crate::db::{self, Pool};
17 
use crate::repos::RepoName;
18 
use crate::Shared;
19 
use anyhow::{bail, Context, Result};
20 
use serde::{Deserialize, Serialize};
21 
use std::collections::HashMap;
22 
use std::path::{Path, PathBuf};
23
24 
/// How often, as the settings offer it.
25 
pub const EVERY: &[(&str, &str)] = &[("off", "Off"), ("hourly", "Every hour"), ("daily", "Every day")];
26 
/// How long runs are kept, in days.
27 
pub const KEEP: &[i64] = &[7, 30, 90];
28
29 
/// What an owner backs up, where and how often.
30 
#[derive(Clone, Debug, sqlx::FromRow, Serialize)]
31 
pub struct Setting {
32 
    pub owner: String,
33 
    pub kind: String,
34 
    pub every: String,
35 
    pub destination: String,
36 
    pub target: String,
37 
    pub prefix: String,
38 
    pub provider: String,
39 
    pub keep_days: i64,
40 
    pub updated_ms: i64,
41 
}
42
43 
impl Setting {
44 
    pub fn off(owner: &str, kind: &str) -> Setting {
45 
        Setting {
46 
            owner: owner.into(),
47 
            kind: kind.into(),
48 
            every: "off".into(),
49 
            destination: "dir".into(),
50 
            target: String::new(),
51 
            prefix: String::new(),
52 
            provider: String::new(),
53 
            keep_days: 30,
54 
            updated_ms: 0,
55 
        }
56 
    }
57
58 
    /// Seconds between runs; none when off.
59 
    pub fn period_s(&self) -> Option<i64> {
60 
        match self.every.as_str() {
61 
            "hourly" => Some(3600),
62 
            "daily" => Some(86_400),
63 
            _ => None,
64 
        }
65 
    }
66 
}
67
68 
/// One run, as the settings page and the API show it.
69 
#[derive(Clone, Debug, sqlx::FromRow, Serialize)]
70 
pub struct Run {
71 
    pub id: i64,
72 
    pub owner: String,
73 
    pub started_ms: i64,
74 
    pub finished_ms: Option<i64>,
75 
    pub status: String,
76 
    pub at: String,
77 
    pub repos: i64,
78 
    pub files: i64,
79 
    pub sent: i64,
80 
    pub bytes: i64,
81 
    pub detail: String,
82 
    pub trigger: String,
83 
}
84
85 
pub async fn setting(pool: &Pool, owner: &str) -> sqlx::Result<Option<Setting>> {
86 
    sqlx::query_as("SELECT * FROM backup_settings WHERE owner = ?").bind(owner).fetch_optional(pool).await
87 
}
88
89 
pub async fn save(pool: &Pool, s: &Setting) -> sqlx::Result<()> {
90 
    sqlx::query(
91 
        "INSERT INTO backup_settings (owner, kind, every, destination, target, prefix, provider, keep_days, updated_ms) \
92 
         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(owner) DO UPDATE SET kind = excluded.kind, every = excluded.every, \
93 
         destination = excluded.destination, target = excluded.target, prefix = excluded.prefix, provider = excluded.provider, \
94 
         keep_days = excluded.keep_days, updated_ms = excluded.updated_ms",
95 
    )
96 
    .bind(&s.owner)
97 
    .bind(&s.kind)
98 
    .bind(&s.every)
99 
    .bind(&s.destination)
100 
    .bind(&s.target)
101 
    .bind(&s.prefix)
102 
    .bind(&s.provider)
103 
    .bind(s.keep_days)
104 
    .bind(db::now_ms())
105 
    .execute(pool)
106 
    .await?;
107 
    Ok(())
108 
}
109
110 
pub async fn runs(pool: &Pool, owner: &str, limit: i64) -> sqlx::Result<Vec<Run>> {
111 
    sqlx::query_as("SELECT * FROM backup_runs WHERE owner = ? ORDER BY id DESC LIMIT ?").bind(owner).bind(limit).fetch_all(pool).await
112 
}
113
114 
// ---------------------------------------------------------------- where
115
116 
/// Where backups are written: keys are `/`-separated paths.
117 
pub trait Destination: Send + Sync {
118 
    fn exists(&self, key: &str) -> Result<bool>;
119 
    /// Write `bytes` at `key`, whole or not at all.
120 
    fn put(&self, key: &str, bytes: &[u8]) -> Result<()>;
121 
    fn get(&self, key: &str) -> Result<Vec<u8>>;
122 
    /// The names directly under `prefix` (files and "directories").
123 
    fn list(&self, prefix: &str) -> Result<Vec<String>>;
124 
    /// Remove `key` and everything under it.
125 
    fn remove(&self, key: &str) -> Result<()>;
126 
}
127
128 
/// A directory on the hub's host.
129 
pub struct Directory {
130 
    pub root: PathBuf,
131 
}
132
133 
impl Directory {
134 
    fn path(&self, key: &str) -> Result<PathBuf> {
135 
        if key.split('/').any(|p| p == ".." || p == ".") || key.starts_with('/') {
136 
            bail!("not a clean key: {key}");
137 
        }
138 
        Ok(self.root.join(key))
139 
    }
140 
}
141
142 
impl Destination for Directory {
143 
    fn exists(&self, key: &str) -> Result<bool> {
144 
        Ok(self.path(key)?.exists())
145 
    }
146 
    fn put(&self, key: &str, bytes: &[u8]) -> Result<()> {
147 
        let p = self.path(key)?;
148 
        std::fs::create_dir_all(p.parent().context("no parent")?)?;
149 
        let tmp = p.with_extension(format!("part-{}", std::process::id()));
150 
        std::fs::write(&tmp, bytes)?;
151 
        std::fs::rename(&tmp, &p)?;
152 
        Ok(())
153 
    }
154 
    fn get(&self, key: &str) -> Result<Vec<u8>> {
155 
        let p = self.path(key)?;
156 
        std::fs::read(&p).with_context(|| format!("read {}", p.display()))
157 
    }
158 
    fn list(&self, prefix: &str) -> Result<Vec<String>> {
159 
        let p = self.path(prefix)?;
160 
        let mut out: Vec<String> = match std::fs::read_dir(&p) {
161 
            Ok(d) => d.flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect(),
162 
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
163 
            Err(e) => return Err(e.into()),
164 
        };
165 
        out.sort();
166 
        Ok(out)
167 
    }
168 
    fn remove(&self, key: &str) -> Result<()> {
169 
        let p = self.path(key)?;
170 
        if p.is_dir() {
171 
            std::fs::remove_dir_all(p)?;
172 
        } else if p.exists() {
173 
            std::fs::remove_file(p)?;
174 
        }
175 
        Ok(())
176 
    }
177 
}
178
179 
/// A Cloud Storage bucket, reached keylessly (docs/backups.md §3, §6):
180 
/// the hub mints a short token as `backup:OWNER`, Google's STS exchanges
181 
/// it for an access token through the owner's workload identity
182 
/// provider, and that writes the bucket. No key exists anywhere; the
183 
/// access token lasts an hour and is fetched again before it runs out.
184 
pub struct Gcs {
185 
    bucket: String,
186 
    /// Keys go under this, when set.
187 
    prefix: String,
188 
    storage: String,
189 
    sts: String,
190 
    /// The provider's resource name, `//iam.googleapis.com/projects/…`.
191 
    provider: String,
192 
    subject: String,
193 
    app: Shared,
194 
    client: reqwest::Client,
195 
    rt: tokio::runtime::Handle,
196 
    token: std::sync::Mutex<Option<(String, i64)>>,
197 
}
198
199 
impl Gcs {
200 
    pub fn new(app: &Shared, s: &Setting) -> Gcs {
201 
        Gcs {
202 
            bucket: s.target.clone(),
203 
            prefix: s.prefix.trim_matches('/').to_string(),
204 
            storage: app.config.backups.storage_url.trim_end_matches('/').to_string(),
205 
            sts: app.config.backups.sts_url.trim_end_matches('/').to_string(),
206 
            provider: s.provider.clone(),
207 
            subject: format!("backup:{}", s.owner),
208 
            app: app.clone(),
209 
            client: reqwest::Client::builder().timeout(std::time::Duration::from_secs(300)).build().unwrap_or_default(),
210 
            rt: tokio::runtime::Handle::current(),
211 
            token: std::sync::Mutex::new(None),
212 
        }
213 
    }
214
215 
    fn name(&self, key: &str) -> String {
216 
        if self.prefix.is_empty() {
217 
            key.to_string()
218 
        } else {
219 
            format!("{}/{key}", self.prefix)
220 
        }
221 
    }
222
223 
    /// An access token, exchanged for a fresh one of the hub's when the
224 
    /// last is within five minutes of running out.
225 
    fn access(&self) -> Result<String> {
226 
        let now = db::now_ms();
227 
        if let Some((t, until)) = self.token.lock().unwrap().as_ref() {
228 
            if *until - now > 300_000 {
229 
                return Ok(t.clone());
230 
            }
231 
        }
232 
        let (token, expires_in) = self.rt.block_on(async {
233 
            let issuer = self.app.oidc().await?;
234 
            // Google's default audience for a provider: its name over https.
235 
            let audience = format!("https:{}", self.provider);
236 
            let jwt = issuer.mint_for(self.app.config.base_url(), &self.subject, &audience, db::now_ms(), 600);
237 
            let res = self
238 
                .client
239 
                .post(format!("{}/v1/token", self.sts))
240 
                .json(&serde_json::json!({
241 
                    "grantType": "urn:ietf:params:oauth:grant-type:token-exchange",
242 
                    "audience": self.provider,
243 
                    "scope": "https://www.googleapis.com/auth/devstorage.read_write",
244 
                    "requestedTokenType": "urn:ietf:params:oauth:token-type:access_token",
245 
                    "subjectToken": jwt,
246 
                    "subjectTokenType": "urn:ietf:params:oauth:token-type:jwt",
247 
                }))
248 
                .send()
249 
                .await?;
250 
            let status = res.status();
251 
            let body: serde_json::Value = res.json().await.unwrap_or_default();
252 
            if !status.is_success() {
253 
                bail!(
254 
                    "Google's token exchange refused {}: {} (check the provider's issuer is this hub's address and its attribute condition admits backup:)",
255 
                    self.subject,
256 
                    body["error_description"].as_str().or(body["error"].as_str()).unwrap_or("no reason given")
257 
                );
258 
            }
259 
            let token = body["access_token"].as_str().context("the token exchange gave no access token")?.to_string();
260 
            anyhow::Ok((token, body["expires_in"].as_i64().unwrap_or(3600)))
261 
        })?;
262 
        *self.token.lock().unwrap() = Some((token.clone(), now + expires_in * 1000));
263 
        Ok(token)
264 
    }
265
266 
    fn object_url(&self, key: &str) -> Result<reqwest::Url> {
267 
        let mut u = reqwest::Url::parse(&format!("{}/storage/v1/b/{}/o", self.storage, self.bucket))?;
268 
        u.path_segments_mut().map_err(|_| anyhow::anyhow!("bad storage URL"))?.push(&self.name(key));
269 
        Ok(u)
270 
    }
271
272 
    fn refused(&self, what: &str, status: reqwest::StatusCode, body: &str) -> anyhow::Error {
273 
        let hint = if status == reqwest::StatusCode::FORBIDDEN || status == reqwest::StatusCode::UNAUTHORIZED {
274 
            format!(" (grant roles/storage.objectAdmin on gs://{} to the principal …/subject/{})", self.bucket, self.subject)
275 
        } else {
276 
            String::new()
277 
        };
278 
        anyhow::anyhow!("GCS {what}: {status}{hint}: {}", body.chars().take(200).collect::<String>())
279 
    }
280
281 
    /// Every object name under `full` (no delimiter), for removal.
282 
    fn all_under(&self, full: &str) -> Result<Vec<String>> {
283 
        let mut out = Vec::new();
284 
        let mut page: Option<String> = None;
285 
        loop {
286 
            let token = self.access()?;
287 
            let mut u = reqwest::Url::parse(&format!("{}/storage/v1/b/{}/o", self.storage, self.bucket))?;
288 
            u.query_pairs_mut().append_pair("prefix", full).append_pair("fields", "items(name),nextPageToken");
289 
            if let Some(p) = &page {
290 
                u.query_pairs_mut().append_pair("pageToken", p);
291 
            }
292 
            let body: serde_json::Value = self.rt.block_on(async {
293 
                let res = self.client.get(u).bearer_auth(&token).send().await?;
294 
                let status = res.status();
295 
                let text = res.text().await?;
296 
                if !status.is_success() {
297 
                    return Err(self.refused("list", status, &text));
298 
                }
299 
                anyhow::Ok(serde_json::from_str(&text)?)
300 
            })?;
301 
            out.extend(body["items"].as_array().into_iter().flatten().filter_map(|i| i["name"].as_str().map(String::from)));
302 
            match body["nextPageToken"].as_str() {
303 
                Some(p) => page = Some(p.to_string()),
304 
                None => return Ok(out),
305 
            }
306 
        }
307 
    }
308 
}
309
310 
impl Destination for Gcs {
311 
    fn exists(&self, key: &str) -> Result<bool> {
312 
        let token = self.access()?;
313 
        let mut u = self.object_url(key)?;
314 
        u.query_pairs_mut().append_pair("fields", "name");
315 
        self.rt.block_on(async {
316 
            let res = self.client.get(u).bearer_auth(&token).send().await?;
317 
            match res.status() {
318 
                s if s.is_success() => Ok(true),
319 
                reqwest::StatusCode::NOT_FOUND => Ok(false),
320 
                s => Err(self.refused("read", s, &res.text().await.unwrap_or_default())),
321 
            }
322 
        })
323 
    }
324 
    fn put(&self, key: &str, bytes: &[u8]) -> Result<()> {
325 
        let token = self.access()?;
326 
        let mut u = reqwest::Url::parse(&format!("{}/upload/storage/v1/b/{}/o", self.storage, self.bucket))?;
327 
        u.query_pairs_mut().append_pair("uploadType", "media").append_pair("name", &self.name(key));
328 
        let body = bytes.to_vec();
329 
        self.rt.block_on(async {
330 
            let res = self.client.post(u).bearer_auth(&token).header("content-type", "application/octet-stream").body(body).send().await?;
331 
            let status = res.status();
332 
            if !status.is_success() {
333 
                return Err(self.refused("write", status, &res.text().await.unwrap_or_default()));
334 
            }
335 
            Ok(())
336 
        })
337 
    }
338 
    fn get(&self, key: &str) -> Result<Vec<u8>> {
339 
        let token = self.access()?;
340 
        let mut u = self.object_url(key)?;
341 
        u.query_pairs_mut().append_pair("alt", "media");
342 
        self.rt.block_on(async {
343 
            let res = self.client.get(u).bearer_auth(&token).send().await?;
344 
            let status = res.status();
345 
            if !status.is_success() {
346 
                return Err(self.refused(&format!("read {key}"), status, &res.text().await.unwrap_or_default()));
347 
            }
348 
            Ok(res.bytes().await?.to_vec())
349 
        })
350 
    }
351 
    fn list(&self, prefix: &str) -> Result<Vec<String>> {
352 
        let full = format!("{}/", self.name(prefix.trim_end_matches('/')));
353 
        let mut out = std::collections::BTreeSet::new();
354 
        let mut page: Option<String> = None;
355 
        loop {
356 
            let token = self.access()?;
357 
            let mut u = reqwest::Url::parse(&format!("{}/storage/v1/b/{}/o", self.storage, self.bucket))?;
358 
            u.query_pairs_mut()
359 
                .append_pair("prefix", &full)
360 
                .append_pair("delimiter", "/")
361 
                .append_pair("fields", "items(name),prefixes,nextPageToken");
362 
            if let Some(p) = &page {
363 
                u.query_pairs_mut().append_pair("pageToken", p);
364 
            }
365 
            let body: serde_json::Value = self.rt.block_on(async {
366 
                let res = self.client.get(u).bearer_auth(&token).send().await?;
367 
                let status = res.status();
368 
                let text = res.text().await?;
369 
                if !status.is_success() {
370 
                    return Err(self.refused("list", status, &text));
371 
                }
372 
                anyhow::Ok(serde_json::from_str(&text)?)
373 
            })?;
374 
            for n in body["items"].as_array().into_iter().flatten().filter_map(|i| i["name"].as_str()) {
375 
                out.insert(n.trim_start_matches(&full).to_string());
376 
            }
377 
            for p in body["prefixes"].as_array().into_iter().flatten().filter_map(|p| p.as_str()) {
378 
                out.insert(p.trim_start_matches(&full).trim_end_matches('/').to_string());
379 
            }
380 
            match body["nextPageToken"].as_str() {
381 
                Some(p) => page = Some(p.to_string()),
382 
                None => return Ok(out.into_iter().filter(|n| !n.is_empty()).collect()),
383 
            }
384 
        }
385 
    }
386 
    fn remove(&self, key: &str) -> Result<()> {
387 
        let mut names = self.all_under(&format!("{}/", self.name(key)))?;
388 
        names.push(self.name(key));
389 
        for n in names {
390 
            let token = self.access()?;
391 
            let mut u = reqwest::Url::parse(&format!("{}/storage/v1/b/{}/o", self.storage, self.bucket))?;
392 
            u.path_segments_mut().map_err(|_| anyhow::anyhow!("bad storage URL"))?.push(&n);
393 
            self.rt.block_on(async {
394 
                let res = self.client.delete(u).bearer_auth(&token).send().await?;
395 
                match res.status() {
396 
                    s if s.is_success() || s == reqwest::StatusCode::NOT_FOUND => Ok(()),
397 
                    s => Err(self.refused("delete", s, &res.text().await.unwrap_or_default())),
398 
                }
399 
            })?;
400 
        }
401 
        Ok(())
402 
    }
403 
}
404
405 
/// Check a destination end to end: write a probe, read it back, remove it.
406 
pub fn probe(dest: &dyn Destination) -> Result<()> {
407 
    let marker = format!("probe {}", db::now_ms());
408 
    dest.put(".oxyhub-probe", marker.as_bytes())?;
409 
    if dest.get(".oxyhub-probe")? != marker.as_bytes() {
410 
        bail!("the probe read back differently");
411 
    }
412 
    dest.remove(".oxyhub-probe")
413 
}
414
415 
/// The owner's destination, from its setting and the site's rules.
416 
pub fn destination(app: &Shared, s: &Setting) -> Result<Box<dyn Destination>> {
417 
    match s.destination.as_str() {
418 
        "dir" => {
419 
            let allowed = &app.config.backups.directories;
420 
            let target = PathBuf::from(&s.target);
421 
            if !allowed.contains(&target) {
422 
                bail!("{} is not a backup directory this hub allows (hub.toml [backups] directories)", s.target);
423 
            }
424 
            let mut root = target.join(&s.owner);
425 
            if !s.prefix.is_empty() {
426 
                root = root.join(s.prefix.trim_matches('/'));
427 
            }
428 
            Ok(Box::new(Directory { root }))
429 
        }
430 
        "gcs" => {
431 
            if s.target.is_empty() || s.provider.is_empty() {
432 
                bail!("a GCS destination needs its bucket and its workload identity provider");
433 
            }
434 
            Ok(Box::new(Gcs::new(app, s)))
435 
        }
436 
        other => bail!("unknown destination {other:?}"),
437 
    }
438 
}
439
440 
// ---------------------------------------------------------------- what
441
442 
/// One file of a repository's store, in a manifest.
443 
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
444 
pub struct File {
445 
    pub path: String,
446 
    pub size: u64,
447 
    /// Its modification time in nanoseconds: unchanged size and time mean
448 
    /// unchanged content, so it is not hashed again.
449 
    pub mtime: i128,
450 
    pub hash: String,
451 
}
452
453 
/// A repository at one run.
454 
#[derive(Clone, Debug, Serialize, Deserialize)]
455 
pub struct Manifest {
456 
    pub owner: String,
457 
    pub name: String,
458 
    /// `repos` or `wikis`.
459 
    pub kind: String,
460 
    pub files: Vec<File>,
461 
}
462
463 
fn blob_key(hash: &str) -> String {
464 
    format!("blobs/{}/{hash}", &hash[..2])
465 
}
466
467 
/// The large files that never change once written come first.
468 
fn immutable(path: &str) -> bool {
469 
    path.starts_with(".oxy/objects/") && (path.contains("/pack-") || path.contains("/loose/"))
470 
}
471
472 
#[derive(Default)]
473 
struct Sent {
474 
    files: i64,
475 
    sent: i64,
476 
    bytes: i64,
477 
}
478
479 
/// Walk `dir`, sending each file the destination lacks; `known` says which
480 
/// paths are unchanged since the last run (and their hashes).
481 
fn copy_dir(dest: &dyn Destination, dir: &Path, known: &HashMap<String, File>, only_immutable: bool, sent: &mut Sent) -> Result<Vec<File>> {
482 
    let mut out = Vec::new();
483 
    let mut stack = vec![dir.to_path_buf()];
484 
    while let Some(d) = stack.pop() {
485 
        let entries = match std::fs::read_dir(&d) {
486 
            Ok(e) => e,
487 
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
488 
            Err(e) => return Err(e.into()),
489 
        };
490 
        for e in entries.flatten() {
491 
            let p = e.path();
492 
            let ft = e.file_type()?;
493 
            if ft.is_dir() {
494 
                stack.push(p);
495 
                continue;
496 
            }
497 
            if !ft.is_file() {
498 
                continue;
499 
            }
500 
            let rel = p.strip_prefix(dir).unwrap_or(&p).to_string_lossy().replace('\\', "/");
501 
            if only_immutable && !immutable(&rel) {
502 
                continue;
503 
            }
504 
            // A file packed away while the walk ran is simply gone.
505 
            let Ok(md) = std::fs::metadata(&p) else { continue };
506 
            let mtime =
507 
                md.modified().ok().and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()).map(|d| d.as_nanos() as i128).unwrap_or(0);
508 
            let size = md.len();
509 
            sent.files += 1;
510 
            if let Some(k) = known.get(&rel).filter(|k| k.size == size && k.mtime == mtime) {
511 
                out.push(k.clone());
512 
                continue;
513 
            }
514 
            let Ok(bytes) = std::fs::read(&p) else { continue };
515 
            let hash = blake3::hash(&bytes).to_hex().to_string();
516 
            let key = blob_key(&hash);
517 
            if !dest.exists(&key)? {
518 
                dest.put(&key, &bytes)?;
519 
                sent.sent += 1;
520 
                sent.bytes += bytes.len() as i64;
521 
            }
522 
            out.push(File { path: rel, size, mtime, hash });
523 
        }
524 
    }
525 
    out.sort_by(|a, b| a.path.cmp(&b.path));
526 
    Ok(out)
527 
}
528
529 
// ---------------------------------------------------------------- records
530
531 
/// Tables and columns never written: secrets, credentials, sessions.
532 
fn private_table(t: &str) -> bool {
533 
    ["secret", "token", "session", "password", "totp", "recovery", "sqlx"].iter().any(|w| t.contains(w))
534 
}
535
536 
fn private_column(c: &str) -> bool {
537 
    ["secret", "token", "password", "salt", "totp", "sealed", "hash_key"].iter().any(|w| c.contains(w))
538 
}
539
540 
async fn tables(pool: &Pool) -> Result<Vec<String>> {
541 
    let mut t: Vec<String> =
542 
        sqlx::query_scalar("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").fetch_all(pool).await?;
543 
    t.retain(|t| !private_table(t));
544 
    t.sort();
545 
    Ok(t)
546 
}
547
548 
async fn columns(pool: &Pool, table: &str) -> Result<Vec<String>> {
549 
    Ok(sqlx::query_scalar(&format!("SELECT name FROM pragma_table_info('{table}')")).fetch_all(pool).await?)
550 
}
551
552 
/// The rows of `table` matching `cond`, as a JSON array, private columns
553 
/// left out and blobs as hex.
554 
async fn rows(pool: &Pool, table: &str, cond: &str) -> Result<serde_json::Value> {
555 
    let cols = columns(pool, table).await?;
556 
    let fields: Vec<String> = cols
557 
        .iter()
558 
        .filter(|c| !private_column(c))
559 
        .map(|c| format!("'{c}', CASE WHEN typeof(\"{c}\") = 'blob' THEN hex(\"{c}\") ELSE \"{c}\" END"))
560 
        .collect();
561 
    if fields.is_empty() {
562 
        return Ok(serde_json::json!([]));
563 
    }
564 
    let sql = format!("SELECT json_group_array(json_object({})) FROM \"{table}\" WHERE {cond}", fields.join(", "));
565 
    let text: String = sqlx::query_scalar(&sql).fetch_one(pool).await?;
566 
    Ok(serde_json::from_str(&text)?)
567 
}
568
569 
/// Every row about something: the tables with column `key` equal to `id`,
570 
/// then, three levels deep, the tables whose foreign keys point at rows
571 
/// already taken (an issue's comments, a proposal's reviews and their
572 
/// threads).
573 
async fn records(pool: &Pool, key: &str, id: i64) -> Result<serde_json::Value> {
574 
    let all = tables(pool).await?;
575 
    let mut conds: Vec<(String, String)> = Vec::new();
576 
    for t in &all {
577 
        if columns(pool, t).await?.iter().any(|c| c == key) {
578 
            conds.push((t.clone(), format!("\"{key}\" = {id}")));
579 
        }
580 
    }
581 
    for _ in 0..3 {
582 
        let mut more = Vec::new();
583 
        for t in &all {
584 
            if conds.iter().any(|(c, _)| c == t) || more.iter().any(|(c, _): &(String, String)| c == t) {
585 
                continue;
586 
            }
587 
            let fks: Vec<(String, String, String)> =
588 
                sqlx::query_as(&format!("SELECT \"table\", \"from\", \"to\" FROM pragma_foreign_key_list('{t}')")).fetch_all(pool).await?;
589 
            if let Some((parent, from, to, pcond)) = fks
590 
                .iter()
591 
                .find_map(|(p, f, to)| conds.iter().find(|(c, _)| c == p).map(|(_, pc)| (p.clone(), f.clone(), to.clone(), pc.clone())))
592 
            {
593 
                more.push((t.clone(), format!("\"{from}\" IN (SELECT \"{to}\" FROM \"{parent}\" WHERE {pcond})")));
594 
            }
595 
        }
596 
        if more.is_empty() {
597 
            break;
598 
        }
599 
        conds.extend(more);
600 
    }
601 
    let mut out = serde_json::Map::new();
602 
    for (t, cond) in conds {
603 
        let r = rows(pool, &t, &cond).await?;
604 
        if r.as_array().is_some_and(|a| !a.is_empty()) {
605 
            out.insert(t, r);
606 
        }
607 
    }
608 
    Ok(serde_json::Value::Object(out))
609 
}
610
611 
/// The key of a run's records naming the accounts, teams and organisations
612 
/// their rows point at, by id: `{"accounts": {"7": "andy"}, …}`.
613 
const NAMES: &str = "_names";
614
615 
/// Tables a restore leaves out: the search index (rebuilt from what is
616 
/// restored), and what is passing or personal rather than the
617 
/// repository's: inboxes, webhook deliveries, runners' declines.
618 
const NOT_RESTORED: [&str; 5] = ["search", "notifications", "deliveries", "flow_declines", NAMES];
619
620 
async fn fk_list(pool: &Pool, table: &str) -> Result<Vec<(String, String, String)>> {
621 
    Ok(sqlx::query_as(&format!("SELECT \"table\", \"from\", \"to\" FROM pragma_foreign_key_list('{table}')")).fetch_all(pool).await?)
622 
}
623
624 
/// The names behind the account, team and organisation ids that `rec`'s
625 
/// rows point at, so a restore on another hub finds them again by name.
626 
async fn names_of(pool: &Pool, rec: &serde_json::Value) -> Result<serde_json::Value> {
627 
    let mut ids: HashMap<String, std::collections::BTreeSet<i64>> = HashMap::new();
628 
    for (table, rows) in rec.as_object().into_iter().flatten() {
629 
        let Some(rows) = rows.as_array() else { continue };
630 
        for (parent, from, _) in fk_list(pool, table).await? {
631 
            if ["accounts", "teams", "organisations"].contains(&parent.as_str()) {
632 
                ids.entry(parent).or_default().extend(rows.iter().filter_map(|r| r[&from].as_i64()));
633 
            }
634 
        }
635 
    }
636 
    let mut out = serde_json::Map::new();
637 
    for (kind, sql) in [
638 
        ("accounts", "SELECT id, username FROM accounts WHERE id = ?"),
639 
        ("organisations", "SELECT id, name FROM organisations WHERE id = ?"),
640 
        ("teams", "SELECT t.id, o.name || '/' || t.name FROM teams t JOIN organisations o ON o.id = t.org_id WHERE t.id = ?"),
641 
    ] {
642 
        let mut m = serde_json::Map::new();
643 
        for id in ids.get(kind).into_iter().flatten() {
644 
            if let Some((id, name)) = sqlx::query_as::<_, (i64, String)>(sql).bind(id).fetch_optional(pool).await? {
645 
                m.insert(id.to_string(), name.into());
646 
            }
647 
        }
648 
        out.insert(kind.into(), m.into());
649 
    }
650 
    Ok(out.into())
651 
}
652
653 
/// The id `name` has on this hub as one of `_names`' kinds.
654 
async fn id_by_name(pool: &Pool, kind: &str, name: &str) -> Result<Option<i64>> {
655 
    Ok(match kind {
656 
        "accounts" => sqlx::query_scalar("SELECT id FROM accounts WHERE username = ?").bind(name).fetch_optional(pool).await?,
657 
        "organisations" => sqlx::query_scalar("SELECT id FROM organisations WHERE name = ?").bind(name).fetch_optional(pool).await?,
658 
        "teams" => match name.split_once('/') {
659 
            Some((org, team)) => {
660 
                sqlx::query_scalar("SELECT t.id FROM teams t JOIN organisations o ON o.id = t.org_id WHERE o.name = ? AND t.name = ?")
661 
                    .bind(org)
662 
                    .bind(team)
663 
                    .fetch_optional(pool)
664 
                    .await?
665 
            }
666 
            None => None,
667 
        },
668 
        _ => None,
669 
    })
670 
}
671
672 
/// One column of a table on this hub.
673 
struct Col {
674 
    name: String,
675 
    ty: String,
676 
    notnull: bool,
677 
    default: Option<String>,
678 
    pk: i64,
679 
}
680
681 
async fn table_cols(pool: &Pool, table: &str) -> Result<Vec<Col>> {
682 
    let rows: Vec<(String, String, i64, Option<String>, i64)> =
683 
        sqlx::query_as(&format!("SELECT name, type, \"notnull\", dflt_value, pk FROM pragma_table_info('{table}')"))
684 
            .fetch_all(pool)
685 
            .await?;
686 
    Ok(rows.into_iter().map(|(name, ty, notnull, default, pk)| Col { name, ty, notnull: notnull != 0, default, pk }).collect())
687 
}
688
689 
/// Bind one value of a backed-up row: blobs were written as hex.
690 
fn bind_json<'q>(
691 
    q: sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>>,
692 
    v: serde_json::Value,
693 
    blob: bool,
694 
) -> sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>> {
695 
    match v {
696 
        serde_json::Value::Null => q.bind(None::<i64>),
697 
        serde_json::Value::Bool(b) => q.bind(b as i64),
698 
        serde_json::Value::Number(n) => match n.as_i64() {
699 
            Some(i) => q.bind(i),
700 
            None => q.bind(n.as_f64().unwrap_or(0.0)),
701 
        },
702 
        serde_json::Value::String(s) if blob => q.bind(hex::decode(&s).unwrap_or_else(|_| s.into_bytes())),
703 
        serde_json::Value::String(s) => q.bind(s),
704 
        other => q.bind(other.to_string()),
705 
    }
706 
}
707
708 
/// Restore one repository's records from a run into this hub, the
709 
/// repository being `repo` here and `old` in the backup (docs/backups.md
710 
/// §7): its issues, proposals, labels, milestones, releases and their
711 
/// assets, checks, Flow runs and the rest, each table after the tables it
712 
/// points at, every id mapped to its new one and every account, team and
713 
/// organisation found again by name. A row pointing at an account this
714 
/// hub lacks keeps a blank reference where it may, and is left out where
715 
/// it may not. What needs re-entering (webhook secrets, never backed up)
716 
/// and what was left out come back as notes.
717 
async fn restore_records(
718 
    app: &Shared,
719 
    dest: &std::sync::Arc<dyn Destination>,
720 
    rec: &serde_json::Value,
721 
    old: i64,
722 
    repo: i64,
723 
) -> Result<Vec<String>> {
724 
    let pool = &app.db;
725 
    let obj = rec.as_object().context("the records are not an object")?;
726 
    let mut by_name: HashMap<(String, i64), Option<i64>> = HashMap::new();
727 
    for (kind, m) in obj.get(NAMES).and_then(|n| n.as_object()).into_iter().flatten() {
728 
        for (id, name) in m.as_object().into_iter().flatten() {
729 
            let (Ok(id), Some(name)) = (id.parse::<i64>(), name.as_str()) else { continue };
730 
            by_name.insert((kind.clone(), id), id_by_name(pool, kind, name).await?);
731 
        }
732 
    }
733 
    let here: std::collections::HashSet<String> = tables(pool).await?.into_iter().collect();
734 
    let mut notes = Vec::new();
735 
    let mut todo: Vec<String> = Vec::new();
736 
    for t in obj.keys().filter(|t| t.as_str() != "repositories" && !NOT_RESTORED.contains(&t.as_str())) {
737 
        if here.contains(t) {
738 
            todo.push(t.clone());
739 
        } else {
740 
            notes.push(format!("{t}: not a table on this hub, left out"));
741 
        }
742 
    }
743 
    let mut fks: HashMap<String, Vec<(String, String, String)>> = HashMap::new();
744 
    for t in &todo {
745 
        fks.insert(t.clone(), fk_list(pool, t).await?);
746 
    }
747 
    // Each table after the tables of the records it points at.
748 
    let mut order = Vec::new();
749 
    while !todo.is_empty() {
750 
        let mut ready: Vec<String> =
751 
            todo.iter().filter(|t| fks[*t].iter().all(|(p, _, _)| p == *t || !todo.contains(p))).cloned().collect();
752 
        if ready.is_empty() {
753 
            ready.push(todo[0].clone());
754 
        }
755 
        todo.retain(|t| !ready.contains(t));
756 
        order.extend(ready);
757 
    }
758 
    let mut maps: HashMap<String, HashMap<i64, i64>> = HashMap::new();
759 
    maps.insert("repositories".into(), [(old, repo)].into());
760 
    let mut left_out: std::collections::BTreeMap<String, usize> = Default::default();
761 
    let mut tx = pool.begin().await?;
762 
    for table in &order {
763 
        let cols = table_cols(pool, table).await?;
764 
        let keys: Vec<&str> = cols.iter().filter(|c| c.pk > 0).map(|c| c.name.as_str()).collect();
765 
        let by_id = keys == ["id"] && cols.iter().any(|c| c.name == "id" && c.ty.eq_ignore_ascii_case("INTEGER"));
766 
        let verb = if keys == ["repo_id"] {
767 
            "INSERT OR REPLACE"
768 
        } else if by_id {
769 
            "INSERT"
770 
        } else {
771 
            "INSERT OR IGNORE"
772 
        };
773 
        let mut rows: Vec<&serde_json::Map<String, serde_json::Value>> =
774 
            obj[table].as_array().into_iter().flatten().filter_map(|r| r.as_object()).collect();
775 
        rows.sort_by_key(|r| r.get("id").and_then(|v| v.as_i64()).unwrap_or(0));
776 
        'row: for row in rows {
777 
            let (mut names, mut values) = (Vec::new(), Vec::new());
778 
            for c in &cols {
779 
                if by_id && c.name == "id" {
780 
                    continue;
781 
                }
782 
                let mut v = row.get(&c.name).cloned().filter(|v| !v.is_null());
783 
                if let Some((parent, _, _)) = fks[table].iter().find(|(_, from, _)| *from == c.name) {
784 
                    if let Some(was) = v.as_ref().and_then(|v| v.as_i64()) {
785 
                        let now = match parent.as_str() {
786 
                            "accounts" | "teams" | "organisations" => by_name.get(&(parent.clone(), was)).copied().flatten(),
787 
                            p => maps.get(p).and_then(|m| m.get(&was)).copied(),
788 
                        };
789 
                        if now.is_none() && c.notnull {
790 
                            *left_out.entry(table.clone()).or_default() += 1;
791 
                            continue 'row;
792 
                        }
793 
                        v = now.map(serde_json::Value::from);
794 
                    }
795 
                }
796 
                let blob = c.ty.to_ascii_uppercase().contains("BLOB");
797 
                match v {
798 
                    Some(v) => {
799 
                        names.push(c.name.clone());
800 
                        values.push((v, blob));
801 
                    }
802 
                    // Never backed up (a secret): blank here, to enter again.
803 
                    None if c.notnull && c.default.is_none() => {
804 
                        names.push(c.name.clone());
805 
                        let blank = if c.ty.to_ascii_uppercase().contains("INT") { serde_json::json!(0) } else { serde_json::json!("") };
806 
                        values.push((blank, false));
807 
                    }
808 
                    None => {}
809 
                }
810 
            }
811 
            let sql = format!(
812 
                "{verb} INTO \"{table}\" ({}) VALUES ({})",
813 
                names.iter().map(|n| format!("\"{n}\"")).collect::<Vec<_>>().join(", "),
814 
                vec!["?"; names.len()].join(", ")
815 
            );
816 
            let mut q = sqlx::query(&sql);
817 
            for (v, blob) in values {
818 
                q = bind_json(q, v, blob);
819 
            }
820 
            let done = q.execute(&mut *tx).await.with_context(|| format!("restore a row of {table}"))?;
821 
            if let (true, Some(was)) = (by_id, row.get("id").and_then(|v| v.as_i64())) {
822 
                maps.entry(table.clone()).or_default().insert(was, done.last_insert_rowid());
823 
            }
824 
        }
825 
    }
826 
    tx.commit().await?;
827 
    for (table, n) in left_out {
828 
        notes.push(format!("{table}: {n} row(s) left out: they name accounts or rows this hub lacks"));
829 
    }
830 
    // Search finds what came back.
831 
    for (kind, table) in [("issue", "issues"), ("proposal", "proposals")] {
832 
        let found: Vec<(i64, i64, String, String)> =
833 
            sqlx::query_as(&format!("SELECT id, number, title, body FROM {table} WHERE repo_id = ?")).bind(repo).fetch_all(pool).await?;
834 
        for (id, number, title, body) in found {
835 
            db::notify::index(pool, kind, repo, id, number, &title, &body).await?;
836 
        }
837 
    }
838 
    // Webhook secrets were never written: they are entered again.
839 
    let hooks: Vec<String> =
840 
        sqlx::query_scalar("SELECT url FROM hooks WHERE repo_id = ? AND secret = ''").bind(repo).fetch_all(pool).await?;
841 
    for url in hooks {
842 
        notes.push(format!("webhook {url}: enter its secret again (Settings › Webhooks)"));
843 
    }
844 
    // Release assets, each checked against its name.
845 
    for hash in obj.get("release_assets").and_then(|a| a.as_array()).into_iter().flatten().filter_map(|a| a["hash"].as_str()) {
846 
        if hash.len() != 64 || !hash.bytes().all(|b| b.is_ascii_hexdigit()) {
847 
            continue;
848 
        }
849 
        let path = crate::web::releases::blob_path(app, hash);
850 
        if path.exists() {
851 
            continue;
852 
        }
853 
        let got = tokio::task::block_in_place(|| dest.get(&blob_key(hash)));
854 
        match got {
855 
            Ok(bytes) if blake3::hash(&bytes).to_hex().as_str() == hash => {
856 
                std::fs::create_dir_all(path.parent().context("no parent")?)?;
857 
                std::fs::write(&path, &bytes)?;
858 
            }
859 
            Ok(_) => notes.push(format!("release asset {}: the copy does not match its hash", &hash[..12])),
860 
            Err(_) => notes.push(format!("release asset {}: not in the backup", &hash[..12])),
861 
        }
862 
    }
863 
    Ok(notes)
864 
}
865
866 
// ---------------------------------------------------------------- a run
867
868 
fn run_name(ms: i64) -> String {
869 
    let t = time::OffsetDateTime::from_unix_timestamp_nanos(ms as i128 * 1_000_000).unwrap_or(time::OffsetDateTime::UNIX_EPOCH);
870 
    format!("{:04}{:02}{:02}T{:02}{:02}{:02}Z", t.year(), u8::from(t.month()), t.day(), t.hour(), t.minute(), t.second())
871 
}
872
873 
/// The last whole run's manifests, by `kind/name`.
874 
fn last_manifests(dest: &dyn Destination) -> HashMap<String, HashMap<String, File>> {
875 
    let mut out = HashMap::new();
876 
    let Ok(at) = dest.get("LATEST").map(|b| String::from_utf8_lossy(&b).trim().to_string()) else { return out };
877 
    for name in dest.list(&format!("runs/{at}/repos")).unwrap_or_default() {
878 
        if let Ok(m) = dest.get(&format!("runs/{at}/repos/{name}")).and_then(|b| Ok(serde_json::from_slice::<Manifest>(&b)?)) {
879 
            out.insert(format!("{}/{}", m.kind, m.name), m.files.into_iter().map(|f| (f.path.clone(), f)).collect());
880 
        }
881 
    }
882 
    out
883 
}
884
885 
/// Back up `owner` now, as its setting says. The run is recorded whatever
886 
/// happens; a failure notifies the owner once until a run succeeds.
887 
pub async fn run(app: &Shared, owner: &str, trigger: &str) -> Result<Run> {
888 
    let s = setting(&app.db, owner).await?.with_context(|| format!("{owner} has no backup set up"))?;
889 
    let started = db::now_ms();
890 
    let at = run_name(started);
891 
    let id: i64 =
892 
        sqlx::query_scalar("INSERT INTO backup_runs (owner, started_ms, status, at, trigger) VALUES (?, ?, 'running', ?, ?) RETURNING id")
893 
            .bind(owner)
894 
            .bind(started)
895 
            .bind(&at)
896 
            .bind(trigger)
897 
            .fetch_one(&app.db)
898 
            .await?;
899 
    let outcome = run_into(app, &s, &at).await;
900 
    let (status, detail, sent) = match outcome {
901 
        Ok((sent, problems)) if problems.is_empty() => ("ok", String::new(), sent),
902 
        Ok((sent, problems)) => ("partial", problems.join("; "), sent),
903 
        Err(e) => ("failed", format!("{e:#}"), (0, Sent::default())),
904 
    };
905 
    sqlx::query("UPDATE backup_runs SET finished_ms = ?, status = ?, detail = ?, repos = ?, files = ?, sent = ?, bytes = ? WHERE id = ?")
906 
        .bind(db::now_ms())
907 
        .bind(status)
908 
        .bind(&detail)
909 
        .bind(sent.0)
910 
        .bind(sent.1.files)
911 
        .bind(sent.1.sent)
912 
        .bind(sent.1.bytes)
913 
        .bind(id)
914 
        .execute(&app.db)
915 
        .await?;
916 
    if status != "ok" {
917 
        notify_failure(app, &s, status, &detail).await;
918 
    }
919 
    let r: Run = sqlx::query_as("SELECT * FROM backup_runs WHERE id = ?").bind(id).fetch_one(&app.db).await?;
920 
    Ok(r)
921 
}
922
923 
async fn run_into(app: &Shared, s: &Setting, at: &str) -> Result<((i64, Sent), Vec<String>)> {
924 
    let dest: std::sync::Arc<dyn Destination> = destination(app, s)?.into();
925 
    let known = {
926 
        let d = dest.clone();
927 
        tokio::task::spawn_blocking(move || last_manifests(d.as_ref())).await?
928 
    };
929 
    let mine: Vec<db::repos::RepoRow> = db::repos::list(&app.db).await?.into_iter().filter(|r| r.owner == s.owner).collect();
930 
    let mut problems = Vec::new();
931 
    let mut sent = Sent::default();
932 
    let mut repos = 0i64;
933 
    for row in &mine {
934 
        let Some(name) = RepoName::new(&row.owner, &row.name) else { continue };
935 
        for (kind, store) in [("repos", &app.repos), ("wikis", &app.wikis)] {
936 
            if !store.exists(&name) {
937 
                continue;
938 
            }
939 
            let dir = store.dir(&name);
940 
            let k = known.get(&format!("{kind}/{}", row.name)).cloned().unwrap_or_default();
941 
            // The large, unchanging files without the lock, then the whole
942 
            // walk again under the read lock: pushes wait for that moment.
943 
            let (d1, dir1, k1) = (dest.clone(), dir.clone(), k.clone());
944 
            let first = tokio::task::spawn_blocking(move || {
945 
                let mut s = Sent::default();
946 
                copy_dir(d1.as_ref(), &dir1, &k1, true, &mut s).map(|f| (f, s))
947 
            })
948 
            .await?;
949 
            let mut k2 = k;
950 
            if let Ok((files, _)) = &first {
951 
                for f in files {
952 
                    k2.insert(f.path.clone(), f.clone());
953 
                }
954 
            }
955 
            let guard = store.read_guard(&name).await;
956 
            let (d2, dir2) = (dest.clone(), dir.clone());
957 
            let second = tokio::task::spawn_blocking(move || {
958 
                let _guard = guard;
959 
                let mut s = Sent::default();
960 
                copy_dir(d2.as_ref(), &dir2, &k2, false, &mut s).map(|f| (f, s))
961 
            })
962 
            .await?;
963 
            match second {
964 
                Ok((files, s2)) => {
965 
                    let s1 = first.map(|(_, s)| s).unwrap_or_default();
966 
                    sent.files += s2.files;
967 
                    sent.sent += s1.sent + s2.sent;
968 
                    sent.bytes += s1.bytes + s2.bytes;
969 
                    let m = Manifest { owner: row.owner.clone(), name: row.name.clone(), kind: kind.into(), files };
970 
                    let key = format!("runs/{at}/repos/{kind}-{}.json", row.name);
971 
                    tokio::task::block_in_place(|| dest.put(&key, &serde_json::to_vec(&m)?))?;
972 
                    if kind == "repos" {
973 
                        repos += 1;
974 
                    }
975 
                }
976 
                Err(e) => problems.push(format!("{}/{} ({kind}): {e:#}", row.owner, row.name)),
977 
            }
978 
        }
979 
        let rec = async {
980 
            let mut r = records(&app.db, "repo_id", row.id).await?;
981 
            // The repository's own row, which restore recreates first, and
982 
            // the names behind the accounts, teams and organisations its
983 
            // rows point at, which another hub finds them by.
984 
            let own = rows(&app.db, "repositories", &format!("id = {}", row.id)).await?;
985 
            let names = names_of(&app.db, &r).await?;
986 
            if let Some(m) = r.as_object_mut() {
987 
                m.insert("repositories".into(), own);
988 
                m.insert(NAMES.into(), names);
989 
            }
990 
            anyhow::Ok(r)
991 
        };
992 
        match rec.await {
993 
            Ok(r) => {
994 
                tokio::task::block_in_place(|| dest.put(&format!("runs/{at}/records/{}.json", row.name), &serde_json::to_vec(&r)?))?;
995 
                // Release assets live beside the stores, named by their
996 
                // BLAKE3: each sent once, as the stores' files are.
997 
                for hash in r["release_assets"].as_array().into_iter().flatten().filter_map(|a| a["hash"].as_str()) {
998 
                    if hash.len() != 64 || !hash.bytes().all(|b| b.is_ascii_hexdigit()) {
999 
                        continue;
1000 
                    }
1001 
                    let path = crate::web::releases::blob_path(app, hash);
1002 
                    let key = blob_key(hash);
1003 
                    let put = tokio::task::block_in_place(|| -> Result<Option<i64>> {
1004 
                        if dest.exists(&key)? {
1005 
                            return Ok(None);
1006 
                        }
1007 
                        let bytes = std::fs::read(&path).with_context(|| format!("read release asset {}", &hash[..12]))?;
1008 
                        dest.put(&key, &bytes)?;
1009 
                        Ok(Some(bytes.len() as i64))
1010 
                    });
1011 
                    match put {
1012 
                        Ok(Some(n)) => {
1013 
                            sent.files += 1;
1014 
                            sent.sent += 1;
1015 
                            sent.bytes += n;
1016 
                        }
1017 
                        Ok(None) => sent.files += 1,
1018 
                        Err(e) => problems.push(format!("{}/{} release asset: {e:#}", row.owner, row.name)),
1019 
                    }
1020 
                }
1021 
            }
1022 
            Err(e) => problems.push(format!("{}/{} records: {e:#}", row.owner, row.name)),
1023 
        }
1024 
    }
1025 
    // The owner itself.
1026 
    let owner_rows = if s.kind == "org" {
1027 
        match db::accounts::org_by_name(&app.db, &s.owner).await? {
1028 
            Some(o) => records(&app.db, "org_id", o.id).await?,
1029 
            None => serde_json::json!({}),
1030 
        }
1031 
    } else {
1032 
        match db::accounts::account_by_name(&app.db, &s.owner).await? {
1033 
            Some(a) => records(&app.db, "account_id", a.id).await?,
1034 
            None => serde_json::json!({}),
1035 
        }
1036 
    };
1037 
    // The blocking calls of a destination (GCS drives its client on this
1038 
    // runtime) run where blocking is allowed.
1039 
    tokio::task::block_in_place(|| -> Result<()> {
1040 
        dest.put(&format!("runs/{at}/owner.json"), &serde_json::to_vec(&owner_rows)?)?;
1041 
        dest.put(&format!("runs/{at}/DONE"), at.as_bytes())?;
1042 
        dest.put("LATEST", at.as_bytes())?;
1043 
        prune(dest.as_ref(), s.keep_days)
1044 
    })?;
1045 
    Ok(((repos, sent), problems))
1046 
}
1047
1048 
/// Drop whole runs older than `keep_days`, and the blobs no kept run
1049 
/// names. The newest whole run is always kept.
1050 
pub fn prune(dest: &dyn Destination, keep_days: i64) -> Result<()> {
1051 
    let cutoff = run_name(db::now_ms() - keep_days.max(1) * 86_400_000);
1052 
    let runs = dest.list("runs")?;
1053 
    let whole: Vec<&String> = runs.iter().filter(|r| dest.exists(&format!("runs/{r}/DONE")).unwrap_or(false)).collect();
1054 
    let newest = whole.last().map(|s| s.to_string());
1055 
    let mut removed = false;
1056 
    for r in &runs {
1057 
        if r.as_str() < cutoff.as_str() && Some(r.to_string()) != newest {
1058 
            dest.remove(&format!("runs/{r}"))?;
1059 
            removed = true;
1060 
        }
1061 
    }
1062 
    if !removed {
1063 
        return Ok(());
1064 
    }
1065 
    let mut wanted = std::collections::HashSet::new();
1066 
    for r in dest.list("runs")? {
1067 
        for m in dest.list(&format!("runs/{r}/repos"))? {
1068 
            if let Ok(m) = dest.get(&format!("runs/{r}/repos/{m}")).and_then(|b| Ok(serde_json::from_slice::<Manifest>(&b)?)) {
1069 
                wanted.extend(m.files.into_iter().map(|f| f.hash));
1070 
            }
1071 
        }
1072 
    }
1073 
    for shard in dest.list("blobs")? {
1074 
        for b in dest.list(&format!("blobs/{shard}"))? {
1075 
            if !wanted.contains(&b) {
1076 
                dest.remove(&format!("blobs/{shard}/{b}"))?;
1077 
            }
1078 
        }
1079 
    }
1080 
    Ok(())
1081 
}
1082
1083 
async fn notify_failure(app: &Shared, s: &Setting, status: &str, detail: &str) {
1084 
    // Once per spell of failures: not when the run before failed too.
1085 
    let before: Option<String> =
1086 
        sqlx::query_scalar("SELECT status FROM backup_runs WHERE owner = ? AND finished_ms IS NOT NULL ORDER BY id DESC LIMIT 1 OFFSET 1")
1087 
            .bind(&s.owner)
1088 
            .fetch_optional(&app.db)
1089 
            .await
1090 
            .ok()
1091 
            .flatten();
1092 
    if before.as_deref().is_some_and(|b| b != "ok") {
1093 
        return;
1094 
    }
1095 
    let (who, url): (Vec<i64>, String) = if s.kind == "org" {
1096 
        match db::accounts::org_by_name(&app.db, &s.owner).await.ok().flatten() {
1097 
            Some(o) => (
1098 
                db::accounts::org_members(&app.db, o.id)
1099 
                    .await
1100 
                    .unwrap_or_default()
1101 
                    .into_iter()
1102 
                    .filter(|(_, r)| r == "owner")
1103 
                    .map(|(a, _)| a.id)
1104 
                    .collect(),
1105 
                format!("/orgs/{}/settings#backups", s.owner),
1106 
            ),
1107 
            None => return,
1108 
        }
1109 
    } else {
1110 
        match db::accounts::account_by_name(&app.db, &s.owner).await.ok().flatten() {
1111 
            Some(a) => (vec![a.id], "/settings/backups".into()),
1112 
            None => return,
1113 
        }
1114 
    };
1115 
    let title = format!("Backup of {} {status}: {}", s.owner, detail.chars().take(160).collect::<String>());
1116 
    for id in who {
1117 
        let _ = db::notify::add(
1118 
            &app.db,
1119 
            db::notify::New {
1120 
                account_id: id,
1121 
                repo_id: None,
1122 
                reason: db::notify::Reason::Participating,
1123 
                kind: "backup",
1124 
                title: &title,
1125 
                url: &url,
1126 
                actor: "",
1127 
            },
1128 
        )
1129 
        .await;
1130 
    }
1131 
}
1132
1133 
/// Run each owner's backup when it is due, one at a time. A run left
1134 
/// "running" by a restart is marked failed first.
1135 
pub async fn scheduler(app: Shared) {
1136 
    let _ = sqlx::query(
1137 
        "UPDATE backup_runs SET status = 'failed', detail = 'the hub restarted during the run', finished_ms = ? WHERE status = 'running'",
1138 
    )
1139 
    .bind(db::now_ms())
1140 
    .execute(&app.db)
1141 
    .await;
1142 
    loop {
1143 
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
1144 
        for s in due(&app.db).await.unwrap_or_default() {
1145 
            if let Err(e) = run(&app, &s.owner, "schedule").await {
1146 
                tracing::warn!("backup of {}: {e:#}", s.owner);
1147 
            }
1148 
        }
1149 
    }
1150 
}
1151
1152 
/// The owners whose next run is due now.
1153 
pub async fn due(pool: &Pool) -> sqlx::Result<Vec<Setting>> {
1154 
    let all: Vec<Setting> = sqlx::query_as("SELECT * FROM backup_settings WHERE every != 'off'").fetch_all(pool).await?;
1155 
    let now = db::now_ms();
1156 
    let mut out = Vec::new();
1157 
    for s in all {
1158 
        let Some(period) = s.period_s() else { continue };
1159 
        let last: Option<i64> =
1160 
            sqlx::query_scalar("SELECT MAX(started_ms) FROM backup_runs WHERE owner = ?").bind(&s.owner).fetch_one(pool).await?;
1161 
        if last.is_none_or(|l| now - l >= period * 1000) {
1162 
            out.push(s);
1163 
        }
1164 
    }
1165 
    Ok(out)
1166 
}
1167
1168 
/// Restore every repository of the whole run `at` (the latest when none)
1169 
/// from `dest` into `repos_root/OWNER/NAME` and `wikis_root/…`, refusing to
1170 
/// overwrite one that exists. Each file is checked against its hash.
1171 
pub fn restore(dest: &dyn Destination, at: Option<&str>, repos_root: &Path, wikis_root: &Path) -> Result<Vec<String>> {
1172 
    let at = match at {
1173 
        Some(a) => a.to_string(),
1174 
        None => String::from_utf8_lossy(&dest.get("LATEST").context("no whole run at this destination")?).trim().to_string(),
1175 
    };
1176 
    if !dest.exists(&format!("runs/{at}/DONE"))? {
1177 
        bail!("run {at} is not whole (no DONE): pick another with --run");
1178 
    }
1179 
    let mut done = Vec::new();
1180 
    for name in dest.list(&format!("runs/{at}/repos"))? {
1181 
        let m: Manifest = serde_json::from_slice(&dest.get(&format!("runs/{at}/repos/{name}"))?)?;
1182 
        let root = if m.kind == "wikis" { wikis_root } else { repos_root };
1183 
        let dir = root.join(&m.owner).join(&m.name);
1184 
        if dir.exists() {
1185 
            bail!("{} exists: restore into an empty hub, or move it aside", dir.display());
1186 
        }
1187 
        for f in &m.files {
1188 
            let bytes = dest.get(&blob_key(&f.hash))?;
1189 
            if blake3::hash(&bytes).to_hex().as_str() != f.hash {
1190 
                bail!("{}/{} {}: the copy does not match its hash", m.owner, m.name, f.path);
1191 
            }
1192 
            let p = dir.join(&f.path);
1193 
            std::fs::create_dir_all(p.parent().context("no parent")?)?;
1194 
            std::fs::write(&p, &bytes)?;
1195 
        }
1196 
        done.push(format!("{}/{} ({})", m.owner, m.name, m.kind));
1197 
    }
1198 
    Ok(done)
1199 
}
1200
1201 
/// Restore the whole run `at` (the latest when none) into this hub: each
1202 
/// repository's and wiki's store byte for byte, then its row and settings,
1203 
/// then its records (issues, proposals, releases and the rest), so it shows
1204 
/// on the hub as it was (docs/backups.md §7). What was restored, then any
1205 
/// notes (`note: …`) on what needs entering again or was left out.
1206 
pub async fn restore_into_hub(app: &Shared, dest: std::sync::Arc<dyn Destination>, at: Option<String>) -> Result<Vec<String>> {
1207 
    let (repos_root, wikis_root) = (app.repos.root().to_path_buf(), app.wikis.root().to_path_buf());
1208 
    let (d, a) = (dest.clone(), at.clone());
1209 
    let done = tokio::task::spawn_blocking(move || restore(d.as_ref(), a.as_deref(), &repos_root, &wikis_root)).await??;
1210 
    let at = match at {
1211 
        Some(a) => a,
1212 
        None => String::from_utf8_lossy(&tokio::task::block_in_place(|| dest.get("LATEST"))?).trim().to_string(),
1213 
    };
1214 
    let mut notes = Vec::new();
1215 
    for name in tokio::task::block_in_place(|| dest.list(&format!("runs/{at}/records")))? {
1216 
        let rec: serde_json::Value =
1217 
            serde_json::from_slice(&tokio::task::block_in_place(|| dest.get(&format!("runs/{at}/records/{name}")))?)?;
1218 
        let Some(row) = rec["repositories"].as_array().and_then(|a| a.first()) else { continue };
1219 
        let (owner, repo) = (row["owner"].as_str().unwrap_or_default(), row["name"].as_str().unwrap_or_default());
1220 
        if owner.is_empty() || db::repos::get(&app.db, owner, repo).await?.is_some() {
1221 
            continue;
1222 
        }
1223 
        let visibility = match row["visibility"].as_str() {
1224 
            Some("private") => hub_api::Visibility::Private,
1225 
            Some("internal") => hub_api::Visibility::Internal,
1226 
            _ => hub_api::Visibility::Public,
1227 
        };
1228 
        let made = db::repos::create(&app.db, owner, repo, row["description"].as_str().unwrap_or_default(), visibility).await?;
1229 
        // Its settings as they were: every column the row kept but its
1230 
        // name, its id and a fork's parent (another hub's id).
1231 
        let cols = table_cols(&app.db, "repositories").await?;
1232 
        let set: Vec<(&Col, serde_json::Value)> = cols
1233 
            .iter()
1234 
            .filter(|c| !["id", "owner", "name", "forked_from"].contains(&c.name.as_str()))
1235 
            .filter_map(|c| row.get(&c.name).filter(|v| !v.is_null()).map(|v| (c, v.clone())))
1236 
            .collect();
1237 
        if !set.is_empty() {
1238 
            let sql = format!(
1239 
                "UPDATE repositories SET {} WHERE id = ?",
1240 
                set.iter().map(|(c, _)| format!("\"{}\" = ?", c.name)).collect::<Vec<_>>().join(", ")
1241 
            );
1242 
            let mut q = sqlx::query(&sql);
1243 
            for (c, v) in set {
1244 
                q = bind_json(q, v, c.ty.to_ascii_uppercase().contains("BLOB"));
1245 
            }
1246 
            q.bind(made.id).execute(&app.db).await?;
1247 
        }
1248 
        let old = row["id"].as_i64().unwrap_or(-1);
1249 
        for n in restore_records(app, &dest, &rec, old, made.id).await.with_context(|| format!("restore {owner}/{repo}'s records"))? {
1250 
            notes.push(format!("note: {owner}/{repo}: {n}"));
1251 
        }
1252 
    }
1253 
    let mut done = done;
1254 
    done.extend(notes);
1255 
    Ok(done)
1256 
}
1257
1258 
#[cfg(test)]
1259 
mod tests {
1260 
    use super::*;
1261
1262 
    #[test]
1263 
    fn a_directory_destination_keeps_keys_inside_its_root() {
1264 
        let d = tempfile::tempdir().unwrap();
1265 
        let dest = Directory { root: d.path().to_path_buf() };
1266 
        dest.put("runs/a/DONE", b"a").unwrap();
1267 
        assert_eq!(dest.get("runs/a/DONE").unwrap(), b"a");
1268 
        assert_eq!(dest.list("runs").unwrap(), vec!["a"]);
1269 
        assert!(dest.put("../escape", b"x").is_err() && dest.put("/abs", b"x").is_err());
1270 
        dest.remove("runs").unwrap();
1271 
        assert!(dest.list("runs").unwrap().is_empty());
1272 
    }
1273
1274 
    #[test]
1275 
    fn files_are_sent_once_and_unchanged_ones_not_read_again() {
1276 
        let src = tempfile::tempdir().unwrap();
1277 
        let out = tempfile::tempdir().unwrap();
1278 
        let dest = Directory { root: out.path().to_path_buf() };
1279 
        std::fs::create_dir_all(src.path().join(".oxy/objects/loose")).unwrap();
1280 
        std::fs::write(src.path().join(".oxy/objects/loose/aa"), b"object").unwrap();
1281 
        std::fs::write(src.path().join(".oxy/ops-head"), b"ref one").unwrap();
1282 
        let mut s = Sent::default();
1283 
        let files = copy_dir(&dest, src.path(), &HashMap::new(), false, &mut s).unwrap();
1284 
        assert_eq!((files.len(), s.sent), (2, 2));
1285 
        let known: HashMap<String, File> = files.iter().map(|f| (f.path.clone(), f.clone())).collect();
1286 
        let mut s = Sent::default();
1287 
        let again = copy_dir(&dest, src.path(), &known, false, &mut s).unwrap();
1288 
        assert_eq!((again, s.sent), (files.clone(), 0), "nothing new travels");
1289 
        // The first pass takes only the files that never change.
1290 
        let mut s = Sent::default();
1291 
        let first = copy_dir(&dest, src.path(), &HashMap::new(), true, &mut s).unwrap();
1292 
        assert_eq!(first.iter().map(|f| f.path.as_str()).collect::<Vec<_>>(), vec![".oxy/objects/loose/aa"]);
1293 
    }
1294
1295 
    #[test]
1296 
    fn run_names_sort_by_time() {
1297 
        assert_eq!(run_name(0), "19700101T000000Z");
1298 
        assert!(run_name(1_791_900_000_000) < run_name(1_791_903_600_000));
1299 
    }
1300 
}