feat: backups for users and organisations, to a directory, with restore (refs #169) d7d182467382

-- Backups (docs/backups.md, oxygenhub#169): what each user and organisation

-- backs up, where and how often, and how each run went.

CREATE TABLE backup_settings (

    owner       TEXT PRIMARY KEY,

    -- 'user' or 'org'

    kind        TEXT NOT NULL,

    -- 'off', 'hourly' or 'daily'

    every       TEXT NOT NULL DEFAULT 'off',

    -- 'dir' (a directory hub.toml allows) or 'gcs'

    destination TEXT NOT NULL DEFAULT 'dir',

    -- the directory, or the bucket

    target      TEXT NOT NULL DEFAULT '',

    prefix      TEXT NOT NULL DEFAULT '',

    -- for 'gcs': the workload identity provider's resource name

    provider    TEXT NOT NULL DEFAULT '',

    keep_days   INTEGER NOT NULL DEFAULT 30,

    updated_ms  INTEGER NOT NULL

);

CREATE TABLE backup_runs (

    id          INTEGER PRIMARY KEY,

    owner       TEXT NOT NULL,

    started_ms  INTEGER NOT NULL,

    finished_ms INTEGER,

    -- 'running', 'ok', 'partial' or 'failed'

    status      TEXT NOT NULL,

    -- the run's name at its destination (runs/<at>/)

    at          TEXT NOT NULL DEFAULT '',

    repos       INTEGER NOT NULL DEFAULT 0,

    files       INTEGER NOT NULL DEFAULT 0,

    sent        INTEGER NOT NULL DEFAULT 0,

    bytes       INTEGER NOT NULL DEFAULT 0,

    detail      TEXT NOT NULL DEFAULT '',

    -- 'schedule' or 'now'

    trigger     TEXT NOT NULL DEFAULT 'schedule'

);

CREATE INDEX backup_runs_by_owner ON backup_runs (owner, id DESC);

//! Backups (docs/backups.md, oxygenhub#169): each user and organisation

//! copies its repositories, on a schedule, to a destination it controls.

//!

//! A repository is its store directory (`.oxy/`): packs and loose objects

//! that never change once written, and small files that do (refs, the

//! operation log, policies, signature indexes). A run copies each file

//! once, named by its BLAKE3 (`blobs/`), and writes a manifest of path to

//! hash per repository (`runs/<at>/repos/`), so only what changed travels

//! and any kept run can be restored byte for byte, signatures and all.

//! The large files are copied first; the walk is then repeated under the

//! repository's read lock, which pushes wait on, so each manifest is one

//! consistent moment. Beside each manifest go the hub's records for the

//! repository as JSON (never a secret value), the owner's records, and

//! last a `DONE` marker: a run without one was cut short and is ignored.

use crate::db::{self, Pool};

use crate::repos::RepoName;

use crate::Shared;

use anyhow::{bail, Context, Result};

use serde::{Deserialize, Serialize};

use std::collections::HashMap;

use std::path::{Path, PathBuf};

/// How often, as the settings offer it.

pub const EVERY: &[(&str, &str)] = &[("off", "Off"), ("hourly", "Every hour"), ("daily", "Every day")];

/// How long runs are kept, in days.

pub const KEEP: &[i64] = &[7, 30, 90];

/// What an owner backs up, where and how often.

#[derive(Clone, Debug, sqlx::FromRow, Serialize)]

pub struct Setting {

    pub owner: String,

    pub kind: String,

    pub every: String,

    pub destination: String,

    pub target: String,

    pub prefix: String,

    pub provider: String,

    pub keep_days: i64,

    pub updated_ms: i64,

}

impl Setting {

    pub fn off(owner: &str, kind: &str) -> Setting {

        Setting {

            owner: owner.into(),

            kind: kind.into(),

            every: "off".into(),

            destination: "dir".into(),

            target: String::new(),

            prefix: String::new(),

            provider: String::new(),

            keep_days: 30,

            updated_ms: 0,

        }

    }

    /// Seconds between runs; none when off.

    pub fn period_s(&self) -> Option<i64> {

        match self.every.as_str() {

            "hourly" => Some(3600),

            "daily" => Some(86_400),

            _ => None,

        }

    }

}

/// One run, as the settings page and the API show it.

#[derive(Clone, Debug, sqlx::FromRow, Serialize)]

pub struct Run {

    pub id: i64,

    pub owner: String,

    pub started_ms: i64,

    pub finished_ms: Option<i64>,

    pub status: String,

    pub at: String,

    pub repos: i64,

    pub files: i64,

    pub sent: i64,

    pub bytes: i64,

    pub detail: String,

    pub trigger: String,

}

pub async fn setting(pool: &Pool, owner: &str) -> sqlx::Result<Option<Setting>> {

    sqlx::query_as("SELECT * FROM backup_settings WHERE owner = ?").bind(owner).fetch_optional(pool).await

}

pub async fn save(pool: &Pool, s: &Setting) -> sqlx::Result<()> {

    sqlx::query(

        "INSERT INTO backup_settings (owner, kind, every, destination, target, prefix, provider, keep_days, updated_ms) \

         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(owner) DO UPDATE SET kind = excluded.kind, every = excluded.every, \

         destination = excluded.destination, target = excluded.target, prefix = excluded.prefix, provider = excluded.provider, \

         keep_days = excluded.keep_days, updated_ms = excluded.updated_ms",

    )

    .bind(&s.owner)

    .bind(&s.kind)

    .bind(&s.every)

    .bind(&s.destination)

    .bind(&s.target)

    .bind(&s.prefix)

    .bind(&s.provider)

    .bind(s.keep_days)

    .bind(db::now_ms())

    .execute(pool)

    .await?;

    Ok(())

}

pub async fn runs(pool: &Pool, owner: &str, limit: i64) -> sqlx::Result<Vec<Run>> {

    sqlx::query_as("SELECT * FROM backup_runs WHERE owner = ? ORDER BY id DESC LIMIT ?").bind(owner).bind(limit).fetch_all(pool).await

}

// ---------------------------------------------------------------- where

/// Where backups are written: keys are `/`-separated paths.

pub trait Destination: Send + Sync {

    fn exists(&self, key: &str) -> Result<bool>;

    /// Write `bytes` at `key`, whole or not at all.

    fn put(&self, key: &str, bytes: &[u8]) -> Result<()>;

    fn get(&self, key: &str) -> Result<Vec<u8>>;

    /// The names directly under `prefix` (files and "directories").

    fn list(&self, prefix: &str) -> Result<Vec<String>>;

    /// Remove `key` and everything under it.

    fn remove(&self, key: &str) -> Result<()>;

}

/// A directory on the hub's host.

pub struct Directory {

    pub root: PathBuf,

}

impl Directory {

    fn path(&self, key: &str) -> Result<PathBuf> {

        if key.split('/').any(|p| p == ".." || p == ".") || key.starts_with('/') {

            bail!("not a clean key: {key}");

        }

        Ok(self.root.join(key))

    }

}

impl Destination for Directory {

    fn exists(&self, key: &str) -> Result<bool> {

        Ok(self.path(key)?.exists())

    }

    fn put(&self, key: &str, bytes: &[u8]) -> Result<()> {

        let p = self.path(key)?;

        std::fs::create_dir_all(p.parent().context("no parent")?)?;

        let tmp = p.with_extension(format!("part-{}", std::process::id()));

        std::fs::write(&tmp, bytes)?;

        std::fs::rename(&tmp, &p)?;

        Ok(())

    }

    fn get(&self, key: &str) -> Result<Vec<u8>> {

        let p = self.path(key)?;

        std::fs::read(&p).with_context(|| format!("read {}", p.display()))

    }

    fn list(&self, prefix: &str) -> Result<Vec<String>> {

        let p = self.path(prefix)?;

        let mut out: Vec<String> = match std::fs::read_dir(&p) {

            Ok(d) => d.flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect(),

            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),

            Err(e) => return Err(e.into()),

        };

        out.sort();

        Ok(out)

    }

    fn remove(&self, key: &str) -> Result<()> {

        let p = self.path(key)?;

        if p.is_dir() {

            std::fs::remove_dir_all(p)?;

        } else if p.exists() {

            std::fs::remove_file(p)?;

        }

        Ok(())

    }

}

/// The owner's destination, from its setting and the site's rules.

pub fn destination(app: &Shared, s: &Setting) -> Result<Box<dyn Destination>> {

    match s.destination.as_str() {

        "dir" => {

            let allowed = &app.config.backups.directories;

            let target = PathBuf::from(&s.target);

            if !allowed.contains(&target) {

                bail!("{} is not a backup directory this hub allows (hub.toml [backups] directories)", s.target);

            }

            let mut root = target.join(&s.owner);

            if !s.prefix.is_empty() {

                root = root.join(s.prefix.trim_matches('/'));

            }

            Ok(Box::new(Directory { root }))

        }

        "gcs" => bail!("GCS destinations come with oxygenhub#169's next step"),

        other => bail!("unknown destination {other:?}"),

    }

}

// ---------------------------------------------------------------- what

/// One file of a repository's store, in a manifest.

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]

pub struct File {

    pub path: String,

    pub size: u64,

    /// Its modification time in nanoseconds: unchanged size and time mean

    /// unchanged content, so it is not hashed again.

    pub mtime: i128,

    pub hash: String,

}

/// A repository at one run.

#[derive(Clone, Debug, Serialize, Deserialize)]

pub struct Manifest {

    pub owner: String,

    pub name: String,

    /// `repos` or `wikis`.

    pub kind: String,

    pub files: Vec<File>,

}

fn blob_key(hash: &str) -> String {

    format!("blobs/{}/{hash}", &hash[..2])

}

/// The large files that never change once written come first.

fn immutable(path: &str) -> bool {

    path.starts_with(".oxy/objects/") && (path.contains("/pack-") || path.contains("/loose/"))

}

#[derive(Default)]

struct Sent {

    files: i64,

    sent: i64,

    bytes: i64,

}

/// Walk `dir`, sending each file the destination lacks; `known` says which

/// paths are unchanged since the last run (and their hashes).

fn copy_dir(dest: &dyn Destination, dir: &Path, known: &HashMap<String, File>, only_immutable: bool, sent: &mut Sent) -> Result<Vec<File>> {

    let mut out = Vec::new();

    let mut stack = vec![dir.to_path_buf()];

    while let Some(d) = stack.pop() {

        let entries = match std::fs::read_dir(&d) {

            Ok(e) => e,

            Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,

            Err(e) => return Err(e.into()),

        };

        for e in entries.flatten() {

            let p = e.path();

            let ft = e.file_type()?;

            if ft.is_dir() {

                stack.push(p);

                continue;

            }

            if !ft.is_file() {

                continue;

            }

            let rel = p.strip_prefix(dir).unwrap_or(&p).to_string_lossy().replace('\\', "/");

            if only_immutable && !immutable(&rel) {

                continue;

            }

            // A file packed away while the walk ran is simply gone.

            let Ok(md) = std::fs::metadata(&p) else { continue };

            let mtime =

                md.modified().ok().and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()).map(|d| d.as_nanos() as i128).unwrap_or(0);

            let size = md.len();

            sent.files += 1;

            if let Some(k) = known.get(&rel).filter(|k| k.size == size && k.mtime == mtime) {

                out.push(k.clone());

                continue;

            }

            let Ok(bytes) = std::fs::read(&p) else { continue };

            let hash = blake3::hash(&bytes).to_hex().to_string();

            let key = blob_key(&hash);

            if !dest.exists(&key)? {

                dest.put(&key, &bytes)?;

                sent.sent += 1;

                sent.bytes += bytes.len() as i64;

            }

            out.push(File { path: rel, size, mtime, hash });

        }

    }

    out.sort_by(|a, b| a.path.cmp(&b.path));

    Ok(out)

}

// ---------------------------------------------------------------- records

/// Tables and columns never written: secrets, credentials, sessions.

fn private_table(t: &str) -> bool {

    ["secret", "token", "session", "password", "totp", "recovery", "sqlx"].iter().any(|w| t.contains(w))

}

fn private_column(c: &str) -> bool {

    ["secret", "token", "password", "salt", "totp", "sealed", "hash_key"].iter().any(|w| c.contains(w))

}

async fn tables(pool: &Pool) -> Result<Vec<String>> {

    let mut t: Vec<String> =

        sqlx::query_scalar("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").fetch_all(pool).await?;

    t.retain(|t| !private_table(t));

    t.sort();

    Ok(t)

}

async fn columns(pool: &Pool, table: &str) -> Result<Vec<String>> {

    Ok(sqlx::query_scalar(&format!("SELECT name FROM pragma_table_info('{table}')")).fetch_all(pool).await?)

}

/// The rows of `table` matching `cond`, as a JSON array, private columns

/// left out and blobs as hex.

async fn rows(pool: &Pool, table: &str, cond: &str) -> Result<serde_json::Value> {

    let cols = columns(pool, table).await?;

    let fields: Vec<String> = cols

        .iter()

        .filter(|c| !private_column(c))

        .map(|c| format!("'{c}', CASE WHEN typeof(\"{c}\") = 'blob' THEN hex(\"{c}\") ELSE \"{c}\" END"))

        .collect();

    if fields.is_empty() {

        return Ok(serde_json::json!([]));

    }

    let sql = format!("SELECT json_group_array(json_object({})) FROM \"{table}\" WHERE {cond}", fields.join(", "));

    let text: String = sqlx::query_scalar(&sql).fetch_one(pool).await?;

    Ok(serde_json::from_str(&text)?)

}

/// Every row about something: the tables with column `key` equal to `id`,

/// then, three levels deep, the tables whose foreign keys point at rows

/// already taken (an issue's comments, a proposal's reviews and their

/// threads).

async fn records(pool: &Pool, key: &str, id: i64) -> Result<serde_json::Value> {

    let all = tables(pool).await?;

    let mut conds: Vec<(String, String)> = Vec::new();

    for t in &all {

        if columns(pool, t).await?.iter().any(|c| c == key) {

            conds.push((t.clone(), format!("\"{key}\" = {id}")));

        }

    }

    for _ in 0..3 {

        let mut more = Vec::new();

        for t in &all {

            if conds.iter().any(|(c, _)| c == t) || more.iter().any(|(c, _): &(String, String)| c == t) {

                continue;

            }

            let fks: Vec<(String, String, String)> =

                sqlx::query_as(&format!("SELECT \"table\", \"from\", \"to\" FROM pragma_foreign_key_list('{t}')")).fetch_all(pool).await?;

            if let Some((parent, from, to, pcond)) = fks

                .iter()

                .find_map(|(p, f, to)| conds.iter().find(|(c, _)| c == p).map(|(_, pc)| (p.clone(), f.clone(), to.clone(), pc.clone())))

            {

                more.push((t.clone(), format!("\"{from}\" IN (SELECT \"{to}\" FROM \"{parent}\" WHERE {pcond})")));

            }

        }

        if more.is_empty() {

            break;

        }

        conds.extend(more);

    }

    let mut out = serde_json::Map::new();

    for (t, cond) in conds {

        let r = rows(pool, &t, &cond).await?;

        if r.as_array().is_some_and(|a| !a.is_empty()) {

            out.insert(t, r);

        }

    }

    Ok(serde_json::Value::Object(out))

}

// ---------------------------------------------------------------- a run

fn run_name(ms: i64) -> String {

    let t = time::OffsetDateTime::from_unix_timestamp_nanos(ms as i128 * 1_000_000).unwrap_or(time::OffsetDateTime::UNIX_EPOCH);

    format!("{:04}{:02}{:02}T{:02}{:02}{:02}Z", t.year(), u8::from(t.month()), t.day(), t.hour(), t.minute(), t.second())

}

/// The last whole run's manifests, by `kind/name`.

fn last_manifests(dest: &dyn Destination) -> HashMap<String, HashMap<String, File>> {

    let mut out = HashMap::new();

    let Ok(at) = dest.get("LATEST").map(|b| String::from_utf8_lossy(&b).trim().to_string()) else { return out };

    for name in dest.list(&format!("runs/{at}/repos")).unwrap_or_default() {

        if let Ok(m) = dest.get(&format!("runs/{at}/repos/{name}")).and_then(|b| Ok(serde_json::from_slice::<Manifest>(&b)?)) {

            out.insert(format!("{}/{}", m.kind, m.name), m.files.into_iter().map(|f| (f.path.clone(), f)).collect());

        }

    }

    out

}

/// Back up `owner` now, as its setting says. The run is recorded whatever

/// happens; a failure notifies the owner once until a run succeeds.

pub async fn run(app: &Shared, owner: &str, trigger: &str) -> Result<Run> {

    let s = setting(&app.db, owner).await?.with_context(|| format!("{owner} has no backup set up"))?;

    let started = db::now_ms();

    let at = run_name(started);

    let id: i64 =

        sqlx::query_scalar("INSERT INTO backup_runs (owner, started_ms, status, at, trigger) VALUES (?, ?, 'running', ?, ?) RETURNING id")

            .bind(owner)

            .bind(started)

            .bind(&at)

            .bind(trigger)

            .fetch_one(&app.db)

            .await?;

    let outcome = run_into(app, &s, &at).await;

    let (status, detail, sent) = match outcome {

        Ok((sent, problems)) if problems.is_empty() => ("ok", String::new(), sent),

        Ok((sent, problems)) => ("partial", problems.join("; "), sent),

        Err(e) => ("failed", format!("{e:#}"), (0, Sent::default())),

    };

    sqlx::query("UPDATE backup_runs SET finished_ms = ?, status = ?, detail = ?, repos = ?, files = ?, sent = ?, bytes = ? WHERE id = ?")

        .bind(db::now_ms())

        .bind(status)

        .bind(&detail)

        .bind(sent.0)

        .bind(sent.1.files)

        .bind(sent.1.sent)

        .bind(sent.1.bytes)

        .bind(id)

        .execute(&app.db)

        .await?;

    if status != "ok" {

        notify_failure(app, &s, status, &detail).await;

    }

    let r: Run = sqlx::query_as("SELECT * FROM backup_runs WHERE id = ?").bind(id).fetch_one(&app.db).await?;

    Ok(r)

}

async fn run_into(app: &Shared, s: &Setting, at: &str) -> Result<((i64, Sent), Vec<String>)> {

    let dest: std::sync::Arc<dyn Destination> = destination(app, s)?.into();

    let known = {

        let d = dest.clone();

        tokio::task::spawn_blocking(move || last_manifests(d.as_ref())).await?

    };

    let mine: Vec<db::repos::RepoRow> = db::repos::list(&app.db).await?.into_iter().filter(|r| r.owner == s.owner).collect();

    let mut problems = Vec::new();

    let mut sent = Sent::default();

    let mut repos = 0i64;

    for row in &mine {

        let Some(name) = RepoName::new(&row.owner, &row.name) else { continue };

        for (kind, store) in [("repos", &app.repos), ("wikis", &app.wikis)] {

            if !store.exists(&name) {

                continue;

            }

            let dir = store.dir(&name);

            let k = known.get(&format!("{kind}/{}", row.name)).cloned().unwrap_or_default();

            // The large, unchanging files without the lock, then the whole

            // walk again under the read lock: pushes wait for that moment.

            let (d1, dir1, k1) = (dest.clone(), dir.clone(), k.clone());

            let first = tokio::task::spawn_blocking(move || {

                let mut s = Sent::default();

                copy_dir(d1.as_ref(), &dir1, &k1, true, &mut s).map(|f| (f, s))

            })

            .await?;

            let mut k2 = k;

            if let Ok((files, _)) = &first {

                for f in files {

                    k2.insert(f.path.clone(), f.clone());

                }

            }

            let guard = store.read_guard(&name).await;

            let (d2, dir2) = (dest.clone(), dir.clone());

            let second = tokio::task::spawn_blocking(move || {

                let _guard = guard;

                let mut s = Sent::default();

                copy_dir(d2.as_ref(), &dir2, &k2, false, &mut s).map(|f| (f, s))

            })

            .await?;

            match second {

                Ok((files, s2)) => {

                    let s1 = first.map(|(_, s)| s).unwrap_or_default();

                    sent.files += s2.files;

                    sent.sent += s1.sent + s2.sent;

                    sent.bytes += s1.bytes + s2.bytes;

                    let m = Manifest { owner: row.owner.clone(), name: row.name.clone(), kind: kind.into(), files };

                    let key = format!("runs/{at}/repos/{kind}-{}.json", row.name);

                    dest.put(&key, &serde_json::to_vec(&m)?)?;

                    if kind == "repos" {

                        repos += 1;

                    }

                }

                Err(e) => problems.push(format!("{}/{} ({kind}): {e:#}", row.owner, row.name)),

            }

        }

        let rec = async {

            let mut r = records(&app.db, "repo_id", row.id).await?;

            // The repository's own row, which restore recreates first.

            let own = rows(&app.db, "repositories", &format!("id = {}", row.id)).await?;

            if let Some(m) = r.as_object_mut() {

                m.insert("repositories".into(), own);

            }

            anyhow::Ok(r)

        };

        match rec.await {

            Ok(r) => dest.put(&format!("runs/{at}/records/{}.json", row.name), &serde_json::to_vec(&r)?)?,

            Err(e) => problems.push(format!("{}/{} records: {e:#}", row.owner, row.name)),

        }

    }

    // The owner itself.

    let owner_rows = if s.kind == "org" {

        match db::accounts::org_by_name(&app.db, &s.owner).await? {

            Some(o) => records(&app.db, "org_id", o.id).await?,

            None => serde_json::json!({}),

        }

    } else {

        match db::accounts::account_by_name(&app.db, &s.owner).await? {

            Some(a) => records(&app.db, "account_id", a.id).await?,

            None => serde_json::json!({}),

        }

    };

    dest.put(&format!("runs/{at}/owner.json"), &serde_json::to_vec(&owner_rows)?)?;

    dest.put(&format!("runs/{at}/DONE"), at.as_bytes())?;

    dest.put("LATEST", at.as_bytes())?;

    prune(dest.as_ref(), s.keep_days)?;

    Ok(((repos, sent), problems))

}

/// Drop whole runs older than `keep_days`, and the blobs no kept run

/// names. The newest whole run is always kept.

pub fn prune(dest: &dyn Destination, keep_days: i64) -> Result<()> {

    let cutoff = run_name(db::now_ms() - keep_days.max(1) * 86_400_000);

    let runs = dest.list("runs")?;

    let whole: Vec<&String> = runs.iter().filter(|r| dest.exists(&format!("runs/{r}/DONE")).unwrap_or(false)).collect();

    let newest = whole.last().map(|s| s.to_string());

    let mut removed = false;

    for r in &runs {

        if r.as_str() < cutoff.as_str() && Some(r.to_string()) != newest {

            dest.remove(&format!("runs/{r}"))?;

            removed = true;

        }

    }

    if !removed {

        return Ok(());

    }

    let mut wanted = std::collections::HashSet::new();

    for r in dest.list("runs")? {

        for m in dest.list(&format!("runs/{r}/repos"))? {

            if let Ok(m) = dest.get(&format!("runs/{r}/repos/{m}")).and_then(|b| Ok(serde_json::from_slice::<Manifest>(&b)?)) {

                wanted.extend(m.files.into_iter().map(|f| f.hash));

            }

        }

    }

    for shard in dest.list("blobs")? {

        for b in dest.list(&format!("blobs/{shard}"))? {

            if !wanted.contains(&b) {

                dest.remove(&format!("blobs/{shard}/{b}"))?;

            }

        }

    }

    Ok(())

}

async fn notify_failure(app: &Shared, s: &Setting, status: &str, detail: &str) {

    // Once per spell of failures: not when the run before failed too.

    let before: Option<String> =

        sqlx::query_scalar("SELECT status FROM backup_runs WHERE owner = ? AND finished_ms IS NOT NULL ORDER BY id DESC LIMIT 1 OFFSET 1")

            .bind(&s.owner)

            .fetch_optional(&app.db)

            .await

            .ok()

            .flatten();

    if before.as_deref().is_some_and(|b| b != "ok") {

        return;

    }

    let (who, url): (Vec<i64>, String) = if s.kind == "org" {

        match db::accounts::org_by_name(&app.db, &s.owner).await.ok().flatten() {

            Some(o) => (

                db::accounts::org_members(&app.db, o.id)

                    .await

                    .unwrap_or_default()

                    .into_iter()

                    .filter(|(_, r)| r == "owner")

                    .map(|(a, _)| a.id)

                    .collect(),

                format!("/orgs/{}/settings#backups", s.owner),

            ),

            None => return,

        }

    } else {

        match db::accounts::account_by_name(&app.db, &s.owner).await.ok().flatten() {

            Some(a) => (vec![a.id], "/settings/backups".into()),

            None => return,

        }

    };

    let title = format!("Backup of {} {status}: {}", s.owner, detail.chars().take(160).collect::<String>());

    for id in who {

        let _ = db::notify::add(

            &app.db,

            db::notify::New {

                account_id: id,

                repo_id: None,

                reason: db::notify::Reason::Participating,

                kind: "backup",

                title: &title,

                url: &url,

                actor: "",

            },

        )

        .await;

    }

}

/// Run each owner's backup when it is due, one at a time. A run left

/// "running" by a restart is marked failed first.

pub async fn scheduler(app: Shared) {

    let _ = sqlx::query(

        "UPDATE backup_runs SET status = 'failed', detail = 'the hub restarted during the run', finished_ms = ? WHERE status = 'running'",

    )

    .bind(db::now_ms())

    .execute(&app.db)

    .await;

    loop {

        tokio::time::sleep(std::time::Duration::from_secs(60)).await;

        for s in due(&app.db).await.unwrap_or_default() {

            if let Err(e) = run(&app, &s.owner, "schedule").await {

                tracing::warn!("backup of {}: {e:#}", s.owner);

            }

        }

    }

}

/// The owners whose next run is due now.

pub async fn due(pool: &Pool) -> sqlx::Result<Vec<Setting>> {

    let all: Vec<Setting> = sqlx::query_as("SELECT * FROM backup_settings WHERE every != 'off'").fetch_all(pool).await?;

    let now = db::now_ms();

    let mut out = Vec::new();

    for s in all {

        let Some(period) = s.period_s() else { continue };

        let last: Option<i64> =

            sqlx::query_scalar("SELECT MAX(started_ms) FROM backup_runs WHERE owner = ?").bind(&s.owner).fetch_one(pool).await?;

        if last.is_none_or(|l| now - l >= period * 1000) {

            out.push(s);

        }

    }

    Ok(out)

}

/// Restore every repository of the whole run `at` (the latest when none)

/// from `dest` into `repos_root/OWNER/NAME` and `wikis_root/…`, refusing to

/// overwrite one that exists. Each file is checked against its hash.

pub fn restore(dest: &dyn Destination, at: Option<&str>, repos_root: &Path, wikis_root: &Path) -> Result<Vec<String>> {

    let at = match at {

        Some(a) => a.to_string(),

        None => String::from_utf8_lossy(&dest.get("LATEST").context("no whole run at this destination")?).trim().to_string(),

    };

    if !dest.exists(&format!("runs/{at}/DONE"))? {

        bail!("run {at} is not whole (no DONE): pick another with --run");

    }

    let mut done = Vec::new();

    for name in dest.list(&format!("runs/{at}/repos"))? {

        let m: Manifest = serde_json::from_slice(&dest.get(&format!("runs/{at}/repos/{name}"))?)?;

        let root = if m.kind == "wikis" { wikis_root } else { repos_root };

        let dir = root.join(&m.owner).join(&m.name);

        if dir.exists() {

            bail!("{} exists: restore into an empty hub, or move it aside", dir.display());

        }

        for f in &m.files {

            let bytes = dest.get(&blob_key(&f.hash))?;

            if blake3::hash(&bytes).to_hex().as_str() != f.hash {

                bail!("{}/{} {}: the copy does not match its hash", m.owner, m.name, f.path);

            }

            let p = dir.join(&f.path);

            std::fs::create_dir_all(p.parent().context("no parent")?)?;

            std::fs::write(&p, &bytes)?;

        }

        done.push(format!("{}/{} ({})", m.owner, m.name, m.kind));

    }

    Ok(done)

}

/// Restore the whole run `at` (the latest when none) into this hub: each

/// repository's and wiki's store byte for byte, then its row, from the

/// run's records, so it shows on the hub as it was. Issues, proposals and

/// the rest stay in the run's records for now (docs/backups.md §7).

pub async fn restore_into_hub(app: &Shared, dest: std::sync::Arc<dyn Destination>, at: Option<String>) -> Result<Vec<String>> {

    let (repos_root, wikis_root) = (app.repos.root().to_path_buf(), app.wikis.root().to_path_buf());

    let (d, a) = (dest.clone(), at.clone());

    let done = tokio::task::spawn_blocking(move || restore(d.as_ref(), a.as_deref(), &repos_root, &wikis_root)).await??;

    let at = match at {

        Some(a) => a,

        None => String::from_utf8_lossy(&dest.get("LATEST")?).trim().to_string(),

    };

    for name in dest.list(&format!("runs/{at}/records"))? {

        let rec: serde_json::Value = serde_json::from_slice(&dest.get(&format!("runs/{at}/records/{name}"))?)?;

        let Some(row) = rec["repositories"].as_array().and_then(|a| a.first()) else { continue };

        let (owner, repo) = (row["owner"].as_str().unwrap_or_default(), row["name"].as_str().unwrap_or_default());

        if owner.is_empty() || db::repos::get(&app.db, owner, repo).await?.is_some() {

            continue;

        }

        let visibility = match row["visibility"].as_str() {

            Some("private") => hub_api::Visibility::Private,

            Some("internal") => hub_api::Visibility::Internal,

            _ => hub_api::Visibility::Public,

        };

        let made = db::repos::create(&app.db, owner, repo, row["description"].as_str().unwrap_or_default(), visibility).await?;

        if let Some(b) = row["default_bookmark"].as_str().filter(|b| !b.is_empty()) {

            db::repos::set_default_bookmark(&app.db, made.id, b).await?;

        }

    }

    Ok(done)

}

#[cfg(test)]

mod tests {

    use super::*;

    #[test]

    fn a_directory_destination_keeps_keys_inside_its_root() {

        let d = tempfile::tempdir().unwrap();

        let dest = Directory { root: d.path().to_path_buf() };

        dest.put("runs/a/DONE", b"a").unwrap();

        assert_eq!(dest.get("runs/a/DONE").unwrap(), b"a");

        assert_eq!(dest.list("runs").unwrap(), vec!["a"]);

        assert!(dest.put("../escape", b"x").is_err() && dest.put("/abs", b"x").is_err());

        dest.remove("runs").unwrap();

        assert!(dest.list("runs").unwrap().is_empty());

    }

    #[test]

    fn files_are_sent_once_and_unchanged_ones_not_read_again() {

        let src = tempfile::tempdir().unwrap();

        let out = tempfile::tempdir().unwrap();

        let dest = Directory { root: out.path().to_path_buf() };

        std::fs::create_dir_all(src.path().join(".oxy/objects/loose")).unwrap();

        std::fs::write(src.path().join(".oxy/objects/loose/aa"), b"object").unwrap();

        std::fs::write(src.path().join(".oxy/ops-head"), b"ref one").unwrap();

        let mut s = Sent::default();

        let files = copy_dir(&dest, src.path(), &HashMap::new(), false, &mut s).unwrap();

        assert_eq!((files.len(), s.sent), (2, 2));

        let known: HashMap<String, File> = files.iter().map(|f| (f.path.clone(), f.clone())).collect();

        let mut s = Sent::default();

        let again = copy_dir(&dest, src.path(), &known, false, &mut s).unwrap();

        assert_eq!((again, s.sent), (files.clone(), 0), "nothing new travels");

        // The first pass takes only the files that never change.

        let mut s = Sent::default();

        let first = copy_dir(&dest, src.path(), &HashMap::new(), true, &mut s).unwrap();

        assert_eq!(first.iter().map(|f| f.path.as_str()).collect::<Vec<_>>(), vec![".oxy/objects/loose/aa"]);

    }

    #[test]

    fn run_names_sort_by_time() {

        assert_eq!(run_name(0), "19700101T000000Z");

        assert!(run_name(1_791_900_000_000) < run_name(1_791_903_600_000));

    }

}

//! Backups' pages (docs/backups.md §5, oxygenhub#169): a user's under

//! Settings › Backups, an organisation's under its settings, each with

//! where its repositories go, how often, how long runs are kept, "back up

//! now" and the recent runs.

use super::settings::{csrf, require};

use super::{filters, Error, Page, Shell};

use crate::auth::Viewer;

use crate::backups::{self, Run, Setting};

use crate::Shared;

use askama::Template;

use axum::extract::{Path, State};

use axum::Form;

use serde::Deserialize;

#[derive(Template)]

#[template(path = "backups.html")]

pub struct BackupsPage {

    shell: Shell,

    /// `user` or `org`.

    kind: &'static str,

    owner: String,

    /// Where the form posts.

    base: String,

    setting: Setting,

    runs: Vec<Run>,

    /// The directories this hub allows.

    dirs: Vec<String>,

    notice: Option<String>,

    error: Option<String>,

}

impl BackupsPage {

    fn every(&self) -> &'static [(&'static str, &'static str)] {

        backups::EVERY

    }

    fn keeps(&self) -> &'static [i64] {

        backups::KEEP

    }

}

async fn page(

    app: &Shared,

    shell: Shell,

    kind: &'static str,

    owner: &str,

    notice: Option<String>,

    error: Option<String>,

) -> Result<Page<BackupsPage>, Error> {

    let e = |e| Error::from_db(shell.clone(), e);

    let setting = backups::setting(&app.db, owner).await.map_err(e)?.unwrap_or_else(|| Setting::off(owner, kind));

    let runs = backups::runs(&app.db, owner, 50).await.map_err(e)?;

    let base = if kind == "org" { format!("/orgs/{owner}/settings/backups") } else { "/settings/backups".into() };

    let dirs = app.config.backups.directories.iter().map(|d| d.display().to_string()).collect();

    Ok(Page(BackupsPage { shell, kind, owner: owner.into(), base, setting, runs, dirs, notice, error }))

}

#[derive(Deserialize)]

pub struct BackupForm {

    #[serde(rename = "_csrf")]

    csrf_token: String,

    every: String,

    #[serde(default)]

    destination: String,

    #[serde(default)]

    dir: String,

    #[serde(default)]

    bucket: String,

    #[serde(default)]

    provider: String,

    #[serde(default)]

    prefix: String,

    #[serde(default)]

    keep_days: String,

}

/// The form as a setting, or why it cannot be one.

fn setting_of(app: &Shared, owner: &str, kind: &str, f: &BackupForm) -> Result<Setting, String> {

    if !backups::EVERY.iter().any(|(v, _)| *v == f.every) {

        return Err("Pick how often: off, hourly or daily.".into());

    }

    let keep_days: i64 = f.keep_days.parse().unwrap_or(30);

    if !backups::KEEP.contains(&keep_days) {

        return Err("Keep runs for 7, 30 or 90 days.".into());

    }

    let prefix = f.prefix.trim().trim_matches('/').to_string();

    if prefix.split('/').any(|p| p == ".." || p == ".") {

        return Err("The prefix is a plain path, without . or .. parts.".into());

    }

    let (target, provider) = match f.destination.as_str() {

        "dir" => {

            if !app.config.backups.directories.iter().any(|d| d.display().to_string() == f.dir) {

                return Err("Pick one of the directories this hub allows.".into());

            }

            (f.dir.clone(), String::new())

        }

        "gcs" => {

            let bucket = f.bucket.trim().trim_start_matches("gs://").trim_end_matches('/').to_string();

            if bucket.is_empty() || !bucket.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || "-_.".contains(c)) {

                return Err("Give the bucket's name, such as andymccall-oxygen-backup.".into());

            }

            let provider = f.provider.trim().to_string();

            if !provider.starts_with("//iam.googleapis.com/projects/") || !provider.contains("/providers/") {

                return Err(

                    "Give the workload identity provider's full name, //iam.googleapis.com/projects/…/providers/… (docs/backups.md §6)."

                        .into(),

                );

            }

            (bucket, provider)

        }

        _ => return Err("Pick where backups go.".into()),

    };

    Ok(Setting {

        owner: owner.into(),

        kind: kind.into(),

        every: f.every.clone(),

        destination: f.destination.clone(),

        target,

        prefix,

        provider,

        keep_days,

        updated_ms: 0,

    })

}

async fn save(app: &Shared, shell: Shell, kind: &'static str, owner: &str, f: &BackupForm) -> Result<Page<BackupsPage>, Error> {

    match setting_of(app, owner, kind, f) {

        Ok(s) => {

            backups::save(&app.db, &s).await.map_err(|e| Error::from_db(shell.clone(), e))?;

            page(app, shell, kind, owner, Some("Saved.".into()), None).await

        }

        Err(why) => page(app, shell, kind, owner, None, Some(why)).await,

    }

}

async fn run_now(app: &Shared, shell: Shell, kind: &'static str, owner: &str) -> Result<Page<BackupsPage>, Error> {

    let set = backups::setting(&app.db, owner).await.map_err(|e| Error::from_db(shell.clone(), e))?;

    if set.is_none() {

        return page(app, shell, kind, owner, None, Some("Save where backups go first.".into())).await;

    }

    let (app2, owner2) = (app.clone(), owner.to_string());

    tokio::spawn(async move {

        if let Err(e) = backups::run(&app2, &owner2, "now").await {

            tracing::warn!("backup of {owner2}: {e:#}");

        }

    });

    page(app, shell, kind, owner, Some("Backing up now: the run shows below when it finishes.".into()), None).await

}

#[derive(Deserialize)]

pub struct Csrf {

    #[serde(rename = "_csrf")]

    csrf_token: String,

}

pub async fn user(State(app): State<Shared>, viewer: Viewer) -> Result<Page<BackupsPage>, Error> {

    let shell = Shell::new(&app, "/settings/backups", &viewer);

    let me = require(&viewer, &shell)?;

    page(&app, shell, "user", &me.account.username, None, None).await

}

pub async fn user_save(State(app): State<Shared>, viewer: Viewer, Form(f): Form<BackupForm>) -> Result<Page<BackupsPage>, Error> {

    let shell = Shell::new(&app, "/settings/backups", &viewer);

    let me = require(&viewer, &shell)?;

    csrf(&viewer, &f.csrf_token, &shell)?;

    save(&app, shell, "user", &me.account.username, &f).await

}

pub async fn user_run(State(app): State<Shared>, viewer: Viewer, Form(f): Form<Csrf>) -> Result<Page<BackupsPage>, Error> {

    let shell = Shell::new(&app, "/settings/backups", &viewer);

    let me = require(&viewer, &shell)?;

    csrf(&viewer, &f.csrf_token, &shell)?;

    run_now(&app, shell, "user", &me.account.username).await

}

pub async fn org(State(app): State<Shared>, viewer: Viewer, Path(name): Path<String>) -> Result<Page<BackupsPage>, Error> {

    let shell = Shell::new(&app, &format!("/orgs/{name}/settings/backups"), &viewer);

    let (_, org) = super::orgs::owned_org(&app, &shell, &viewer, &name).await?;

    page(&app, shell, "org", &org.name, None, None).await

}

pub async fn org_save(

    State(app): State<Shared>,

    viewer: Viewer,

    Path(name): Path<String>,

    Form(f): Form<BackupForm>,

) -> Result<Page<BackupsPage>, Error> {

    let shell = Shell::new(&app, &format!("/orgs/{name}/settings/backups"), &viewer);

    let (_, org) = super::orgs::owned_org(&app, &shell, &viewer, &name).await?;

    csrf(&viewer, &f.csrf_token, &shell)?;

    save(&app, shell, "org", &org.name, &f).await

}

pub async fn org_run(

    State(app): State<Shared>,

    viewer: Viewer,

    Path(name): Path<String>,

    Form(f): Form<Csrf>,

) -> Result<Page<BackupsPage>, Error> {

    let shell = Shell::new(&app, &format!("/orgs/{name}/settings/backups"), &viewer);

    let (_, org) = super::orgs::owned_org(&app, &shell, &viewer, &name).await?;

    csrf(&viewer, &f.csrf_token, &shell)?;

    run_now(&app, shell, "org", &org.name).await

}

//! Backups (docs/backups.md, oxygenhub#169), end to end: a real `oxy` push,

//! a backup to a directory, a second run that sends nothing new, and a

//! restore into a fresh hub that gives back the same repository, byte for

//! byte; and an owner's backup never holding another's repositories.

use oxy_core::repo::Repo;

use oxygenhub::backups::{self, Destination, Directory, Setting};

use oxygenhub::config::Config;

use oxygenhub::repos::RepoName;

use oxygenhub::{db, Shared};

use std::path::Path;

use std::sync::Arc;

fn hermetic() {

    static ONCE: std::sync::Once = std::sync::Once::new();

    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();

    ONCE.call_once(|| {

        let d = tempfile::tempdir().unwrap();

        std::env::set_var("OXY_HOME", d.path());

        DIR.set(d).ok();

    });

}

struct Hub {

    app: Shared,

    base: String,

    _dir: tempfile::TempDir,

}

async fn hub(backups: &Path) -> Hub {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let mut config = Config::load(dir.path()).unwrap();

    config.auth.push_tokens = vec!["site".into()];

    config.backups.directories = vec![backups.to_path_buf()];

    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();

    let port = listener.local_addr().unwrap().port();

    config.server.base_url = format!("http://127.0.0.1:{port}");

    let app = oxygenhub::build(config).await.unwrap();

    let router = oxygenhub::router(app.clone());

    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });

    Hub { app, base: format!("http://127.0.0.1:{port}"), _dir: dir }

}

fn write(root: &Path, rel: &str, content: &str) {

    let p = root.join(rel);

    std::fs::create_dir_all(p.parent().unwrap()).unwrap();

    std::fs::write(p, content).unwrap();

}

async fn create(app: &Shared, owner: &str, name: &str, visibility: hub_api::Visibility) {

    app.repos.create(&RepoName::parse(&format!("{owner}/{name}")).unwrap(), None).unwrap();

    db::repos::create(&app.db, owner, name, "a demo", visibility).await.unwrap();

}

fn push(url: &str, work: &Path, files: &[(&str, &str)]) {

    let mut c = Repo::init(work).unwrap();

    c.config.set("remote.origin.url", url).unwrap();

    c.config.set("remote.origin.token", "site").unwrap();

    c.save_config().unwrap();

    for (f, t) in files {

        write(work, f, t);

    }

    c.snapshot("t").unwrap();

    c.commit("Some work", &[]).unwrap();

    assert!(c.push("origin", false).unwrap().iter().all(|r| r.ok));

}

/// Every file under `dir`, with its bytes.

fn files(dir: &Path) -> Vec<(String, Vec<u8>)> {

    let mut out = Vec::new();

    let mut stack = vec![dir.to_path_buf()];

    while let Some(d) = stack.pop() {

        for e in std::fs::read_dir(&d).unwrap().flatten() {

            let p = e.path();

            if p.is_dir() {

                stack.push(p);

            } else {

                out.push((p.strip_prefix(dir).unwrap().to_string_lossy().into_owned(), std::fs::read(&p).unwrap()));

            }

        }

    }

    out.sort();

    out

}

#[tokio::test(flavor = "multi_thread")]

async fn a_backup_restores_the_same_repository_and_sends_only_what_changed() {

    let store = tempfile::tempdir().unwrap();

    let h = hub(store.path()).await;

    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();

    db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();

    create(&h.app, "andy", "demo", hub_api::Visibility::Private).await;

    create(&h.app, "bob", "other", hub_api::Visibility::Public).await;

    let work = tempfile::tempdir().unwrap();

    push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n"), ("src/main.rs", "fn main() {}\n")]);

    let other = tempfile::tempdir().unwrap();

    push(&format!("{}/bob/other", h.base), other.path(), &[("x.txt", "bob's\n")]);

    // Andy backs up to the directory the hub allows.

    let target = store.path().display().to_string();

    let mut s = Setting::off("andy", "user");

    s.every = "hourly".into();

    s.target = target.clone();

    backups::save(&h.app.db, &s).await.unwrap();

    let first = backups::run(&h.app, "andy", "now").await.unwrap();

    assert_eq!((first.status.as_str(), first.repos), ("ok", 1), "{}", first.detail);

    assert!(first.sent > 0 && first.bytes > 0);

    let dest = Directory { root: store.path().join("andy") };

    assert!(dest.exists(&format!("runs/{}/DONE", first.at)).unwrap());

    let records: serde_json::Value = serde_json::from_slice(&dest.get(&format!("runs/{}/records/demo.json", first.at)).unwrap()).unwrap();

    assert_eq!(records["repositories"][0]["visibility"], "private");

    let text = String::from_utf8_lossy(&dest.get(&format!("runs/{}/records/demo.json", first.at)).unwrap()).into_owned();

    assert!(!text.contains("bob/other") && !text.contains("\"other\""), "nothing of another owner");

    assert!(!store.path().join("bob").exists(), "bob set up no backup");

    // Nothing changed: the next run sends nothing new.

    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;

    let second = backups::run(&h.app, "andy", "now").await.unwrap();

    assert_eq!((second.status.as_str(), second.sent), ("ok", 0), "{}", second.detail);

    // A fresh hub, restored from the backup, has the same repository.

    let empty = tempfile::tempdir().unwrap();

    let fresh = hub(empty.path()).await;

    let dest: Arc<dyn Destination> = Arc::new(Directory { root: store.path().join("andy") });

    let done = backups::restore_into_hub(&fresh.app, dest, None).await.unwrap();

    assert_eq!(done, vec!["andy/demo (repos)"]);

    let name = RepoName::parse("andy/demo").unwrap();

    assert_eq!(files(&fresh.app.repos.dir(&name)), files(&h.app.repos.dir(&name)), "byte for byte");

    let row = db::repos::get(&fresh.app.db, "andy", "demo").await.unwrap().expect("listed on the hub");

    assert_eq!((row.visibility.as_str(), row.description.as_str()), ("private", "a demo"));

    let (a, b) = (h.app.repos.open(&name).unwrap(), fresh.app.repos.open(&name).unwrap());

    assert_eq!(

        a.bookmarks().unwrap().iter().map(|b| b.heads.clone()).collect::<Vec<_>>(),

        b.bookmarks().unwrap().iter().map(|b| b.heads.clone()).collect::<Vec<_>>()

    );

    // A directory the hub does not allow is refused.

    let mut bad = Setting::off("andy", "user");

    bad.every = "hourly".into();

    bad.target = "/tmp/somewhere-else".into();

    backups::save(&h.app.db, &bad).await.unwrap();

    let refused = backups::run(&h.app, "andy", "now").await.unwrap();

    assert_eq!(refused.status, "failed");

    assert!(refused.detail.contains("not a backup directory this hub allows"), "{}", refused.detail);

}

#[tokio::test(flavor = "multi_thread")]

async fn backups_fall_due_on_their_schedule() {

    let store = tempfile::tempdir().unwrap();

    let h = hub(store.path()).await;

    let mut s = Setting::off("andy", "user");

    s.target = store.path().display().to_string();

    backups::save(&h.app.db, &s).await.unwrap();

    assert!(backups::due(&h.app.db).await.unwrap().is_empty(), "off is never due");

    s.every = "hourly".into();

    backups::save(&h.app.db, &s).await.unwrap();

    assert_eq!(backups::due(&h.app.db).await.unwrap().len(), 1, "never run: due");

    sqlx::query("INSERT INTO backup_runs (owner, started_ms, status) VALUES ('andy', ?, 'ok')")

        .bind(db::now_ms())

        .execute(&h.app.db)

        .await

        .unwrap();

    assert!(backups::due(&h.app.db).await.unwrap().is_empty(), "ran just now");

}

{% extends "base.html" %}

{% import "settings_tabs.html" as t %}

{% block title %}Backups{% if kind == "org" %} · {{ owner }}{% endif %}{% endblock %}

{% block tabs %}{% if kind == "user" %}{% call t::tabs("backups") %}{% endif %}{% endblock %}

{% block content %}

<div class="head"><h1>Backups <small>{% if kind == "org" %}<a href="/orgs/{{ owner }}/settings">{{ owner }}</a>'s repositories{% else %}your repositories{% endif %}, copied to a place you control</small></h1></div>

{% if let Some(n) = notice %}<p class="note">{{ n }}</p>{% endif %}

{% if let Some(e) = error %}<p class="error">{{ e }}</p>{% endif %}

<p class="small mute backup-what">Each run copies every repository {% if kind == "org" %}the organisation{% else %}you{% endif %} own, with its wiki, byte for byte (signatures and all), its issues, proposals, releases and settings (never a secret value){% if kind == "org" %}, and the organisation's members and teams{% endif %}. Only what changed since the last run travels, and any kept run can be restored with <code>oxygenhub admin restore</code>.</p>

<form class="card wide backup-form" method="post" action="{{ base }}">

  <input type="hidden" name="_csrf" value="{{ shell.csrf }}">

  <h2>Where and how often</h2>

  <fieldset class="choices">

    <legend class="small mute">How often</legend>

    {% for (v, label) in every() %}<label class="choice"><input type="radio" name="every" value="{{ v }}"{% if setting.every == *v %} checked{% endif %}> {{ label }}</label>{% endfor %}

  </fieldset>

  <fieldset class="choices">

    <legend class="small mute">Where</legend>

    {% if !dirs.is_empty() %}<label class="choice"><input type="radio" name="destination" value="dir"{% if setting.destination == "dir" %} checked{% endif %}> A directory on this hub's host</label>{% endif %}

    <label class="choice"><input type="radio" name="destination" value="gcs"{% if setting.destination == "gcs" || dirs.is_empty() %} checked{% endif %}> A Google Cloud Storage bucket, with no key</label>

  </fieldset>

  {% if !dirs.is_empty() %}

  <label for="dir">Directory</label>

  <select id="dir" name="dir">{% for d in dirs %}<option value="{{ d }}"{% if setting.destination == "dir" && setting.target == *d %} selected{% endif %}>{{ d }}/{{ owner }}</option>{% endfor %}</select>

  {% endif %}

  <label for="bucket">GCS bucket</label>

  <input type="text" id="bucket" name="bucket" placeholder="{{ owner }}-oxygen-backup" value="{% if setting.destination == "gcs" %}{{ setting.target }}{% endif %}">

  <label for="provider">Workload identity provider</label>

  <input type="text" id="provider" name="provider" placeholder="//iam.googleapis.com/projects/NUMBER/locations/global/workloadIdentityPools/oxygenhub/providers/oxygenhub-io" value="{{ setting.provider }}">

  <p class="small mute">The hub signs in as <code>backup:{{ owner }}</code> with a token it issues itself; Google exchanges it for a few minutes' access to this bucket. No key file exists. One-time setup: <a href="/help#backups">Help › Backups</a> (docs/backups.md §6).</p>

  <label for="prefix">Prefix (optional)</label>

  <input type="text" id="prefix" name="prefix" value="{{ setting.prefix }}" placeholder="oxygenhub">

  <label for="keep_days">Keep runs for</label>

  <select id="keep_days" name="keep_days">{% for k in keeps() %}<option value="{{ k }}"{% if setting.keep_days == *k %} selected{% endif %}>{{ k }} days</option>{% endfor %}</select>

  <button type="submit">SAVE</button>

</form>

<section class="card-box" id="runs">

  <div class="card-head"><h2>Runs</h2><span class="count-chip">{{ runs.len() }}</span>

    <form class="inline more" method="post" action="{{ base }}/run"><input type="hidden" name="_csrf" value="{{ shell.csrf }}"><button type="submit" class="small primary-btn"{% if setting.updated_ms == 0 %} disabled title="Save where backups go first"{% endif %}>Back up now</button></form>

  </div>

  {% if runs.is_empty() %}<p class="empty small">No runs yet.{% if setting.every == "off" %} Pick how often and where, then save.{% endif %}</p>

  {% else %}

  <div class="table"><table class="backup-runs">

    <tr><th>Started</th><th>Took</th><th>Outcome</th><th>Repositories</th><th>Files sent</th><th>Size sent</th><th></th></tr>

    {% for r in runs %}

    <tr>

      <td><time title="{{ r.started_ms|when_i }}">{{ r.started_ms|ago_i }}</time>{% if r.trigger == "now" %} <span class="mute small">by hand</span>{% endif %}</td>

      <td>{% if let Some(f) = r.finished_ms %}{{ ((f - r.started_ms) / 1000) }} s{% else %}<span class="mute">running</span>{% endif %}</td>

      <td><span class="backup-status st-{{ r.status }}">{{ r.status }}</span></td>

      <td>{{ r.repos }}</td>

      <td>{{ r.sent }} <span class="mute small">of {{ r.files }}</span></td>

      <td>{{ (r.bytes / 1024) }} KiB</td>

      <td class="small mute">{{ r.detail }}</td>

    </tr>

    {% endfor %}

  </table></div>

  {% endif %}

</section>

{% endblock %}

- **Backups (#169, docs/backups.md):** a user (Settings › Backups) or an organisation's owners copy every repository they own, with its wiki, issues, proposals, releases and settings (never a secret value), hourly or daily, to a directory the hub allows. Only what changed travels, runs are kept 7, 30 or 90 days, a failure lands in the owner's inbox, and `oxygenhub admin backup restore` brings repositories back byte for byte. GCS, with no key, is next.

    pub backups: Backups,

/// Backups (docs/backups.md, oxygenhub#169): the directories on this host

/// an owner may back up to. Empty, only GCS destinations can be chosen.

#[derive(Clone, Debug, Default, Deserialize)]

#[serde(default, deny_unknown_fields)]

pub struct Backups {

    pub directories: Vec<PathBuf>,

}

            backups: Backups::default(),

pub mod backups;

    tokio::spawn(backups::scheduler(app.clone()));

    /// Backups (docs/backups.md)

    Backup {

        #[command(subcommand)]

        cmd: BackupCmd,

    },

}

#[derive(Subcommand)]

enum BackupCmd {

    /// Back up a user's or organisation's repositories now, as its settings say

    Run { owner: String },

    /// Restore every repository of a whole run from a backup directory

    /// (the one holding LATEST, runs/ and blobs/) into this hub

    Restore {

        #[arg(long)]

        from: std::path::PathBuf,

        /// The run (runs/<RUN>); the latest whole one when not given

        #[arg(long)]

        run: Option<String>,

    },

        }

        Cmd::Admin { cmd: AdminCmd::Backup { cmd } } => {

            let config = Config::load(&cli.data)?;

            let app = oxygenhub::build(config).await?;

            match cmd {