Backups to S3, keylessly 7b9f4570daf5

-- S3 destinations (docs/backups.md §3, oxygenhub#178): the bucket's AWS

-- region, which signs every request.

ALTER TABLE backup_settings ADD COLUMN region TEXT NOT NULL DEFAULT '';

    /// An S3 bucket's region (migration 0063).

    #[sqlx(default)]

    pub region: String,

            region: String::new(),

        "INSERT INTO backup_settings (owner, kind, every, destination, target, prefix, provider, keep_days, updated_ms, region) \

         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(owner) DO UPDATE SET kind = excluded.kind, every = excluded.every, \

         keep_days = excluded.keep_days, updated_ms = excluded.updated_ms, region = excluded.region",

    .bind(&s.region)

            "s3" if s.prefix.is_empty() => format!("s3://{} ({})", s.target, s.region),

            "s3" => format!("s3://{}/{} ({})", s.target, s.prefix, s.region),

/// An S3 bucket, written keylessly (§3): the hub's own short-lived token

/// (`sub = backup:OWNER`, audience `sts.amazonaws.com`) is exchanged at AWS

/// STS (AssumeRoleWithWebIdentity) for an hour's credentials of the role

/// the owner names, and they sign each request (Signature Version 4).

pub struct S3 {

    bucket: String,

    prefix: String,

    region: String,

    /// The bucket's endpoint: scheme, host and any path before keys.

    base: reqwest::Url,

    sts: String,

    /// The IAM role's ARN.

    role: String,

    subject: String,

    app: Shared,

    client: reqwest::Client,

    rt: tokio::runtime::Handle,

    creds: std::sync::Mutex<Option<Creds>>,

}

#[derive(Clone)]

struct Creds {

    id: String,

    secret: String,

    token: String,

    until: i64,

}

/// The audience AWS expects of a web identity token.

pub const AWS_AUDIENCE: &str = "sts.amazonaws.com";

/// Percent-encoding as AWS signs it: everything but unreserved characters,

/// and `/` too unless `keep_slash`.

pub fn aws_encode(s: &str, keep_slash: bool) -> String {

    let mut out = String::new();

    for b in s.bytes() {

        match b {

            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(b as char),

            b'/' if keep_slash => out.push('/'),

            _ => out.push_str(&format!("%{b:02X}")),

        }

    }

    out

}

fn hmac(key: &[u8], data: &str) -> Vec<u8> {

    use hmac::Mac;

    let mut m = hmac::Hmac::<sha2::Sha256>::new_from_slice(key).expect("any key length");

    m.update(data.as_bytes());

    m.finalize().into_bytes().to_vec()

}

fn sha256_hex(data: &[u8]) -> String {

    use sha2::Digest;

    hex::encode(sha2::Sha256::digest(data))

}

/// AWS Signature Version 4: the `Authorization` header for one request.

/// `uri` is the path as encoded on the wire, `query` the decoded pairs,

/// `headers` the signed ones (any case), `payload` the body's SHA-256 in

/// hex, `amz_date` as `20130524T000000Z`.

#[allow(clippy::too_many_arguments)]

pub fn sigv4(

    method: &str,

    uri: &str,

    query: &[(String, String)],

    headers: &[(String, String)],

    payload: &str,

    (id, secret): (&str, &str),

    region: &str,

    service: &str,

    amz_date: &str,

) -> String {

    let mut q: Vec<(String, String)> = query.iter().map(|(k, v)| (aws_encode(k, false), aws_encode(v, false))).collect();

    q.sort();

    let query = q.iter().map(|(k, v)| format!("{k}={v}")).collect::<Vec<_>>().join("&");

    let mut h: Vec<(String, String)> = headers.iter().map(|(k, v)| (k.to_ascii_lowercase(), v.trim().to_string())).collect();

    h.sort();

    let canonical_headers: String = h.iter().map(|(k, v)| format!("{k}:{v}\n")).collect();

    let signed = h.iter().map(|(k, _)| k.as_str()).collect::<Vec<_>>().join(";");

    let request = format!("{method}\n{uri}\n{query}\n{canonical_headers}\n{signed}\n{payload}");

    let date = &amz_date[..8];

    let scope = format!("{date}/{region}/{service}/aws4_request");

    let to_sign = format!("AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{}", sha256_hex(request.as_bytes()));

    let key = hmac(&hmac(&hmac(&hmac(format!("AWS4{secret}").as_bytes(), date), region), service), "aws4_request");

    let signature = hex::encode(hmac(&key, &to_sign));

    format!("AWS4-HMAC-SHA256 Credential={id}/{scope}, SignedHeaders={signed}, Signature={signature}")

}

/// The text of the first `<tag>` in `xml`, unescaped.

fn xml_text(xml: &str, tag: &str) -> Option<String> {

    xml_blocks(xml, tag).into_iter().next().map(xml_unescape)

}

/// The inside of every `<tag>…</tag>` in `xml`.

fn xml_blocks<'a>(xml: &'a str, tag: &str) -> Vec<&'a str> {

    let (open, close) = (format!("<{tag}>"), format!("</{tag}>"));

    let mut out = Vec::new();

    let mut rest = xml;

    while let Some(i) = rest.find(&open) {

        let after = &rest[i + open.len()..];

        let Some(j) = after.find(&close) else { break };

        out.push(&after[..j]);

        rest = &after[j + close.len()..];

    }

    out

}

fn xml_unescape(s: &str) -> String {

    s.replace("&lt;", "<").replace("&gt;", ">").replace("&quot;", "\"").replace("&apos;", "'").replace("&amp;", "&")

}

fn amz_date(ms: i64) -> String {

    let t = time::OffsetDateTime::from_unix_timestamp_nanos(ms as i128 * 1_000_000).unwrap_or(time::OffsetDateTime::UNIX_EPOCH);

    format!("{:04}{:02}{:02}T{:02}{:02}{:02}Z", t.year(), u8::from(t.month()), t.day(), t.hour(), t.minute(), t.second())

}

impl S3 {

    pub fn new(app: &Shared, s: &Setting) -> S3 {

        let url = app.config.backups.s3_url.replace("{bucket}", &s.target).replace("{region}", &s.region);

        S3 {

            bucket: s.target.clone(),

            prefix: s.prefix.trim_matches('/').to_string(),

            region: s.region.clone(),

            base: reqwest::Url::parse(url.trim_end_matches('/')).unwrap_or_else(|_| reqwest::Url::parse("https://invalid.").unwrap()),

            sts: app.config.backups.aws_sts_url.trim_end_matches('/').to_string(),

            role: s.provider.clone(),

            subject: format!("backup:{}", s.owner),

            app: app.clone(),

            client: reqwest::Client::builder().timeout(std::time::Duration::from_secs(300)).build().unwrap_or_default(),

            rt: tokio::runtime::Handle::current(),

            creds: std::sync::Mutex::new(None),

        }

    }

    fn name(&self, key: &str) -> String {

        if self.prefix.is_empty() {

            key.to_string()

        } else {

            format!("{}/{key}", self.prefix)

        }

    }

    /// Credentials of the owner's role, exchanged again when the last are

    /// within five minutes of running out.

    fn creds(&self) -> Result<Creds> {

        let now = db::now_ms();

        if let Some(c) = self.creds.lock().unwrap().as_ref() {

            if c.until - now > 300_000 {

                return Ok(c.clone());

            }

        }

        let session: String = format!("oxygenhub-{}", self.subject.trim_start_matches("backup:"))

            .chars()

            .filter(|c| c.is_ascii_alphanumeric() || "=,.@-_".contains(*c))

            .take(64)

            .collect();

        let c = self.rt.block_on(async {

            let issuer = self.app.oidc().await?;

            let jwt = issuer.mint_for(self.app.config.base_url(), &self.subject, AWS_AUDIENCE, db::now_ms(), 600);

            let res = self

                .client

                .post(format!("{}/", self.sts))

                .form(&[

                    ("Action", "AssumeRoleWithWebIdentity"),

                    ("Version", "2011-06-15"),

                    ("RoleArn", self.role.as_str()),

                    ("RoleSessionName", session.as_str()),

                    ("WebIdentityToken", jwt.as_str()),

                    ("DurationSeconds", "3600"),

                ])

                .send()

                .await?;

            let status = res.status();

            let text = res.text().await.unwrap_or_default();

            if !status.is_success() {

                bail!(

                    "AWS STS refused {} for {}: {} (check the role's trust policy: this hub's address as its OIDC provider, audience {AWS_AUDIENCE}, and sub {})",

                    self.subject,

                    self.role,

                    xml_text(&text, "Message").unwrap_or_else(|| status.to_string()),

                    self.subject

                );

            }

            let get = |t: &str| xml_text(&text, t).with_context(|| format!("AWS STS gave no {t}"));

            anyhow::Ok(Creds { id: get("AccessKeyId")?, secret: get("SecretAccessKey")?, token: get("SessionToken")?, until: now + 3_600_000 })

        })?;

        *self.creds.lock().unwrap() = Some(c.clone());

        Ok(c)

    }

    /// One signed request: `key` is an object's (none for the bucket).

    fn send(

        &self,

        method: reqwest::Method,

        key: Option<&str>,

        query: &[(String, String)],

        body: Vec<u8>,

    ) -> Result<(reqwest::StatusCode, Vec<u8>)> {

        let c = self.creds()?;

        let base_path = self.base.path().trim_end_matches('/');

        let uri = match key {

            Some(k) => format!("{base_path}/{}", aws_encode(&self.name(k), true)),

            None => format!("{base_path}/"),

        };

        let host = match self.base.port() {

            Some(p) => format!("{}:{p}", self.base.host_str().unwrap_or_default()),

            None => self.base.host_str().unwrap_or_default().to_string(),

        };

        let payload = sha256_hex(&body);

        let date = amz_date(db::now_ms());

        let headers = vec![

            ("host".to_string(), host.clone()),

            ("x-amz-content-sha256".to_string(), payload.clone()),

            ("x-amz-date".to_string(), date.clone()),

            ("x-amz-security-token".to_string(), c.token.clone()),

        ];

        let auth = sigv4(method.as_str(), &uri, query, &headers, &payload, (&c.id, &c.secret), &self.region, "s3", &date);

        let mut q: Vec<(String, String)> = query.iter().map(|(k, v)| (aws_encode(k, false), aws_encode(v, false))).collect();

        q.sort();

        let qs = q.iter().map(|(k, v)| format!("{k}={v}")).collect::<Vec<_>>().join("&");

        let url = format!("{}://{host}{uri}{}{qs}", self.base.scheme(), if qs.is_empty() { "" } else { "?" });

        self.rt.block_on(async {

            let mut req = self

                .client

                .request(method, &url)

                .header("authorization", auth)

                .header("x-amz-content-sha256", &payload)

                .header("x-amz-date", &date)

                .header("x-amz-security-token", &c.token);

            if !body.is_empty() {

                req = req.header("content-type", "application/octet-stream").body(body);

            }

            let res = req.send().await?;

            let status = res.status();

            anyhow::Ok((status, res.bytes().await?.to_vec()))

        })

    }

    fn refused(&self, what: &str, status: reqwest::StatusCode, body: &[u8]) -> anyhow::Error {

        let text = String::from_utf8_lossy(body);

        let hint = if status == reqwest::StatusCode::FORBIDDEN {

            format!(" (the role {} needs s3:GetObject, s3:PutObject, s3:DeleteObject and s3:ListBucket on {})", self.role, self.bucket)

        } else {

            String::new()

        };

        anyhow::anyhow!("S3 {what}: {status}{hint}: {}", xml_text(&text, "Message").unwrap_or_else(|| text.chars().take(200).collect()))

    }

    /// The keys under `full` (`delimited`: and the next level's prefixes).

    fn listing(&self, full: &str, delimited: bool) -> Result<(Vec<String>, Vec<String>)> {

        let (mut keys, mut prefixes) = (Vec::new(), Vec::new());

        let mut token: Option<String> = None;

        loop {

            let mut q = vec![("list-type".to_string(), "2".to_string()), ("prefix".to_string(), full.to_string())];

            if delimited {

                q.push(("delimiter".to_string(), "/".to_string()));

            }

            if let Some(t) = &token {

                q.push(("continuation-token".to_string(), t.clone()));

            }

            let (status, body) = self.send(reqwest::Method::GET, None, &q, Vec::new())?;

            if !status.is_success() {

                return Err(self.refused("list", status, &body));

            }

            let text = String::from_utf8_lossy(&body).into_owned();

            for c in xml_blocks(&text, "Contents") {

                keys.extend(xml_text(c, "Key"));

            }

            for c in xml_blocks(&text, "CommonPrefixes") {

                prefixes.extend(xml_text(c, "Prefix"));

            }

            match (xml_text(&text, "IsTruncated").as_deref(), xml_text(&text, "NextContinuationToken")) {

                (Some("true"), Some(t)) => token = Some(t),

                _ => return Ok((keys, prefixes)),

            }

        }

    }

}

impl Destination for S3 {

    fn exists(&self, key: &str) -> Result<bool> {

        let (status, body) = self.send(reqwest::Method::HEAD, Some(key), &[], Vec::new())?;

        match status {

            s if s.is_success() => Ok(true),

            reqwest::StatusCode::NOT_FOUND => Ok(false),

            s => Err(self.refused("read", s, &body)),

        }

    }

    fn put(&self, key: &str, bytes: &[u8]) -> Result<()> {

        let (status, body) = self.send(reqwest::Method::PUT, Some(key), &[], bytes.to_vec())?;

        if !status.is_success() {

            return Err(self.refused("write", status, &body));

        }

        Ok(())

    }

    fn get(&self, key: &str) -> Result<Vec<u8>> {

        let (status, body) = self.send(reqwest::Method::GET, Some(key), &[], Vec::new())?;

        if !status.is_success() {

            return Err(self.refused(&format!("read {key}"), status, &body));

        }

        Ok(body)

    }

    fn list(&self, prefix: &str) -> Result<Vec<String>> {

        let full = format!("{}/", self.name(prefix.trim_end_matches('/')));

        let (keys, prefixes) = self.listing(&full, true)?;

        let mut out: std::collections::BTreeSet<String> = keys.iter().map(|k| k.trim_start_matches(&full).to_string()).collect();

        out.extend(prefixes.iter().map(|p| p.trim_start_matches(&full).trim_end_matches('/').to_string()));

        Ok(out.into_iter().filter(|n| !n.is_empty()).collect())

    }

    fn remove(&self, key: &str) -> Result<()> {

        let full = self.name(key);

        let (mut keys, _) = self.listing(&format!("{full}/"), false)?;

        keys.push(full);

        for k in keys {

            let rel = if self.prefix.is_empty() { k.clone() } else { k.trim_start_matches(&format!("{}/", self.prefix)).to_string() };

            let (status, body) = self.send(reqwest::Method::DELETE, Some(&rel), &[], Vec::new())?;

            if !status.is_success() && status != reqwest::StatusCode::NOT_FOUND {

                return Err(self.refused("remove", status, &body));

            }

        }

        Ok(())

    }

}

        }

        "s3" => {

            if s.target.is_empty() || s.provider.is_empty() || s.region.is_empty() {

                bail!("an S3 destination needs its bucket, its region and the IAM role the hub assumes");

            }

            Ok(Box::new(S3::new(app, s)))

    }

    #[test]

    fn requests_are_signed_as_aws_documents() {

        // AWS's own example (S3 API reference, "Signature calculations":

        // GET Object), with its published signature.

        let empty = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";

        let headers = vec![

            ("Host".to_string(), "examplebucket.s3.amazonaws.com".to_string()),

            ("Range".to_string(), "bytes=0-9".to_string()),

            ("x-amz-content-sha256".to_string(), empty.to_string()),

            ("x-amz-date".to_string(), "20130524T000000Z".to_string()),

        ];

        let auth = sigv4(

            "GET",

            "/test.txt",

            &[],

            &headers,

            empty,

            ("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"),

            "us-east-1",

            "s3",

            "20130524T000000Z",

        );

        assert_eq!(

            auth,

            "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, \

             SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, \

             Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"

        );

        assert_eq!(aws_encode("runs/2026 x+y", true), "runs/2026%20x%2By");

        assert_eq!(aws_encode("a/b", false), "a%2Fb");

        assert_eq!(xml_text("<A><Key>runs/x&amp;y</Key></A>", "Key").as_deref(), Some("runs/x&y"));

    /// AWS STS and S3, for S3 destinations; `{bucket}` and `{region}` in

    /// `s3_url` are filled in. Set only to point them elsewhere.

    pub aws_sts_url: String,

    pub s3_url: String,

            aws_sts_url: "https://sts.amazonaws.com".into(),

            s3_url: "https://{bucket}.s3.{region}.amazonaws.com".into(),

    /// An S3 destination's bucket, region and the IAM role the hub assumes.

    #[serde(default)]

    s3_bucket: String,

    #[serde(default)]

    s3_region: String,

    #[serde(default)]

    s3_role: String,

    let mut region = String::new();

        }

        "s3" => {

            let bucket = f.s3_bucket.trim().trim_start_matches("s3://").trim_end_matches('/').to_string();

            if bucket.is_empty() || !bucket.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || "-.".contains(c)) {

                return Err("Give the S3 bucket's name, such as andymccall-oxygen-backup.".into());

            }

            region = f.s3_region.trim().to_string();

            if region.is_empty() || !region.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') {

                return Err("Give the bucket's region, such as eu-west-2.".into());

            }

            let role = f.s3_role.trim().to_string();

            if !role.starts_with("arn:aws:iam::") || !role.contains(":role/") {

                return Err("Give the IAM role's ARN, arn:aws:iam::ACCOUNT:role/NAME (docs/backups.md §6).".into());

            }

            (bucket, role)

        region,

/// AWS STS and S3, as much as backups use (path style, under `/s3`): an

/// AssumeRoleWithWebIdentity that admits `backup:` subjects with audience

/// `sts.amazonaws.com`, and buckets (`OWNER-backup`) that check every

/// request's Signature Version 4 and admit only their owner's session.

async fn fake_aws() -> (String, Shared2) {

    use axum::extract::{Request, State};

    use axum::http::StatusCode;

    use axum::response::IntoResponse;

    let state: Shared2 = Default::default();

    async fn handle(State(g): State<Shared2>, req: Request) -> axum::response::Response {

        let (parts, body) = req.into_parts();

        let bytes = http_body_util::BodyExt::collect(body).await.unwrap().to_bytes();

        let path = parts.uri.path().to_string();

        let header = |h: &str| parts.headers.get(h).and_then(|v| v.to_str().ok()).unwrap_or("").to_string();

        if path == "/" && parts.method == "POST" {

            let form: Vec<(String, String)> = String::from_utf8_lossy(&bytes)

                .split('&')

                .filter_map(|kv| kv.split_once('=').map(|(k, v)| (k.to_string(), decode(&v.replace('+', " ")))))

                .collect();

            let field = |k: &str| form.iter().find(|(f, _)| f == k).map(|(_, v)| v.clone()).unwrap_or_default();

            let jwt = field("WebIdentityToken");

            let claims: serde_json::Value = serde_json::from_slice(&base64url(jwt.split('.').nth(1).unwrap())).unwrap();

            let (sub, aud) = (claims["sub"].as_str().unwrap().to_string(), claims["aud"].as_str().unwrap().to_string());

            g.lock().unwrap().exchanged.push((sub.clone(), aud.clone()));

            if field("Action") != "AssumeRoleWithWebIdentity" || !sub.starts_with("backup:") || aud != "sts.amazonaws.com" {

                return (StatusCode::FORBIDDEN, "<ErrorResponse><Error><Code>AccessDenied</Code><Message>Not authorized to perform sts:AssumeRoleWithWebIdentity</Message></Error></ErrorResponse>").into_response();

            }

            let owner = sub.trim_start_matches("backup:");

            return format!(

                "<AssumeRoleWithWebIdentityResponse><AssumeRoleWithWebIdentityResult><Credentials>\

                 <AccessKeyId>ASIA{owner}</AccessKeyId><SecretAccessKey>secret/{owner}</SecretAccessKey>\

                 <SessionToken>session-{owner}</SessionToken><Expiration>2030-01-01T00:00:00Z</Expiration>\

                 </Credentials></AssumeRoleWithWebIdentityResult></AssumeRoleWithWebIdentityResponse>"

            )

            .into_response();

        }

        // /s3/{bucket}/{key…}

        let rest = path.strip_prefix("/s3/").unwrap_or("");

        let (bucket, key) = rest.split_once('/').unwrap_or((rest, ""));

        let (bucket, key) = (bucket.to_string(), decode(key));

        let owner = bucket.trim_end_matches("-backup").to_string();

        // Check the signature as AWS would, with the session's secret.

        let session = header("x-amz-security-token");

        let who = session.trim_start_matches("session-").to_string();

        let query: Vec<(String, String)> = parts

            .uri

            .query()

            .unwrap_or("")

            .split('&')

            .filter(|kv| !kv.is_empty())

            .map(|kv| kv.split_once('=').map(|(k, v)| (decode(k), decode(v))).unwrap_or((decode(kv), String::new())))

            .collect();

        let signed = vec![

            ("host".to_string(), header("host")),

            ("x-amz-content-sha256".to_string(), header("x-amz-content-sha256")),

            ("x-amz-date".to_string(), header("x-amz-date")),

            ("x-amz-security-token".to_string(), session.clone()),

        ];

        let id = format!("ASIA{who}");

        let secret = format!("secret/{who}");

        let want = oxygenhub::backups::sigv4(

            parts.method.as_str(),

            &path,

            &query,

            &signed,

            &header("x-amz-content-sha256"),

            (&id, &secret),

            "eu-west-2",

            "s3",

            &header("x-amz-date"),

        );

        use sha2::Digest;

        let body_hash = hex::encode(sha2::Sha256::digest(&bytes));

        if header("authorization") != want || header("x-amz-content-sha256") != body_hash {

            return (

                StatusCode::FORBIDDEN,

                "<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>",

            )

                .into_response();

        }

        if who != owner {

            return (StatusCode::FORBIDDEN, "<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>").into_response();

        }

        let mut g = g.lock().unwrap();

        match (parts.method.as_str(), key.is_empty()) {

            ("PUT", false) => {

                g.objects.insert((bucket, key), bytes.to_vec());

                StatusCode::OK.into_response()

            }

            ("GET", false) | ("HEAD", false) => match g.objects.get(&(bucket, key)) {

                Some(b) => b.clone().into_response(),

                None => (StatusCode::NOT_FOUND, "<Error><Code>NoSuchKey</Code></Error>").into_response(),

            },

            ("DELETE", false) => {

                g.objects.remove(&(bucket, key));

                StatusCode::NO_CONTENT.into_response()

            }

            ("GET", true) => {

                let prefix = query.iter().find(|(k, _)| k == "prefix").map(|(_, v)| v.clone()).unwrap_or_default();

                let delimited = query.iter().any(|(k, _)| k == "delimiter");

                let (mut keys, mut prefixes) = (String::new(), std::collections::BTreeSet::new());

                for (b, n) in g.objects.keys() {

                    if *b != bucket || !n.starts_with(&prefix) {

                        continue;

                    }

                    match n[prefix.len()..].find('/') {

                        Some(i) if delimited => {

                            prefixes.insert(format!("{prefix}{}", &n[prefix.len()..prefix.len() + i + 1]));

                        }

                        _ => keys.push_str(&format!("<Contents><Key>{n}</Key></Contents>")),

                    }

                }

                let common: String = prefixes.iter().map(|p| format!("<CommonPrefixes><Prefix>{p}</Prefix></CommonPrefixes>")).collect();

                format!("<ListBucketResult><Prefix>{prefix}</Prefix><IsTruncated>false</IsTruncated>{keys}{common}</ListBucketResult>")

                    .into_response()

            }

            _ => StatusCode::METHOD_NOT_ALLOWED.into_response(),

        }

    }

    let app = axum::Router::new().fallback(handle).with_state(state.clone());

    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();

    let port = listener.local_addr().unwrap().port();

    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });

    (format!("http://127.0.0.1:{port}"), state)

}

async fn hub_aws(backups: &Path, aws: &str) -> Hub {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let mut config = Config::load(dir.path()).unwrap();

    config.auth.push_tokens = vec!["site".into()];

    config.backups.directories = vec![backups.to_path_buf()];

    config.backups.aws_sts_url = aws.to_string();

    config.backups.s3_url = format!("{aws}/s3/{{bucket}}");

    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();

    let port = listener.local_addr().unwrap().port();

    config.server.base_url = format!("http://127.0.0.1:{port}");

    let app = oxygenhub::build(config).await.unwrap();

    let router = oxygenhub::router(app.clone());

    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });

    Hub { app, base: format!("http://127.0.0.1:{port}"), _dir: dir }

}

const ROLE: &str = "arn:aws:iam::123456789012:role/oxygenhub-backup";

fn s3(owner: &str, bucket: &str) -> Setting {

    let mut s = Setting::off(owner, "user");

    s.every = "hourly".into();

    s.destination = "s3".into();

    s.target = bucket.into();

    s.provider = ROLE.into();

    s.region = "eu-west-2".into();

    s.prefix = "oxygen hub".into();

    s

}

/// docs/backups.md §3 (BP4): an S3 bucket, keylessly: the hub's own token

/// for AWS STS, then signed requests; a second run sends nothing; the

/// probe leaves the bucket as it was; a fresh hub restores the same

/// repository from it; another owner's session is refused, saying what the

/// role needs.

#[tokio::test(flavor = "multi_thread")]

async fn an_s3_backup_needs_no_key_and_restores_the_same_repository() {

    let (aws, seen) = fake_aws().await;

    let store = tempfile::tempdir().unwrap();

    let h = hub_aws(store.path(), &aws).await;

    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();

    db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();

    create(&h.app, "andy", "demo", hub_api::Visibility::Public).await;

    let work = tempfile::tempdir().unwrap();

    push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n"), ("src/a b+c.rs", "fn main() {}\n")]);

    backups::save(&h.app.db, &s3("andy", "andy-backup")).await.unwrap();

    let first = backups::run(&h.app, "andy", "now").await.unwrap();

    assert_eq!((first.status.as_str(), first.repos), ("ok", 1), "{}", first.detail);

    {

        let g = seen.lock().unwrap();

        assert!(

            g.objects.keys().any(|(b, n)| b == "andy-backup" && n == &format!("oxygen hub/runs/{}/DONE", first.at)),

            "under the prefix"

        );

        assert_eq!(g.exchanged.first().map(|(s, a)| (s.as_str(), a.as_str())), Some(("backup:andy", "sts.amazonaws.com")));

    }

    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;

    let second = backups::run(&h.app, "andy", "now").await.unwrap();

    assert_eq!((second.status.as_str(), second.sent), ("ok", 0), "{}", second.detail);

    let before = seen.lock().unwrap().objects.len();

    let dest = backups::destination(&h.app, &s3("andy", "andy-backup")).unwrap();

    tokio::task::spawn_blocking(move || backups::probe(dest.as_ref())).await.unwrap().unwrap();

    assert_eq!(seen.lock().unwrap().objects.len(), before, "the probe cleans up");

    let empty = tempfile::tempdir().unwrap();

    let fresh = hub_aws(empty.path(), &aws).await;

    let dest: Arc<dyn Destination> = backups::destination(&fresh.app, &s3("andy", "andy-backup")).unwrap().into();

    assert_eq!(backups::restore_into_hub(&fresh.app, dest, None).await.unwrap(), vec!["andy/demo (repos)"]);

    let name = RepoName::parse("andy/demo").unwrap();

    assert_eq!(files(&fresh.app.repos.dir(&name)), files(&h.app.repos.dir(&name)), "byte for byte");

    backups::save(&h.app.db, &s3("bob", "andy-backup")).await.unwrap();

    let refused = backups::run(&h.app, "bob", "now").await.unwrap();

    assert_eq!(refused.status, "failed");

    assert!(refused.detail.contains("s3:PutObject") && refused.detail.contains(ROLE), "{}", refused.detail);

}

    // An S3 bucket (BP4): its region and the role the hub assumes.

    let bad = post(

        app,

        "/settings/backups",

        Some(&andy),

        &format!("_csrf={csrf}&every=hourly&destination=s3&s3_bucket=andy-backup&s3_region=eu-west-2&s3_role=admin&keep_days=30"),

    )

    .await;

    assert!(bad.body.contains("the IAM role"), "{}", bad.body);

    let role = "arn%3Aaws%3Aiam%3A%3A123456789012%3Arole%2Foxygenhub-backup";

    let ok = post(

        app,

        "/settings/backups",

        Some(&andy),

        &format!(

            "_csrf={csrf}&every=daily&destination=s3&s3_bucket=s3%3A%2F%2Fandy-backup&s3_region=eu-west-2&s3_role={role}&keep_days=30"

        ),

    )

    .await;

    assert!(ok.body.contains("Saved.") && ok.body.contains(r#"value="s3" checked"#), "{}", ok.body);

    let s = oxygenhub::backups::setting(&app.db, "andy").await.unwrap().unwrap();

    assert_eq!((s.destination.as_str(), s.target.as_str(), s.region.as_str()), ("s3", "andy-backup", "eu-west-2"));

    assert_eq!(s.provider, "arn:aws:iam::123456789012:role/oxygenhub-backup");

    // Back to GCS for what follows.

    let ok = post(

        app,

        "/settings/backups",

        Some(&andy),

        &format!("_csrf={csrf}&every=hourly&destination=gcs&bucket=gs%3A%2F%2Fandy-backup&provider={provider}&keep_days=7"),

    )

    .await;

    assert!(ok.body.contains("Saved."));

per-repository settings (#177); S3 destinations, keylessly (#178).

- **An S3 bucket, keylessly** (built 2026-10-14, oxygenhub#178): the hub

  mints a 10-minute token with `sub = backup:OWNER` and audience

  `sts.amazonaws.com`, AWS STS's AssumeRoleWithWebIdentity gives an

  hour's credentials of the IAM role the owner names (exchanged again

  five minutes before they run out), and each request is signed with

  Signature Version 4 (checked against AWS's published example). The

  setting holds the bucket, its region and the role's ARN; setup is §6.

- **Later:** another hub.

### 6.1 Setting up an S3 destination (for Andy, by hand)

Once per AWS account, the hub as an identity provider (AWS fetches its

keys from `https://oxygenhub.io/.well-known/openid-configuration`):

```

aws iam create-open-id-connect-provider \

  --url https://oxygenhub.io --client-id-list sts.amazonaws.com

```

Once per owner and bucket (here `andymccall`, in `eu-west-2`):

```

B=andymccall-oxygen-backup; ACCOUNT=$(aws sts get-caller-identity --query Account --output text)

aws s3api create-bucket --bucket $B --region eu-west-2 \

  --create-bucket-configuration LocationConstraint=eu-west-2

aws s3api put-bucket-versioning --bucket $B --versioning-configuration Status=Enabled

aws s3api put-public-access-block --bucket $B --public-access-block-configuration \

  BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true

cat > trust.json <<JSON

{"Version": "2012-10-17", "Statement": [{"Effect": "Allow",

  "Principal": {"Federated": "arn:aws:iam::$ACCOUNT:oidc-provider/oxygenhub.io"},

  "Action": "sts:AssumeRoleWithWebIdentity",

  "Condition": {"StringEquals": {"oxygenhub.io:aud": "sts.amazonaws.com",

                                 "oxygenhub.io:sub": "backup:andymccall"}}}]}

JSON

cat > policy.json <<JSON

{"Version": "2012-10-17", "Statement": [

  {"Effect": "Allow", "Action": "s3:ListBucket", "Resource": "arn:aws:s3:::$B"},

  {"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],

   "Resource": "arn:aws:s3:::$B/*"}]}

JSON

aws iam create-role --role-name oxygenhub-backup-andymccall --assume-role-policy-document file://trust.json

aws iam put-role-policy --role-name oxygenhub-backup-andymccall --policy-name backup --policy-document file://policy.json

```

Then, in Settings › Backups: destination S3, the bucket, region

`eu-west-2`, role `arn:aws:iam::ACCOUNT:role/oxygenhub-backup-andymccall`,

SAVE and TEST. `s3:ListBucket` matters: without it S3 answers a missing

object with 403 rather than 404, and every run fails. No access key exists

anywhere.

3. Admin › Backups, then per-repository settings, then S3: all built

   2026-10-14 (#176, #177, #178).

    <label class="choice"><input type="radio" name="destination" value="gcs"{% if setting.destination == "gcs" || (dirs.is_empty() && setting.destination != "s3") %} checked{% endif %}> A Google Cloud Storage bucket, with no key</label>

    <label class="choice"><input type="radio" name="destination" value="s3"{% if setting.destination == "s3" %} checked{% endif %}> An Amazon S3 bucket, with no key</label>

  <input type="text" id="provider" name="provider" placeholder="//iam.googleapis.com/projects/NUMBER/locations/global/workloadIdentityPools/oxygenhub/providers/oxygenhub-io" value="{% if setting.destination == "gcs" %}{{ setting.provider }}{% endif %}">

  <label for="s3_bucket">S3 bucket</label>

  <input type="text" id="s3_bucket" name="s3_bucket" placeholder="{{ owner }}-oxygen-backup" value="{% if setting.destination == "s3" %}{{ setting.target }}{% endif %}">

  <label for="s3_region">S3 region</label>

  <input type="text" id="s3_region" name="s3_region" placeholder="eu-west-2" value="{{ setting.region }}">

  <label for="s3_role">IAM role</label>

  <input type="text" id="s3_role" name="s3_role" placeholder="arn:aws:iam::ACCOUNT:role/oxygenhub-backup-{{ owner }}" value="{% if setting.destination == "s3" %}{{ setting.provider }}{% endif %}">

  <p class="small mute">The hub signs in to AWS as <code>backup:{{ owner }}</code> with a token it issues itself (audience <code>sts.amazonaws.com</code>); AWS gives it an hour's credentials of this role. No access key exists. One-time setup: docs/backups.md §6.</p>