OXYGENHUB
◆ v0.5.2 oxygenhub / crates / hub / tests / backups.rs
45.2 KiB · Rust · last changed Notifications that reach you, and 2FA without the password twice 21 days ago source blame history raw
1
//! Backups (docs/backups.md, oxygenhub#169), end to end: a real `oxy` push,
2
//! a backup to a directory, a second run that sends nothing new, and a
3
//! restore into a fresh hub that gives back the same repository, byte for
4
//! byte; and an owner's backup never holding another's repositories.
5
6
use oxy_core::repo::Repo;
7
use oxygenhub::backups::{self, Destination, Directory, Setting};
8
use oxygenhub::config::Config;
9
use oxygenhub::repos::RepoName;
10
use oxygenhub::{db, Shared};
11
use std::path::Path;
12
use std::sync::Arc;
13
14
fn hermetic() {
15
    static ONCE: std::sync::Once = std::sync::Once::new();
16
    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();
17
    ONCE.call_once(|| {
18
        let d = tempfile::tempdir().unwrap();
19
        std::env::set_var("OXY_HOME", d.path());
20
        DIR.set(d).ok();
21
    });
22
}
23
24
struct Hub {
25
    app: Shared,
26
    base: String,
27
    _dir: tempfile::TempDir,
28
}
29
30
async fn hub(backups: &Path) -> Hub {
31
    hub_with(backups, None).await
32
}
33
34
async fn hub_with(backups: &Path, google: Option<&str>) -> Hub {
35
    hermetic();
36
    let dir = tempfile::tempdir().unwrap();
37
    let mut config = Config::load(dir.path()).unwrap();
38
    config.auth.push_tokens = vec!["site".into()];
39
    config.backups.directories = vec![backups.to_path_buf()];
40
    if let Some(g) = google {
41
        config.backups.sts_url = g.to_string();
42
        config.backups.storage_url = g.to_string();
43
    }
44
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
45
    let port = listener.local_addr().unwrap().port();
46
    config.server.base_url = format!("http://127.0.0.1:{port}");
47
    let app = oxygenhub::build(config).await.unwrap();
48
    let router = oxygenhub::router(app.clone());
49
    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
50
    Hub { app, base: format!("http://127.0.0.1:{port}"), _dir: dir }
51
}
52
53
fn write(root: &Path, rel: &str, content: &str) {
54
    let p = root.join(rel);
55
    std::fs::create_dir_all(p.parent().unwrap()).unwrap();
56
    std::fs::write(p, content).unwrap();
57
}
58
59
async fn create(app: &Shared, owner: &str, name: &str, visibility: hub_api::Visibility) {
60
    app.repos.create(&RepoName::parse(&format!("{owner}/{name}")).unwrap(), None).unwrap();
61
    db::repos::create(&app.db, owner, name, "a demo", visibility).await.unwrap();
62
}
63
64
fn push(url: &str, work: &Path, files: &[(&str, &str)]) {
65
    let mut c = Repo::init(work).unwrap();
66
    c.config.set("remote.origin.url", url).unwrap();
67
    c.config.set("remote.origin.token", "site").unwrap();
68
    c.save_config().unwrap();
69
    for (f, t) in files {
70
        write(work, f, t);
71
    }
72
    c.snapshot("t").unwrap();
73
    c.commit("Some work", &[]).unwrap();
74
    assert!(c.push("origin", false).unwrap().iter().all(|r| r.ok));
75
}
76
77
/// Every file under `dir`, with its bytes.
78
fn files(dir: &Path) -> Vec<(String, Vec<u8>)> {
79
    let mut out = Vec::new();
80
    let mut stack = vec![dir.to_path_buf()];
81
    while let Some(d) = stack.pop() {
82
        for e in std::fs::read_dir(&d).unwrap().flatten() {
83
            let p = e.path();
84
            if p.is_dir() {
85
                stack.push(p);
86
            } else {
87
                out.push((p.strip_prefix(dir).unwrap().to_string_lossy().into_owned(), std::fs::read(&p).unwrap()));
88
            }
89
        }
90
    }
91
    out.sort();
92
    out
93
}
94
95
#[tokio::test(flavor = "multi_thread")]
96
async fn a_backup_restores_the_same_repository_and_sends_only_what_changed() {
97
    let store = tempfile::tempdir().unwrap();
98
    let h = hub(store.path()).await;
99
    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
100
    db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();
101
    create(&h.app, "andy", "demo", hub_api::Visibility::Private).await;
102
    create(&h.app, "bob", "other", hub_api::Visibility::Public).await;
103
    let work = tempfile::tempdir().unwrap();
104
    push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n"), ("src/main.rs", "fn main() {}\n")]);
105
    let other = tempfile::tempdir().unwrap();
106
    push(&format!("{}/bob/other", h.base), other.path(), &[("x.txt", "bob's\n")]);
107
108
    // Andy backs up to the directory the hub allows.
109
    let target = store.path().display().to_string();
110
    let mut s = Setting::off("andy", "user");
111
    s.every = "hourly".into();
112
    s.target = target.clone();
113
    backups::save(&h.app.db, &s).await.unwrap();
114
    let first = backups::run(&h.app, "andy", "now").await.unwrap();
115
    assert_eq!((first.status.as_str(), first.repos), ("ok", 1), "{}", first.detail);
116
    assert!(first.sent > 0 && first.bytes > 0);
117
    let dest = Directory { root: store.path().join("andy") };
118
    assert!(dest.exists(&format!("runs/{}/DONE", first.at)).unwrap());
119
    let records: serde_json::Value = serde_json::from_slice(&dest.get(&format!("runs/{}/records/demo.json", first.at)).unwrap()).unwrap();
120
    assert_eq!(records["repositories"][0]["visibility"], "private");
121
    let text = String::from_utf8_lossy(&dest.get(&format!("runs/{}/records/demo.json", first.at)).unwrap()).into_owned();
122
    assert!(!text.contains("bob/other") && !text.contains("\"other\""), "nothing of another owner");
123
    assert!(!store.path().join("bob").exists(), "bob set up no backup");
124
125
    // Nothing changed: the next run sends nothing new.
126
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
127
    let second = backups::run(&h.app, "andy", "now").await.unwrap();
128
    assert_eq!((second.status.as_str(), second.sent), ("ok", 0), "{}", second.detail);
129
130
    // A fresh hub, restored from the backup, has the same repository.
131
    let empty = tempfile::tempdir().unwrap();
132
    let fresh = hub(empty.path()).await;
133
    let dest: Arc<dyn Destination> = Arc::new(Directory { root: store.path().join("andy") });
134
    let done = backups::restore_into_hub(&fresh.app, dest, None).await.unwrap();
135
    assert_eq!(done, vec!["andy/demo (repos)"]);
136
    let name = RepoName::parse("andy/demo").unwrap();
137
    assert_eq!(files(&fresh.app.repos.dir(&name)), files(&h.app.repos.dir(&name)), "byte for byte");
138
    let row = db::repos::get(&fresh.app.db, "andy", "demo").await.unwrap().expect("listed on the hub");
139
    assert_eq!((row.visibility.as_str(), row.description.as_str()), ("private", "a demo"));
140
    let (a, b) = (h.app.repos.open(&name).unwrap(), fresh.app.repos.open(&name).unwrap());
141
    assert_eq!(
142
        a.bookmarks().unwrap().iter().map(|b| b.heads.clone()).collect::<Vec<_>>(),
143
        b.bookmarks().unwrap().iter().map(|b| b.heads.clone()).collect::<Vec<_>>()
144
    );
145
146
    // A directory the hub does not allow is refused.
147
    let mut bad = Setting::off("andy", "user");
148
    bad.every = "hourly".into();
149
    bad.target = "/tmp/somewhere-else".into();
150
    backups::save(&h.app.db, &bad).await.unwrap();
151
    let refused = backups::run(&h.app, "andy", "now").await.unwrap();
152
    assert_eq!(refused.status, "failed");
153
    assert!(refused.detail.contains("not a backup directory this hub allows"), "{}", refused.detail);
154
}
155
156
#[tokio::test(flavor = "multi_thread")]
157
async fn backups_fall_due_on_their_schedule() {
158
    let store = tempfile::tempdir().unwrap();
159
    let h = hub(store.path()).await;
160
    let mut s = Setting::off("andy", "user");
161
    s.target = store.path().display().to_string();
162
    backups::save(&h.app.db, &s).await.unwrap();
163
    assert!(backups::due(&h.app.db).await.unwrap().is_empty(), "off is never due");
164
    s.every = "hourly".into();
165
    backups::save(&h.app.db, &s).await.unwrap();
166
    assert_eq!(backups::due(&h.app.db).await.unwrap().len(), 1, "never run: due");
167
    sqlx::query("INSERT INTO backup_runs (owner, started_ms, status) VALUES ('andy', ?, 'ok')")
168
        .bind(db::now_ms())
169
        .execute(&h.app.db)
170
        .await
171
        .unwrap();
172
    assert!(backups::due(&h.app.db).await.unwrap().is_empty(), "ran just now");
173
}
174
175
// ------------------------------------------------------------ a fake Google
176
177
/// What the fake has seen and holds.
178
#[derive(Default)]
179
struct Google {
180
    /// Objects by bucket and name.
181
    objects: std::collections::BTreeMap<(String, String), Vec<u8>>,
182
    /// The subjects and audiences of the tokens exchanged.
183
    exchanged: Vec<(String, String)>,
184
}
185
186
type Shared2 = Arc<std::sync::Mutex<Google>>;
187
188
fn decode(s: &str) -> String {
189
    let b = s.as_bytes();
190
    let mut out = Vec::new();
191
    let mut i = 0;
192
    while i < b.len() {
193
        if b[i] == b'%' && i + 2 < b.len() {
194
            out.push(u8::from_str_radix(&s[i + 1..i + 3], 16).unwrap());
195
            i += 3;
196
        } else {
197
            out.push(b[i]);
198
            i += 1;
199
        }
200
    }
201
    String::from_utf8(out).unwrap()
202
}
203
204
fn query(uri: &axum::http::Uri, key: &str) -> Option<String> {
205
    uri.query()?.split('&').find_map(|kv| kv.split_once('=').filter(|(k, _)| *k == key).map(|(_, v)| decode(v)))
206
}
207
208
/// Google's STS and Cloud Storage, as much as backups use: a token
209
/// exchange that admits `backup:` subjects for the provider's audience,
210
/// and buckets that admit only their owner's token (bucket `OWNER-backup`).
211
async fn fake_google() -> (String, Shared2) {
212
    use axum::extract::{Request, State};
213
    use axum::http::StatusCode;
214
    use axum::response::IntoResponse;
215
    let state: Shared2 = Default::default();
216
    async fn handle(State(g): State<Shared2>, req: Request) -> axum::response::Response {
217
        let (parts, body) = req.into_parts();
218
        let bytes = http_body_util::BodyExt::collect(body).await.unwrap().to_bytes();
219
        let path = parts.uri.path().to_string();
220
        if path == "/v1/token" {
221
            let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
222
            let jwt = v["subjectToken"].as_str().unwrap();
223
            let payload = jwt.split('.').nth(1).unwrap();
224
            let claims: serde_json::Value = serde_json::from_slice(&base64url(payload)).unwrap();
225
            let (sub, aud) = (claims["sub"].as_str().unwrap().to_string(), claims["aud"].as_str().unwrap().to_string());
226
            let provider = v["audience"].as_str().unwrap();
227
            g.lock().unwrap().exchanged.push((sub.clone(), aud.clone()));
228
            if !sub.starts_with("backup:") || aud != format!("https:{provider}") {
229
                return (
230
                    StatusCode::BAD_REQUEST,
231
                    axum::Json(serde_json::json!({"error": "invalid_grant", "error_description": "not admitted"})),
232
                )
233
                    .into_response();
234
            }
235
            return axum::Json(serde_json::json!({ "access_token": format!("tok-{sub}"), "expires_in": 3600 })).into_response();
236
        }
237
        let auth = parts.headers.get("authorization").and_then(|v| v.to_str().ok()).unwrap_or("").to_string();
238
        let segs: Vec<&str> = path.split('/').collect();
239
        // /storage/v1/b/{bucket}/o[/{name}]  or  /upload/storage/v1/b/{bucket}/o
240
        let upload = segs.get(1) == Some(&"upload");
241
        let at = if upload { 5 } else { 4 };
242
        let bucket = segs.get(at).unwrap_or(&"").to_string();
243
        let owner = bucket.trim_end_matches("-backup");
244
        if auth != format!("Bearer tok-backup:{owner}") {
245
            return (StatusCode::FORBIDDEN, "caller does not have storage.objects access").into_response();
246
        }
247
        let mut g = g.lock().unwrap();
248
        if upload {
249
            // As Google does: an upload must say its length, an empty one too.
250
            if !parts.headers.contains_key("content-length") {
251
                return (StatusCode::LENGTH_REQUIRED, "Error 411 (Length Required)").into_response();
252
            }
253
            let name = query(&parts.uri, "name").unwrap();
254
            g.objects.insert((bucket, name), bytes.to_vec());
255
            return axum::Json(serde_json::json!({})).into_response();
256
        }
257
        let name = segs.get(at + 2).map(|n| decode(n));
258
        match (parts.method.as_str(), name) {
259
            ("GET", Some(n)) => match g.objects.get(&(bucket, n.clone())) {
260
                Some(b) if query(&parts.uri, "alt").as_deref() == Some("media") => b.clone().into_response(),
261
                Some(_) => axum::Json(serde_json::json!({ "name": n })).into_response(),
262
                None => StatusCode::NOT_FOUND.into_response(),
263
            },
264
            ("DELETE", Some(n)) => {
265
                g.objects.remove(&(bucket, n));
266
                StatusCode::NO_CONTENT.into_response()
267
            }
268
            ("GET", None) => {
269
                let prefix = query(&parts.uri, "prefix").unwrap_or_default();
270
                let delimited = query(&parts.uri, "delimiter").is_some();
271
                let (mut items, mut prefixes) = (Vec::new(), std::collections::BTreeSet::new());
272
                for (b, n) in g.objects.keys() {
273
                    if *b != bucket || !n.starts_with(&prefix) {
274
                        continue;
275
                    }
276
                    let rest = &n[prefix.len()..];
277
                    match rest.find('/') {
278
                        Some(i) if delimited => {
279
                            prefixes.insert(format!("{prefix}{}", &rest[..=i]));
280
                        }
281
                        _ => items.push(serde_json::json!({ "name": n })),
282
                    }
283
                }
284
                axum::Json(serde_json::json!({ "items": items, "prefixes": prefixes })).into_response()
285
            }
286
            _ => StatusCode::METHOD_NOT_ALLOWED.into_response(),
287
        }
288
    }
289
    let app = axum::Router::new().fallback(handle).with_state(state.clone());
290
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
291
    let port = listener.local_addr().unwrap().port();
292
    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
293
    (format!("http://127.0.0.1:{port}"), state)
294
}
295
296
fn base64url(s: &str) -> Vec<u8> {
297
    let table = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
298
    let mut bits = 0u32;
299
    let mut n = 0;
300
    let mut out = Vec::new();
301
    for c in s.bytes() {
302
        let v = table.iter().position(|t| *t == c).unwrap() as u32;
303
        bits = (bits << 6) | v;
304
        n += 6;
305
        if n >= 8 {
306
            n -= 8;
307
            out.push((bits >> n) as u8);
308
            bits &= (1 << n) - 1;
309
        }
310
    }
311
    out
312
}
313
314
const PROVIDER: &str = "//iam.googleapis.com/projects/1/locations/global/workloadIdentityPools/oxygenhub/providers/oxygenhub-io";
315
316
fn gcs(owner: &str, bucket: &str) -> Setting {
317
    let mut s = Setting::off(owner, "user");
318
    s.every = "hourly".into();
319
    s.destination = "gcs".into();
320
    s.target = bucket.into();
321
    s.provider = PROVIDER.into();
322
    s
323
}
324
325
#[tokio::test(flavor = "multi_thread")]
326
async fn a_gcs_backup_needs_no_key_and_restores_the_same_repository() {
327
    let (google, seen) = fake_google().await;
328
    let store = tempfile::tempdir().unwrap();
329
    let h = hub_with(store.path(), Some(&google)).await;
330
    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
331
    db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();
332
    create(&h.app, "andy", "demo", hub_api::Visibility::Public).await;
333
    let work = tempfile::tempdir().unwrap();
334
    push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n")]);
335
    // A store holds empty files too (as oxygenhub.io's did, 2026-10-14):
336
    // they go up with a length of 0.
337
    std::fs::write(h.app.repos.dir(&RepoName::parse("andy/demo").unwrap()).join(".oxy/empty-marker"), b"").unwrap();
338
339
    backups::save(&h.app.db, &gcs("andy", "andy-backup")).await.unwrap();
340
    let first = backups::run(&h.app, "andy", "now").await.unwrap();
341
    assert_eq!((first.status.as_str(), first.repos), ("ok", 1), "{}", first.detail);
342
    {
343
        let g = seen.lock().unwrap();
344
        assert!(g.objects.keys().any(|(b, n)| b == "andy-backup" && n == &format!("runs/{}/DONE", first.at)));
345
        assert!(g.objects.keys().all(|(b, _)| b == "andy-backup"));
346
        let (sub, aud) = g.exchanged.first().unwrap();
347
        assert_eq!(
348
            (sub.as_str(), aud.as_str()),
349
            ("backup:andy", format!("https:{PROVIDER}").as_str()),
350
            "the hub signs in as the owner, for the provider"
351
        );
352
    }
353
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
354
    let second = backups::run(&h.app, "andy", "now").await.unwrap();
355
    assert_eq!((second.status.as_str(), second.sent), ("ok", 0), "{}", second.detail);
356
    // The probe writes, reads and removes; the bucket is as it was.
357
    let before = seen.lock().unwrap().objects.len();
358
    let dest = backups::destination(&h.app, &gcs("andy", "andy-backup")).unwrap();
359
    tokio::task::spawn_blocking(move || backups::probe(dest.as_ref())).await.unwrap().unwrap();
360
    assert_eq!(seen.lock().unwrap().objects.len(), before);
361
362
    // Restored from the bucket into a fresh hub: the same repository.
363
    let empty = tempfile::tempdir().unwrap();
364
    let fresh = hub_with(empty.path(), Some(&google)).await;
365
    let dest: Arc<dyn Destination> = backups::destination(&fresh.app, &gcs("andy", "andy-backup")).unwrap().into();
366
    assert_eq!(backups::restore_into_hub(&fresh.app, dest, None).await.unwrap(), vec!["andy/demo (repos)"]);
367
    let name = RepoName::parse("andy/demo").unwrap();
368
    assert_eq!(files(&fresh.app.repos.dir(&name)), files(&h.app.repos.dir(&name)), "byte for byte");
369
370
    // Bob cannot write Andy's bucket: Google refuses his token, and the
371
    // run says what to grant.
372
    backups::save(&h.app.db, &gcs("bob", "andy-backup")).await.unwrap();
373
    let refused = backups::run(&h.app, "bob", "now").await.unwrap();
374
    assert_eq!(refused.status, "failed");
375
    assert!(refused.detail.contains("roles/storage.objectAdmin") && refused.detail.contains("backup:bob"), "{}", refused.detail);
376
}
377
378
/// docs/backups.md §7 (BP1): a restore brings back the records as well as
379
/// the stores: issues with their comments, labels, milestone and
380
/// assignees, proposals with their reviews and events, releases with their
381
/// asset files, and webhooks without their secrets, every id mapped and
382
/// every account found again by name; what this hub lacks is said.
383
#[tokio::test(flavor = "multi_thread")]
384
async fn a_restore_brings_back_issues_proposals_and_releases() {
385
    let store = tempfile::tempdir().unwrap();
386
    let h = hub(store.path()).await;
387
    let andy = db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
388
    let bob = db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();
389
    create(&h.app, "andy", "demo", hub_api::Visibility::Public).await;
390
    let work = tempfile::tempdir().unwrap();
391
    push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n")]);
392
    let row = db::repos::get(&h.app.db, "andy", "demo").await.unwrap().unwrap();
393
    db::repos::set_diffusion(&h.app.db, row.id, "green").await.unwrap();
394
    // Ids on the first hub start further on than on the fresh one.
395
    for n in 0..3 {
396
        db::issues::upsert_label(&h.app.db, row.id, &format!("spare{n}"), "#000000", "").await.unwrap();
397
    }
398
    let bug = db::issues::upsert_label(&h.app.db, row.id, "bug", "#d73a4a", "Something is wrong").await.unwrap();
399
    let m = db::issues::upsert_milestone(&h.app.db, row.id, "1.0", "the first", None).await.unwrap();
400
    let issue = db::issues::create(
401
        &h.app.db,
402
        db::issues::NewIssue { repo_id: row.id, title: "Sprites flicker", body: "on level two", author_id: Some(andy.id), author: "andy" },
403
    )
404
    .await
405
    .unwrap();
406
    db::issues::add_comment(&h.app.db, issue.id, Some(bob.id), "bob", "seen it too").await.unwrap();
407
    db::issues::set_issue_label(&h.app.db, issue.id, bug.id, true).await.unwrap();
408
    db::issues::set_milestone(&h.app.db, issue.id, Some(m.id)).await.unwrap();
409
    db::issues::set_assignee(&h.app.db, issue.id, andy.id, true).await.unwrap();
410
    db::issues::set_assignee(&h.app.db, issue.id, bob.id, true).await.unwrap();
411
    let p = db::proposals::create(
412
        &h.app.db,
413
        db::proposals::NewProposal {
414
            repo_id: row.id,
415
            title: "Double-buffer the sprites",
416
            body: "fixes the flicker",
417
            author_id: Some(bob.id),
418
            author: "bob",
419
            draft: false,
420
            target: "main",
421
            source_kind: "bookmark",
422
            source_repo_id: Some(row.id),
423
            source_bookmark: "fix",
424
            source_changes: "",
425
        },
426
    )
427
    .await
428
    .unwrap();
429
    db::proposals::add_event(&h.app.db, p.id, "bob", "opened", "").await.unwrap();
430
    db::proposals::add_review(&h.app.db, p.id, andy.id, "andy", "approve", "nice").await.unwrap();
431
    let release = db::releases::create(&h.app.db, row.id, "v1.0", "One", "notes", false, Some((andy.id, "andy"))).await.unwrap();
432
    let asset = b"a game image".to_vec();
433
    let hash = blake3::hash(&asset).to_hex().to_string();
434
    let path = h.app.config.data.join("assets").join(&hash[..2]).join(&hash);
435
    std::fs::create_dir_all(path.parent().unwrap()).unwrap();
436
    std::fs::write(&path, &asset).unwrap();
437
    db::releases::add_asset(&h.app.db, release.id, "game.prg", asset.len() as i64, &hash, "application/octet-stream").await.unwrap();
438
    db::hooks::create_hook(&h.app.db, Some(row.id), None, "https://example.com/hook", "s3cret-value", "push").await.unwrap();
439
440
    let mut s = Setting::off("andy", "user");
441
    s.every = "hourly".into();
442
    s.target = store.path().display().to_string();
443
    backups::save(&h.app.db, &s).await.unwrap();
444
    let run = backups::run(&h.app, "andy", "now").await.unwrap();
445
    assert_eq!(run.status, "ok", "{}", run.detail);
446
    let dest = Directory { root: store.path().join("andy") };
447
    let text = String::from_utf8_lossy(&dest.get(&format!("runs/{}/records/demo.json", run.at)).unwrap()).into_owned();
448
    assert!(!text.contains("s3cret-value"), "never a secret");
449
    assert!(text.contains("\"_names\"") && text.contains("\"bob\""), "accounts by name");
450
451
    // A fresh hub that knows andy but not bob.
452
    let empty = tempfile::tempdir().unwrap();
453
    let fresh = hub(empty.path()).await;
454
    let andy2 = db::accounts::create_account(&fresh.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
455
    let dest: Arc<dyn Destination> = Arc::new(Directory { root: store.path().join("andy") });
456
    let done = backups::restore_into_hub(&fresh.app, dest, None).await.unwrap();
457
    assert_eq!(done[0], "andy/demo (repos)");
458
    let notes = done[1..].join("\n");
459
    assert!(notes.contains("webhook https://example.com/hook: enter its secret again"), "{notes}");
460
    assert!(notes.contains("issue_assignees: 1 row(s) left out"), "bob is not on this hub: {notes}");
461
    let r2 = db::repos::get(&fresh.app.db, "andy", "demo").await.unwrap().unwrap();
462
    assert_eq!(r2.diffusion, "green", "settings come back");
463
    // The issue, its comment, label, milestone and the assignee this hub has.
464
    let issues: Vec<(i64, i64, String, Option<i64>)> = sqlx::query_as("SELECT id, number, title, author_id FROM issues WHERE repo_id = ?")
465
        .bind(r2.id)
466
        .fetch_all(&fresh.app.db)
467
        .await
468
        .unwrap();
469
    assert_eq!(issues.len(), 1);
470
    let (iid, number, title, author) = issues[0].clone();
471
    assert_eq!((number, title.as_str(), author), (issue.number, "Sprites flicker", Some(andy2.id)));
472
    let comments = db::issues::comments(&fresh.app.db, iid).await.unwrap();
473
    assert_eq!(comments.len(), 1);
474
    assert_eq!((comments[0].author.as_str(), comments[0].body.as_str()), ("bob", "seen it too"));
475
    let labels: Vec<String> =
476
        sqlx::query_scalar("SELECT l.name FROM issue_labels il JOIN labels l ON l.id = il.label_id WHERE il.issue_id = ?")
477
            .bind(iid)
478
            .fetch_all(&fresh.app.db)
479
            .await
480
            .unwrap();
481
    assert_eq!(labels, vec!["bug"], "the label, its id mapped");
482
    let milestone: Option<String> =
483
        sqlx::query_scalar("SELECT m.title FROM issues i JOIN milestones m ON m.id = i.milestone_id WHERE i.id = ?")
484
            .bind(iid)
485
            .fetch_one(&fresh.app.db)
486
            .await
487
            .unwrap();
488
    assert_eq!(milestone.as_deref(), Some("1.0"));
489
    assert_eq!(db::issues::assignees(&fresh.app.db, iid).await.unwrap(), vec!["andy"]);
490
    // The proposal, its review and its timeline.
491
    let p2 = db::proposals::get(&fresh.app.db, r2.id, p.number).await.unwrap().expect("the proposal");
492
    assert_eq!((p2.title.as_str(), p2.author.as_str(), p2.author_id), ("Double-buffer the sprites", "bob", None));
493
    let reviews: Vec<(String, String)> = sqlx::query_as("SELECT author, verdict FROM proposal_reviews WHERE proposal_id = ?")
494
        .bind(p2.id)
495
        .fetch_all(&fresh.app.db)
496
        .await
497
        .unwrap();
498
    assert_eq!(reviews, vec![("andy".to_string(), "approve".to_string())]);
499
    assert!(db::proposals::events(&fresh.app.db, p2.id).await.unwrap().iter().any(|e| e.event == "opened"));
500
    // The release and its asset's bytes.
501
    let rel = db::releases::get(&fresh.app.db, r2.id, "v1.0").await.unwrap().expect("the release");
502
    let assets = db::releases::assets(&fresh.app.db, rel.id).await.unwrap();
503
    assert_eq!(assets.len(), 1);
504
    assert_eq!(std::fs::read(fresh.app.config.data.join("assets").join(&hash[..2]).join(&hash)).unwrap(), asset);
505
    // Search finds what came back.
506
    let hits = db::notify::search(&fresh.app.db, "flicker", Some(r2.id), 10).await.unwrap();
507
    assert!(hits.iter().any(|h| h.kind == "issue" && h.target_id == iid), "{:?}", hits.iter().map(|h| &h.title).collect::<Vec<_>>());
508
    // New issues number on from where the old hub was.
509
    let next = db::issues::create(
510
        &fresh.app.db,
511
        db::issues::NewIssue { repo_id: r2.id, title: "After", body: "", author_id: Some(andy2.id), author: "andy" },
512
    )
513
    .await
514
    .unwrap();
515
    assert!(next.number > p.number, "{} after {}", next.number, p.number);
516
}
517
518
/// docs/backups.md §5 (BP3): a repository left out is in no run; one with
519
/// its own schedule makes its owner's backup run at least that often, and
520
/// alone when its owner's is off; a run by hand takes all but those left
521
/// out.
522
#[tokio::test(flavor = "multi_thread")]
523
async fn repositories_choose_their_own_backup() {
524
    let store = tempfile::tempdir().unwrap();
525
    let h = hub(store.path()).await;
526
    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
527
    for n in ["game", "notes", "secret"] {
528
        create(&h.app, "andy", n, hub_api::Visibility::Public).await;
529
        let work = tempfile::tempdir().unwrap();
530
        push(&format!("{}/andy/{n}", h.base), work.path(), &[("README.md", n)]);
531
    }
532
    let id = |n: &'static str| {
533
        let db = h.app.db.clone();
534
        async move { db::repos::get(&db, "andy", n).await.unwrap().unwrap().id }
535
    };
536
    db::repos::set_backup(&h.app.db, id("secret").await, "off").await.unwrap();
537
    let mut s = Setting::off("andy", "user");
538
    s.every = "daily".into();
539
    s.target = store.path().display().to_string();
540
    backups::save(&h.app.db, &s).await.unwrap();
541
    let names = |at: String| {
542
        let dest = Directory { root: store.path().join("andy") };
543
        let mut v = dest.list(&format!("runs/{at}/records")).unwrap();
544
        v.sort();
545
        v
546
    };
547
    let first = backups::run(&h.app, "andy", "schedule").await.unwrap();
548
    assert_eq!((first.status.as_str(), first.repos), ("ok", 2), "{}", first.detail);
549
    assert_eq!(names(first.at.clone()), vec!["game.json", "notes.json"], "the one left out is not in it");
550
    // Daily, and it just ran: not due, until a repository asks for hourly.
551
    assert!(backups::due(&h.app.db).await.unwrap().is_empty());
552
    db::repos::set_backup(&h.app.db, id("game").await, "hourly").await.unwrap();
553
    sqlx::query("UPDATE backup_runs SET started_ms = started_ms - 2 * 3600 * 1000").execute(&h.app.db).await.unwrap();
554
    assert_eq!(backups::due(&h.app.db).await.unwrap().len(), 1, "hourly for the game");
555
    // The owner's schedule off: only the game, which asks for its own.
556
    s.every = "off".into();
557
    backups::save(&h.app.db, &s).await.unwrap();
558
    assert_eq!(backups::due(&h.app.db).await.unwrap().len(), 1);
559
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
560
    let second = backups::run(&h.app, "andy", "schedule").await.unwrap();
561
    assert_eq!(names(second.at.clone()), vec!["game.json"]);
562
    // By hand: everything but the one left out.
563
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
564
    let third = backups::run(&h.app, "andy", "now").await.unwrap();
565
    assert_eq!(names(third.at.clone()), vec!["game.json", "notes.json"]);
566
    // Admin › Backups counts what the next scheduled run covers.
567
    let (owners, uncovered) = backups::overview(&h.app.db).await.unwrap();
568
    assert_eq!((owners[0].repos, uncovered.len()), (1, 0));
569
}
570
571
/// AWS STS and S3, as much as backups use (path style, under `/s3`): an
572
/// AssumeRoleWithWebIdentity that admits `backup:` subjects with audience
573
/// `sts.amazonaws.com`, and buckets (`OWNER-backup`) that check every
574
/// request's Signature Version 4 and admit only their owner's session.
575
async fn fake_aws() -> (String, Shared2) {
576
    use axum::extract::{Request, State};
577
    use axum::http::StatusCode;
578
    use axum::response::IntoResponse;
579
    let state: Shared2 = Default::default();
580
    async fn handle(State(g): State<Shared2>, req: Request) -> axum::response::Response {
581
        let (parts, body) = req.into_parts();
582
        let bytes = http_body_util::BodyExt::collect(body).await.unwrap().to_bytes();
583
        let path = parts.uri.path().to_string();
584
        let header = |h: &str| parts.headers.get(h).and_then(|v| v.to_str().ok()).unwrap_or("").to_string();
585
        if path == "/" && parts.method == "POST" {
586
            let form: Vec<(String, String)> = String::from_utf8_lossy(&bytes)
587
                .split('&')
588
                .filter_map(|kv| kv.split_once('=').map(|(k, v)| (k.to_string(), decode(&v.replace('+', " ")))))
589
                .collect();
590
            let field = |k: &str| form.iter().find(|(f, _)| f == k).map(|(_, v)| v.clone()).unwrap_or_default();
591
            let jwt = field("WebIdentityToken");
592
            let claims: serde_json::Value = serde_json::from_slice(&base64url(jwt.split('.').nth(1).unwrap())).unwrap();
593
            let (sub, aud) = (claims["sub"].as_str().unwrap().to_string(), claims["aud"].as_str().unwrap().to_string());
594
            g.lock().unwrap().exchanged.push((sub.clone(), aud.clone()));
595
            if field("Action") != "AssumeRoleWithWebIdentity" || !sub.starts_with("backup:") || aud != "sts.amazonaws.com" {
596
                return (StatusCode::FORBIDDEN, "<ErrorResponse><Error><Code>AccessDenied</Code><Message>Not authorized to perform sts:AssumeRoleWithWebIdentity</Message></Error></ErrorResponse>").into_response();
597
            }
598
            let owner = sub.trim_start_matches("backup:");
599
            return format!(
600
                "<AssumeRoleWithWebIdentityResponse><AssumeRoleWithWebIdentityResult><Credentials>\
601
                 <AccessKeyId>ASIA{owner}</AccessKeyId><SecretAccessKey>secret/{owner}</SecretAccessKey>\
602
                 <SessionToken>session-{owner}</SessionToken><Expiration>2030-01-01T00:00:00Z</Expiration>\
603
                 </Credentials></AssumeRoleWithWebIdentityResult></AssumeRoleWithWebIdentityResponse>"
604
            )
605
            .into_response();
606
        }
607
        // /s3/{bucket}/{key…}
608
        let rest = path.strip_prefix("/s3/").unwrap_or("");
609
        let (bucket, key) = rest.split_once('/').unwrap_or((rest, ""));
610
        let (bucket, key) = (bucket.to_string(), decode(key));
611
        let owner = bucket.trim_end_matches("-backup").to_string();
612
        // Check the signature as AWS would, with the session's secret.
613
        let session = header("x-amz-security-token");
614
        let who = session.trim_start_matches("session-").to_string();
615
        let query: Vec<(String, String)> = parts
616
            .uri
617
            .query()
618
            .unwrap_or("")
619
            .split('&')
620
            .filter(|kv| !kv.is_empty())
621
            .map(|kv| kv.split_once('=').map(|(k, v)| (decode(k), decode(v))).unwrap_or((decode(kv), String::new())))
622
            .collect();
623
        let signed = vec![
624
            ("host".to_string(), header("host")),
625
            ("x-amz-content-sha256".to_string(), header("x-amz-content-sha256")),
626
            ("x-amz-date".to_string(), header("x-amz-date")),
627
            ("x-amz-security-token".to_string(), session.clone()),
628
        ];
629
        let id = format!("ASIA{who}");
630
        let secret = format!("secret/{who}");
631
        let want = oxygenhub::backups::sigv4(
632
            parts.method.as_str(),
633
            &path,
634
            &query,
635
            &signed,
636
            &header("x-amz-content-sha256"),
637
            (&id, &secret),
638
            "eu-west-2",
639
            "s3",
640
            &header("x-amz-date"),
641
        );
642
        use sha2::Digest;
643
        let body_hash = hex::encode(sha2::Sha256::digest(&bytes));
644
        if header("authorization") != want || header("x-amz-content-sha256") != body_hash {
645
            return (
646
                StatusCode::FORBIDDEN,
647
                "<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>",
648
            )
649
                .into_response();
650
        }
651
        if who != owner {
652
            return (StatusCode::FORBIDDEN, "<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>").into_response();
653
        }
654
        let mut g = g.lock().unwrap();
655
        match (parts.method.as_str(), key.is_empty()) {
656
            ("PUT", false) if !parts.headers.contains_key("content-length") => {
657
                (StatusCode::LENGTH_REQUIRED, "<Error><Code>MissingContentLength</Code></Error>").into_response()
658
            }
659
            ("PUT", false) => {
660
                g.objects.insert((bucket, key), bytes.to_vec());
661
                StatusCode::OK.into_response()
662
            }
663
            ("GET", false) | ("HEAD", false) => match g.objects.get(&(bucket, key)) {
664
                Some(b) => b.clone().into_response(),
665
                None => (StatusCode::NOT_FOUND, "<Error><Code>NoSuchKey</Code></Error>").into_response(),
666
            },
667
            ("DELETE", false) => {
668
                g.objects.remove(&(bucket, key));
669
                StatusCode::NO_CONTENT.into_response()
670
            }
671
            ("GET", true) => {
672
                let prefix = query.iter().find(|(k, _)| k == "prefix").map(|(_, v)| v.clone()).unwrap_or_default();
673
                let delimited = query.iter().any(|(k, _)| k == "delimiter");
674
                let (mut keys, mut prefixes) = (String::new(), std::collections::BTreeSet::new());
675
                for (b, n) in g.objects.keys() {
676
                    if *b != bucket || !n.starts_with(&prefix) {
677
                        continue;
678
                    }
679
                    match n[prefix.len()..].find('/') {
680
                        Some(i) if delimited => {
681
                            prefixes.insert(format!("{prefix}{}", &n[prefix.len()..prefix.len() + i + 1]));
682
                        }
683
                        _ => keys.push_str(&format!("<Contents><Key>{n}</Key></Contents>")),
684
                    }
685
                }
686
                let common: String = prefixes.iter().map(|p| format!("<CommonPrefixes><Prefix>{p}</Prefix></CommonPrefixes>")).collect();
687
                format!("<ListBucketResult><Prefix>{prefix}</Prefix><IsTruncated>false</IsTruncated>{keys}{common}</ListBucketResult>")
688
                    .into_response()
689
            }
690
            _ => StatusCode::METHOD_NOT_ALLOWED.into_response(),
691
        }
692
    }
693
    let app = axum::Router::new().fallback(handle).with_state(state.clone());
694
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
695
    let port = listener.local_addr().unwrap().port();
696
    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
697
    (format!("http://127.0.0.1:{port}"), state)
698
}
699
700
async fn hub_aws(backups: &Path, aws: &str) -> Hub {
701
    hermetic();
702
    let dir = tempfile::tempdir().unwrap();
703
    let mut config = Config::load(dir.path()).unwrap();
704
    config.auth.push_tokens = vec!["site".into()];
705
    config.backups.directories = vec![backups.to_path_buf()];
706
    config.backups.aws_sts_url = aws.to_string();
707
    config.backups.s3_url = format!("{aws}/s3/{{bucket}}");
708
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
709
    let port = listener.local_addr().unwrap().port();
710
    config.server.base_url = format!("http://127.0.0.1:{port}");
711
    let app = oxygenhub::build(config).await.unwrap();
712
    let router = oxygenhub::router(app.clone());
713
    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
714
    Hub { app, base: format!("http://127.0.0.1:{port}"), _dir: dir }
715
}
716
717
const ROLE: &str = "arn:aws:iam::123456789012:role/oxygenhub-backup";
718
719
fn s3(owner: &str, bucket: &str) -> Setting {
720
    let mut s = Setting::off(owner, "user");
721
    s.every = "hourly".into();
722
    s.destination = "s3".into();
723
    s.target = bucket.into();
724
    s.provider = ROLE.into();
725
    s.region = "eu-west-2".into();
726
    s.prefix = "oxygen hub".into();
727
    s
728
}
729
730
/// docs/backups.md §3 (BP4): an S3 bucket, keylessly: the hub's own token
731
/// for AWS STS, then signed requests; a second run sends nothing; the
732
/// probe leaves the bucket as it was; a fresh hub restores the same
733
/// repository from it; another owner's session is refused, saying what the
734
/// role needs.
735
#[tokio::test(flavor = "multi_thread")]
736
async fn an_s3_backup_needs_no_key_and_restores_the_same_repository() {
737
    let (aws, seen) = fake_aws().await;
738
    let store = tempfile::tempdir().unwrap();
739
    let h = hub_aws(store.path(), &aws).await;
740
    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
741
    db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();
742
    create(&h.app, "andy", "demo", hub_api::Visibility::Public).await;
743
    let work = tempfile::tempdir().unwrap();
744
    push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n"), ("src/a b+c.rs", "fn main() {}\n")]);
745
    std::fs::write(h.app.repos.dir(&RepoName::parse("andy/demo").unwrap()).join(".oxy/empty-marker"), b"").unwrap();
746
747
    backups::save(&h.app.db, &s3("andy", "andy-backup")).await.unwrap();
748
    let first = backups::run(&h.app, "andy", "now").await.unwrap();
749
    assert_eq!((first.status.as_str(), first.repos), ("ok", 1), "{}", first.detail);
750
    {
751
        let g = seen.lock().unwrap();
752
        assert!(
753
            g.objects.keys().any(|(b, n)| b == "andy-backup" && n == &format!("oxygen hub/runs/{}/DONE", first.at)),
754
            "under the prefix"
755
        );
756
        assert_eq!(g.exchanged.first().map(|(s, a)| (s.as_str(), a.as_str())), Some(("backup:andy", "sts.amazonaws.com")));
757
    }
758
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
759
    let second = backups::run(&h.app, "andy", "now").await.unwrap();
760
    assert_eq!((second.status.as_str(), second.sent), ("ok", 0), "{}", second.detail);
761
    let before = seen.lock().unwrap().objects.len();
762
    let dest = backups::destination(&h.app, &s3("andy", "andy-backup")).unwrap();
763
    tokio::task::spawn_blocking(move || backups::probe(dest.as_ref())).await.unwrap().unwrap();
764
    assert_eq!(seen.lock().unwrap().objects.len(), before, "the probe cleans up");
765
766
    let empty = tempfile::tempdir().unwrap();
767
    let fresh = hub_aws(empty.path(), &aws).await;
768
    let dest: Arc<dyn Destination> = backups::destination(&fresh.app, &s3("andy", "andy-backup")).unwrap().into();
769
    assert_eq!(backups::restore_into_hub(&fresh.app, dest, None).await.unwrap(), vec!["andy/demo (repos)"]);
770
    let name = RepoName::parse("andy/demo").unwrap();
771
    assert_eq!(files(&fresh.app.repos.dir(&name)), files(&h.app.repos.dir(&name)), "byte for byte");
772
773
    backups::save(&h.app.db, &s3("bob", "andy-backup")).await.unwrap();
774
    let refused = backups::run(&h.app, "bob", "now").await.unwrap();
775
    assert_eq!(refused.status, "failed");
776
    assert!(refused.detail.contains("s3:PutObject") && refused.detail.contains(ROLE), "{}", refused.detail);
777
}
778
779
/// One API call with a bearer token: the status and the JSON answer.
780
async fn api(
781
    app: &Shared,
782
    method: &str,
783
    path: &str,
784
    token: &str,
785
    body: Option<serde_json::Value>,
786
) -> (axum::http::StatusCode, serde_json::Value) {
787
    use axum::body::Body;
788
    use http_body_util::BodyExt;
789
    use tower::ServiceExt;
790
    let mut req =
791
        axum::http::Request::builder().method(method).uri(format!("/api/v1{path}")).header("authorization", format!("Bearer {token}"));
792
    let body = match body {
793
        Some(b) => {
794
            req = req.header("content-type", "application/json");
795
            Body::from(b.to_string())
796
        }
797
        None => Body::empty(),
798
    };
799
    let res = oxygenhub::router(app.clone()).oneshot(req.body(body).unwrap()).await.unwrap();
800
    let st = res.status();
801
    let bytes = res.into_body().collect().await.unwrap().to_bytes();
802
    (st, serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null))
803
}
804
805
/// oxygenhub#179: the API does what the Backups pages do, for the caller's
806
/// own repositories and an organisation it owns, with repo:admin: the
807
/// setting (checked as the page checks it), TEST, back up now, the runs.
808
#[tokio::test(flavor = "multi_thread")]
809
async fn backups_are_set_up_and_run_through_the_api() {
810
    let store = tempfile::tempdir().unwrap();
811
    let h = hub(store.path()).await;
812
    let andy = db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
813
    let bob = db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();
814
    db::accounts::create_org(&h.app.db, "club", "Club", andy.id).await.unwrap();
815
    create(&h.app, "andy", "demo", hub_api::Visibility::Public).await;
816
    create(&h.app, "club", "game", hub_api::Visibility::Private).await;
817
    let token = |id: i64, scopes: &'static str| {
818
        let db = h.app.db.clone();
819
        async move {
820
            let (t, hash) = oxygenhub::auth::new_token();
821
            db::accounts::create_token(&db, id, "t", &hash, scopes, "", None).await.unwrap();
822
            t
823
        }
824
    };
825
    let (andy_t, bob_t, weak) = (token(andy.id, "repo:admin").await, token(bob.id, "repo:admin").await, token(andy.id, "repo:read").await);
826
    let dir = store.path().display().to_string();
827
828
    let (st, empty) = api(&h.app, "GET", "/user/backups", &andy_t, None).await;
829
    assert_eq!(st, 200);
830
    assert!(empty["setting"].is_null() && empty["runs"].as_array().unwrap().is_empty(), "{empty}");
831
    let (st, _) = api(&h.app, "POST", "/user/backups/run", &andy_t, None).await;
832
    assert_eq!(st, 409, "nothing saved yet");
833
    let (st, _) =
834
        api(&h.app, "PUT", "/user/backups", &weak, Some(serde_json::json!({ "every": "hourly", "destination": "dir", "dir": dir }))).await;
835
    assert_eq!(st, 403, "repo:admin only");
836
    let (st, bad) = api(
837
        &h.app,
838
        "PUT",
839
        "/user/backups",
840
        &andy_t,
841
        Some(serde_json::json!({ "every": "hourly", "destination": "gcs", "bucket": "andy-backup", "provider": "nope" })),
842
    )
843
    .await;
844
    assert_eq!(st, 400, "checked as the page checks it: {bad}");
845
    let (st, saved) = api(
846
        &h.app,
847
        "PUT",
848
        "/user/backups",
849
        &andy_t,
850
        Some(serde_json::json!({ "every": "hourly", "destination": "dir", "dir": dir, "keep_days": 30 })),
851
    )
852
    .await;
853
    assert_eq!(st, 200, "{saved}");
854
    assert_eq!((saved["setting"]["every"].as_str(), saved["place"].as_str()), (Some("hourly"), Some(format!("{dir}/andy").as_str())));
855
    let (st, probe) = api(&h.app, "POST", "/user/backups/test", &andy_t, None).await;
856
    assert_eq!((st.as_u16(), probe["ok"].as_bool()), (200, Some(true)), "{probe}");
857
    let (st, started) = api(&h.app, "POST", "/user/backups/run", &andy_t, None).await;
858
    assert_eq!(st, 202, "{started}");
859
    let mut done = serde_json::Value::Null;
860
    for _ in 0..200 {
861
        let (_, runs) = api(&h.app, "GET", "/user/backups/runs", &andy_t, None).await;
862
        if runs[0]["status"].as_str().is_some_and(|s| s != "running") {
863
            done = runs[0].clone();
864
            break;
865
        }
866
        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
867
    }
868
    assert_eq!((done["status"].as_str(), done["repos"].as_i64(), done["trigger"].as_str()), (Some("ok"), Some(1), Some("now")), "{done}");
869
    // A whole run after a failed one says it works again; the day's
870
    // summary tells the owner what ran (#183).
871
    sqlx::query("INSERT INTO backup_runs (owner, started_ms, finished_ms, status, at, trigger, detail) VALUES ('andy', 1, 2, 'failed', 'x', 'schedule', 'GCS refused')")
872
        .execute(&h.app.db)
873
        .await
874
        .unwrap();
875
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
876
    let again = backups::run(&h.app, "andy", "now").await.unwrap();
877
    assert_eq!(again.status, "ok");
878
    let told = |db: sqlx::SqlitePool| async move { db::notify::inbox(&db, andy.id, 20).await.unwrap() };
879
    assert!(told(h.app.db.clone()).await.iter().any(|n| n.kind == "backup.recovered" && n.title.contains("works again")));
880
    assert_eq!(backups::daily_summary(&h.app, db::now_ms(), true).await.unwrap(), 1);
881
    let summary = told(h.app.db.clone()).await.into_iter().find(|n| n.kind == "backup.summary").expect("a summary");
882
    assert!(summary.title.contains("Backups of andy today") && summary.title.contains("runs"), "{}", summary.title);
883
884
    // An organisation's, by its owner; another account sees no such thing.
885
    let (st, _) =
886
        api(&h.app, "PUT", "/orgs/club/backups", &bob_t, Some(serde_json::json!({ "every": "hourly", "destination": "dir", "dir": dir })))
887
            .await;
888
    assert_eq!(st, 404);
889
    let (st, org) = api(
890
        &h.app,
891
        "PUT",
892
        "/orgs/club/backups",
893
        &andy_t,
894
        Some(serde_json::json!({ "every": "daily", "destination": "dir", "dir": dir, "keep_days": 90 })),
895
    )
896
    .await;
897
    assert_eq!(st, 200, "{org}");
898
    assert_eq!((org["setting"]["kind"].as_str(), org["setting"]["keep_days"].as_i64()), (Some("org"), Some(90)));
899
    let (st, _) = api(&h.app, "GET", "/orgs/club/backups", &bob_t, None).await;
900
    assert_eq!(st, 404);
901
}