OXYGENHUB
◆ v0.6.1 oxygenhub / crates / hub / tests / backups.rs
45.5 KiB · Rust · last changed A new hub gives administrator only to whoever brings its setup code 19 days ago source blame history raw
1
//! Backups (docs/backups.md, oxygenhub#169), end to end: a real `oxy` push,
2
//! a backup to a directory, a second run that sends nothing new, and a
3
//! restore into a fresh hub that gives back the same repository, byte for
4
//! byte; and an owner's backup never holding another's repositories.
5
6
use oxy_core::repo::Repo;
7
use oxygenhub::backups::{self, Destination, Directory, Setting};
8
use oxygenhub::config::Config;
9
use oxygenhub::repos::RepoName;
10
use oxygenhub::{db, Shared};
11
use std::path::Path;
12
use std::sync::Arc;
13
14
fn hermetic() {
15
    static ONCE: std::sync::Once = std::sync::Once::new();
16
    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();
17
    ONCE.call_once(|| {
18
        let d = tempfile::tempdir().unwrap();
19
        std::env::set_var("OXY_HOME", d.path());
20
        DIR.set(d).ok();
21
    });
22
}
23
24
struct Hub {
25
    app: Shared,
26
    base: String,
27
    _dir: tempfile::TempDir,
28
}
29
30
async fn hub(backups: &Path) -> Hub {
31
    hub_with(backups, None).await
32
}
33
34
async fn hub_with(backups: &Path, google: Option<&str>) -> Hub {
35
    hermetic();
36
    let dir = tempfile::tempdir().unwrap();
37
    let mut config = Config::load(dir.path()).unwrap();
38
    // The first account's setup code (oxygenhub#245).
39
    config.registration.setup_code = "test-setup".into();
40
    config.auth.push_tokens = vec!["site".into()];
41
    config.backups.directories = vec![backups.to_path_buf()];
42
    if let Some(g) = google {
43
        config.backups.sts_url = g.to_string();
44
        config.backups.storage_url = g.to_string();
45
    }
46
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
47
    let port = listener.local_addr().unwrap().port();
48
    config.server.base_url = format!("http://127.0.0.1:{port}");
49
    let app = oxygenhub::build(config).await.unwrap();
50
    let router = oxygenhub::router(app.clone());
51
    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
52
    Hub { app, base: format!("http://127.0.0.1:{port}"), _dir: dir }
53
}
54
55
fn write(root: &Path, rel: &str, content: &str) {
56
    let p = root.join(rel);
57
    std::fs::create_dir_all(p.parent().unwrap()).unwrap();
58
    std::fs::write(p, content).unwrap();
59
}
60
61
async fn create(app: &Shared, owner: &str, name: &str, visibility: hub_api::Visibility) {
62
    app.repos.create(&RepoName::parse(&format!("{owner}/{name}")).unwrap(), None).unwrap();
63
    db::repos::create(&app.db, owner, name, "a demo", visibility).await.unwrap();
64
}
65
66
fn push(url: &str, work: &Path, files: &[(&str, &str)]) {
67
    let mut c = Repo::init(work).unwrap();
68
    c.config.set("remote.origin.url", url).unwrap();
69
    c.config.set("remote.origin.token", "site").unwrap();
70
    c.save_config().unwrap();
71
    for (f, t) in files {
72
        write(work, f, t);
73
    }
74
    c.snapshot("t").unwrap();
75
    c.commit("Some work", &[]).unwrap();
76
    assert!(c.push("origin", false).unwrap().iter().all(|r| r.ok));
77
}
78
79
/// Every file under `dir`, with its bytes.
80
fn files(dir: &Path) -> Vec<(String, Vec<u8>)> {
81
    let mut out = Vec::new();
82
    let mut stack = vec![dir.to_path_buf()];
83
    while let Some(d) = stack.pop() {
84
        for e in std::fs::read_dir(&d).unwrap().flatten() {
85
            let p = e.path();
86
            if p.is_dir() {
87
                stack.push(p);
88
            } else {
89
                out.push((p.strip_prefix(dir).unwrap().to_string_lossy().into_owned(), std::fs::read(&p).unwrap()));
90
            }
91
        }
92
    }
93
    out.sort();
94
    out
95
}
96
97
#[tokio::test(flavor = "multi_thread")]
98
async fn a_backup_restores_the_same_repository_and_sends_only_what_changed() {
99
    let store = tempfile::tempdir().unwrap();
100
    let h = hub(store.path()).await;
101
    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
102
    db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();
103
    create(&h.app, "andy", "demo", hub_api::Visibility::Private).await;
104
    create(&h.app, "bob", "other", hub_api::Visibility::Public).await;
105
    let work = tempfile::tempdir().unwrap();
106
    push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n"), ("src/main.rs", "fn main() {}\n")]);
107
    let other = tempfile::tempdir().unwrap();
108
    push(&format!("{}/bob/other", h.base), other.path(), &[("x.txt", "bob's\n")]);
109
110
    // Andy backs up to the directory the hub allows.
111
    let target = store.path().display().to_string();
112
    let mut s = Setting::off("andy", "user");
113
    s.every = "hourly".into();
114
    s.target = target.clone();
115
    backups::save(&h.app.db, &s).await.unwrap();
116
    let first = backups::run(&h.app, "andy", "now").await.unwrap();
117
    assert_eq!((first.status.as_str(), first.repos), ("ok", 1), "{}", first.detail);
118
    assert!(first.sent > 0 && first.bytes > 0);
119
    let dest = Directory { root: store.path().join("andy") };
120
    assert!(dest.exists(&format!("runs/{}/DONE", first.at)).unwrap());
121
    let records: serde_json::Value = serde_json::from_slice(&dest.get(&format!("runs/{}/records/demo.json", first.at)).unwrap()).unwrap();
122
    assert_eq!(records["repositories"][0]["visibility"], "private");
123
    let text = String::from_utf8_lossy(&dest.get(&format!("runs/{}/records/demo.json", first.at)).unwrap()).into_owned();
124
    assert!(!text.contains("bob/other") && !text.contains("\"other\""), "nothing of another owner");
125
    assert!(!store.path().join("bob").exists(), "bob set up no backup");
126
127
    // Nothing changed: the next run sends nothing new.
128
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
129
    let second = backups::run(&h.app, "andy", "now").await.unwrap();
130
    assert_eq!((second.status.as_str(), second.sent), ("ok", 0), "{}", second.detail);
131
132
    // A fresh hub, restored from the backup, has the same repository.
133
    let empty = tempfile::tempdir().unwrap();
134
    let fresh = hub(empty.path()).await;
135
    let dest: Arc<dyn Destination> = Arc::new(Directory { root: store.path().join("andy") });
136
    let done = backups::restore_into_hub(&fresh.app, dest, None).await.unwrap();
137
    assert_eq!(done, vec!["andy/demo (repos)"]);
138
    let name = RepoName::parse("andy/demo").unwrap();
139
    assert_eq!(files(&fresh.app.repos.dir(&name)), files(&h.app.repos.dir(&name)), "byte for byte");
140
    let row = db::repos::get(&fresh.app.db, "andy", "demo").await.unwrap().expect("listed on the hub");
141
    assert_eq!((row.visibility.as_str(), row.description.as_str()), ("private", "a demo"));
142
    let (a, b) = (h.app.repos.open(&name).unwrap(), fresh.app.repos.open(&name).unwrap());
143
    assert_eq!(
144
        a.bookmarks().unwrap().iter().map(|b| b.heads.clone()).collect::<Vec<_>>(),
145
        b.bookmarks().unwrap().iter().map(|b| b.heads.clone()).collect::<Vec<_>>()
146
    );
147
148
    // A directory the hub does not allow is refused.
149
    let mut bad = Setting::off("andy", "user");
150
    bad.every = "hourly".into();
151
    bad.target = "/tmp/somewhere-else".into();
152
    backups::save(&h.app.db, &bad).await.unwrap();
153
    let refused = backups::run(&h.app, "andy", "now").await.unwrap();
154
    assert_eq!(refused.status, "failed");
155
    assert!(refused.detail.contains("not a backup directory this hub allows"), "{}", refused.detail);
156
}
157
158
#[tokio::test(flavor = "multi_thread")]
159
async fn backups_fall_due_on_their_schedule() {
160
    let store = tempfile::tempdir().unwrap();
161
    let h = hub(store.path()).await;
162
    let mut s = Setting::off("andy", "user");
163
    s.target = store.path().display().to_string();
164
    backups::save(&h.app.db, &s).await.unwrap();
165
    assert!(backups::due(&h.app.db).await.unwrap().is_empty(), "off is never due");
166
    s.every = "hourly".into();
167
    backups::save(&h.app.db, &s).await.unwrap();
168
    assert_eq!(backups::due(&h.app.db).await.unwrap().len(), 1, "never run: due");
169
    sqlx::query("INSERT INTO backup_runs (owner, started_ms, status) VALUES ('andy', $1, 'ok')")
170
        .bind(db::now_ms())
171
        .execute(&h.app.db)
172
        .await
173
        .unwrap();
174
    assert!(backups::due(&h.app.db).await.unwrap().is_empty(), "ran just now");
175
}
176
177
// ------------------------------------------------------------ a fake Google
178
179
/// What the fake has seen and holds.
180
#[derive(Default)]
181
struct Google {
182
    /// Objects by bucket and name.
183
    objects: std::collections::BTreeMap<(String, String), Vec<u8>>,
184
    /// The subjects and audiences of the tokens exchanged.
185
    exchanged: Vec<(String, String)>,
186
}
187
188
type Shared2 = Arc<std::sync::Mutex<Google>>;
189
190
fn decode(s: &str) -> String {
191
    let b = s.as_bytes();
192
    let mut out = Vec::new();
193
    let mut i = 0;
194
    while i < b.len() {
195
        if b[i] == b'%' && i + 2 < b.len() {
196
            out.push(u8::from_str_radix(&s[i + 1..i + 3], 16).unwrap());
197
            i += 3;
198
        } else {
199
            out.push(b[i]);
200
            i += 1;
201
        }
202
    }
203
    String::from_utf8(out).unwrap()
204
}
205
206
fn query(uri: &axum::http::Uri, key: &str) -> Option<String> {
207
    uri.query()?.split('&').find_map(|kv| kv.split_once('=').filter(|(k, _)| *k == key).map(|(_, v)| decode(v)))
208
}
209
210
/// Google's STS and Cloud Storage, as much as backups use: a token
211
/// exchange that admits `backup:` subjects for the provider's audience,
212
/// and buckets that admit only their owner's token (bucket `OWNER-backup`).
213
async fn fake_google() -> (String, Shared2) {
214
    use axum::extract::{Request, State};
215
    use axum::http::StatusCode;
216
    use axum::response::IntoResponse;
217
    let state: Shared2 = Default::default();
218
    async fn handle(State(g): State<Shared2>, req: Request) -> axum::response::Response {
219
        let (parts, body) = req.into_parts();
220
        let bytes = http_body_util::BodyExt::collect(body).await.unwrap().to_bytes();
221
        let path = parts.uri.path().to_string();
222
        if path == "/v1/token" {
223
            let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
224
            let jwt = v["subjectToken"].as_str().unwrap();
225
            let payload = jwt.split('.').nth(1).unwrap();
226
            let claims: serde_json::Value = serde_json::from_slice(&base64url(payload)).unwrap();
227
            let (sub, aud) = (claims["sub"].as_str().unwrap().to_string(), claims["aud"].as_str().unwrap().to_string());
228
            let provider = v["audience"].as_str().unwrap();
229
            g.lock().unwrap().exchanged.push((sub.clone(), aud.clone()));
230
            if !sub.starts_with("backup:") || aud != format!("https:{provider}") {
231
                return (
232
                    StatusCode::BAD_REQUEST,
233
                    axum::Json(serde_json::json!({"error": "invalid_grant", "error_description": "not admitted"})),
234
                )
235
                    .into_response();
236
            }
237
            return axum::Json(serde_json::json!({ "access_token": format!("tok-{sub}"), "expires_in": 3600 })).into_response();
238
        }
239
        let auth = parts.headers.get("authorization").and_then(|v| v.to_str().ok()).unwrap_or("").to_string();
240
        let segs: Vec<&str> = path.split('/').collect();
241
        // /storage/v1/b/{bucket}/o[/{name}]  or  /upload/storage/v1/b/{bucket}/o
242
        let upload = segs.get(1) == Some(&"upload");
243
        let at = if upload { 5 } else { 4 };
244
        let bucket = segs.get(at).unwrap_or(&"").to_string();
245
        let owner = bucket.trim_end_matches("-backup");
246
        if auth != format!("Bearer tok-backup:{owner}") {
247
            return (StatusCode::FORBIDDEN, "caller does not have storage.objects access").into_response();
248
        }
249
        let mut g = g.lock().unwrap();
250
        if upload {
251
            // As Google does: an upload must say its length, an empty one too.
252
            if !parts.headers.contains_key("content-length") {
253
                return (StatusCode::LENGTH_REQUIRED, "Error 411 (Length Required)").into_response();
254
            }
255
            let name = query(&parts.uri, "name").unwrap();
256
            g.objects.insert((bucket, name), bytes.to_vec());
257
            return axum::Json(serde_json::json!({})).into_response();
258
        }
259
        let name = segs.get(at + 2).map(|n| decode(n));
260
        match (parts.method.as_str(), name) {
261
            ("GET", Some(n)) => match g.objects.get(&(bucket, n.clone())) {
262
                Some(b) if query(&parts.uri, "alt").as_deref() == Some("media") => b.clone().into_response(),
263
                Some(_) => axum::Json(serde_json::json!({ "name": n })).into_response(),
264
                None => StatusCode::NOT_FOUND.into_response(),
265
            },
266
            ("DELETE", Some(n)) => {
267
                g.objects.remove(&(bucket, n));
268
                StatusCode::NO_CONTENT.into_response()
269
            }
270
            ("GET", None) => {
271
                let prefix = query(&parts.uri, "prefix").unwrap_or_default();
272
                let delimited = query(&parts.uri, "delimiter").is_some();
273
                let (mut items, mut prefixes) = (Vec::new(), std::collections::BTreeSet::new());
274
                for (b, n) in g.objects.keys() {
275
                    if *b != bucket || !n.starts_with(&prefix) {
276
                        continue;
277
                    }
278
                    let rest = &n[prefix.len()..];
279
                    match rest.find('/') {
280
                        Some(i) if delimited => {
281
                            prefixes.insert(format!("{prefix}{}", &rest[..=i]));
282
                        }
283
                        _ => items.push(serde_json::json!({ "name": n })),
284
                    }
285
                }
286
                axum::Json(serde_json::json!({ "items": items, "prefixes": prefixes })).into_response()
287
            }
288
            _ => StatusCode::METHOD_NOT_ALLOWED.into_response(),
289
        }
290
    }
291
    let app = axum::Router::new().fallback(handle).with_state(state.clone());
292
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
293
    let port = listener.local_addr().unwrap().port();
294
    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
295
    (format!("http://127.0.0.1:{port}"), state)
296
}
297
298
fn base64url(s: &str) -> Vec<u8> {
299
    let table = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
300
    let mut bits = 0u32;
301
    let mut n = 0;
302
    let mut out = Vec::new();
303
    for c in s.bytes() {
304
        let v = table.iter().position(|t| *t == c).unwrap() as u32;
305
        bits = (bits << 6) | v;
306
        n += 6;
307
        if n >= 8 {
308
            n -= 8;
309
            out.push((bits >> n) as u8);
310
            bits &= (1 << n) - 1;
311
        }
312
    }
313
    out
314
}
315
316
const PROVIDER: &str = "//iam.googleapis.com/projects/1/locations/global/workloadIdentityPools/oxygenhub/providers/oxygenhub-io";
317
318
fn gcs(owner: &str, bucket: &str) -> Setting {
319
    let mut s = Setting::off(owner, "user");
320
    s.every = "hourly".into();
321
    s.destination = "gcs".into();
322
    s.target = bucket.into();
323
    s.provider = PROVIDER.into();
324
    s
325
}
326
327
#[tokio::test(flavor = "multi_thread")]
328
async fn a_gcs_backup_needs_no_key_and_restores_the_same_repository() {
329
    let (google, seen) = fake_google().await;
330
    let store = tempfile::tempdir().unwrap();
331
    let h = hub_with(store.path(), Some(&google)).await;
332
    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
333
    db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();
334
    create(&h.app, "andy", "demo", hub_api::Visibility::Public).await;
335
    let work = tempfile::tempdir().unwrap();
336
    push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n")]);
337
    // A store holds empty files too (as oxygenhub.io's did, 2026-10-14):
338
    // they go up with a length of 0.
339
    std::fs::write(h.app.repos.dir(&RepoName::parse("andy/demo").unwrap()).join(".oxy/empty-marker"), b"").unwrap();
340
341
    backups::save(&h.app.db, &gcs("andy", "andy-backup")).await.unwrap();
342
    let first = backups::run(&h.app, "andy", "now").await.unwrap();
343
    assert_eq!((first.status.as_str(), first.repos), ("ok", 1), "{}", first.detail);
344
    {
345
        let g = seen.lock().unwrap();
346
        assert!(g.objects.keys().any(|(b, n)| b == "andy-backup" && n == &format!("runs/{}/DONE", first.at)));
347
        assert!(g.objects.keys().all(|(b, _)| b == "andy-backup"));
348
        let (sub, aud) = g.exchanged.first().unwrap();
349
        assert_eq!(
350
            (sub.as_str(), aud.as_str()),
351
            ("backup:andy", format!("https:{PROVIDER}").as_str()),
352
            "the hub signs in as the owner, for the provider"
353
        );
354
    }
355
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
356
    let second = backups::run(&h.app, "andy", "now").await.unwrap();
357
    assert_eq!((second.status.as_str(), second.sent), ("ok", 0), "{}", second.detail);
358
    // The probe writes, reads and removes; the bucket is as it was.
359
    let before = seen.lock().unwrap().objects.len();
360
    let dest = backups::destination(&h.app, &gcs("andy", "andy-backup")).unwrap();
361
    tokio::task::spawn_blocking(move || backups::probe(dest.as_ref())).await.unwrap().unwrap();
362
    assert_eq!(seen.lock().unwrap().objects.len(), before);
363
364
    // Restored from the bucket into a fresh hub: the same repository.
365
    let empty = tempfile::tempdir().unwrap();
366
    let fresh = hub_with(empty.path(), Some(&google)).await;
367
    let dest: Arc<dyn Destination> = backups::destination(&fresh.app, &gcs("andy", "andy-backup")).unwrap().into();
368
    assert_eq!(backups::restore_into_hub(&fresh.app, dest, None).await.unwrap(), vec!["andy/demo (repos)"]);
369
    let name = RepoName::parse("andy/demo").unwrap();
370
    assert_eq!(files(&fresh.app.repos.dir(&name)), files(&h.app.repos.dir(&name)), "byte for byte");
371
372
    // Bob cannot write Andy's bucket: Google refuses his token, and the
373
    // run says what to grant.
374
    backups::save(&h.app.db, &gcs("bob", "andy-backup")).await.unwrap();
375
    let refused = backups::run(&h.app, "bob", "now").await.unwrap();
376
    assert_eq!(refused.status, "failed");
377
    assert!(refused.detail.contains("roles/storage.objectAdmin") && refused.detail.contains("backup:bob"), "{}", refused.detail);
378
}
379
380
/// docs/backups.md §7 (BP1): a restore brings back the records as well as
381
/// the stores: issues with their comments, labels, milestone and
382
/// assignees, proposals with their reviews and events, releases with their
383
/// asset files, and webhooks without their secrets, every id mapped and
384
/// every account found again by name; what this hub lacks is said.
385
#[tokio::test(flavor = "multi_thread")]
386
async fn a_restore_brings_back_issues_proposals_and_releases() {
387
    let store = tempfile::tempdir().unwrap();
388
    let h = hub(store.path()).await;
389
    let andy = db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
390
    let bob = db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();
391
    create(&h.app, "andy", "demo", hub_api::Visibility::Public).await;
392
    let work = tempfile::tempdir().unwrap();
393
    push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n")]);
394
    let row = db::repos::get(&h.app.db, "andy", "demo").await.unwrap().unwrap();
395
    db::repos::set_diffusion(&h.app.db, row.id, "green").await.unwrap();
396
    // Ids on the first hub start further on than on the fresh one.
397
    for n in 0..3 {
398
        db::issues::upsert_label(&h.app.db, row.id, &format!("spare{n}"), "#000000", "").await.unwrap();
399
    }
400
    let bug = db::issues::upsert_label(&h.app.db, row.id, "bug", "#d73a4a", "Something is wrong").await.unwrap();
401
    let m = db::issues::upsert_milestone(&h.app.db, row.id, "1.0", "the first", None).await.unwrap();
402
    let issue = db::issues::create(
403
        &h.app.db,
404
        db::issues::NewIssue { repo_id: row.id, title: "Sprites flicker", body: "on level two", author_id: Some(andy.id), author: "andy" },
405
    )
406
    .await
407
    .unwrap();
408
    db::issues::add_comment(&h.app.db, issue.id, Some(bob.id), "bob", "seen it too").await.unwrap();
409
    db::issues::set_issue_label(&h.app.db, issue.id, bug.id, true).await.unwrap();
410
    db::issues::set_milestone(&h.app.db, issue.id, Some(m.id)).await.unwrap();
411
    db::issues::set_assignee(&h.app.db, issue.id, andy.id, true).await.unwrap();
412
    db::issues::set_assignee(&h.app.db, issue.id, bob.id, true).await.unwrap();
413
    let p = db::proposals::create(
414
        &h.app.db,
415
        db::proposals::NewProposal {
416
            repo_id: row.id,
417
            title: "Double-buffer the sprites",
418
            body: "fixes the flicker",
419
            author_id: Some(bob.id),
420
            author: "bob",
421
            draft: false,
422
            target: "main",
423
            source_kind: "bookmark",
424
            source_repo_id: Some(row.id),
425
            source_bookmark: "fix",
426
            source_changes: "",
427
        },
428
    )
429
    .await
430
    .unwrap();
431
    db::proposals::add_event(&h.app.db, p.id, "bob", "opened", "").await.unwrap();
432
    db::proposals::add_review(&h.app.db, p.id, andy.id, "andy", "approve", "nice").await.unwrap();
433
    let release = db::releases::create(&h.app.db, row.id, "v1.0", "One", "notes", false, Some((andy.id, "andy"))).await.unwrap();
434
    let asset = b"a game image".to_vec();
435
    let hash = blake3::hash(&asset).to_hex().to_string();
436
    let path = h.app.config.data.join("assets").join(&hash[..2]).join(&hash);
437
    std::fs::create_dir_all(path.parent().unwrap()).unwrap();
438
    std::fs::write(&path, &asset).unwrap();
439
    db::releases::add_asset(&h.app.db, release.id, "game.prg", asset.len() as i64, &hash, "application/octet-stream").await.unwrap();
440
    db::hooks::create_hook(&h.app.db, Some(row.id), None, "https://example.com/hook", "s3cret-value", "push").await.unwrap();
441
442
    let mut s = Setting::off("andy", "user");
443
    s.every = "hourly".into();
444
    s.target = store.path().display().to_string();
445
    backups::save(&h.app.db, &s).await.unwrap();
446
    let run = backups::run(&h.app, "andy", "now").await.unwrap();
447
    assert_eq!(run.status, "ok", "{}", run.detail);
448
    let dest = Directory { root: store.path().join("andy") };
449
    let text = String::from_utf8_lossy(&dest.get(&format!("runs/{}/records/demo.json", run.at)).unwrap()).into_owned();
450
    assert!(!text.contains("s3cret-value"), "never a secret");
451
    assert!(text.contains("\"_names\"") && text.contains("\"bob\""), "accounts by name");
452
453
    // A fresh hub that knows andy but not bob.
454
    let empty = tempfile::tempdir().unwrap();
455
    let fresh = hub(empty.path()).await;
456
    let andy2 = db::accounts::create_account(&fresh.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
457
    let dest: Arc<dyn Destination> = Arc::new(Directory { root: store.path().join("andy") });
458
    let done = backups::restore_into_hub(&fresh.app, dest, None).await.unwrap();
459
    assert_eq!(done[0], "andy/demo (repos)");
460
    let notes = done[1..].join("\n");
461
    assert!(notes.contains("webhook https://example.com/hook: enter its secret again"), "{notes}");
462
    assert!(notes.contains("issue_assignees: 1 row(s) left out"), "bob is not on this hub: {notes}");
463
    let r2 = db::repos::get(&fresh.app.db, "andy", "demo").await.unwrap().unwrap();
464
    assert_eq!(r2.diffusion, "green", "settings come back");
465
    // The issue, its comment, label, milestone and the assignee this hub has.
466
    let issues: Vec<(i64, i64, String, Option<i64>)> = sqlx::query_as("SELECT id, number, title, author_id FROM issues WHERE repo_id = $1")
467
        .bind(r2.id)
468
        .fetch_all(&fresh.app.db)
469
        .await
470
        .unwrap();
471
    assert_eq!(issues.len(), 1);
472
    let (iid, number, title, author) = issues[0].clone();
473
    assert_eq!((number, title.as_str(), author), (issue.number, "Sprites flicker", Some(andy2.id)));
474
    let comments = db::issues::comments(&fresh.app.db, iid).await.unwrap();
475
    assert_eq!(comments.len(), 1);
476
    assert_eq!((comments[0].author.as_str(), comments[0].body.as_str()), ("bob", "seen it too"));
477
    let labels: Vec<String> =
478
        sqlx::query_scalar("SELECT l.name FROM issue_labels il JOIN labels l ON l.id = il.label_id WHERE il.issue_id = $1")
479
            .bind(iid)
480
            .fetch_all(&fresh.app.db)
481
            .await
482
            .unwrap();
483
    assert_eq!(labels, vec!["bug"], "the label, its id mapped");
484
    let milestone: Option<String> =
485
        sqlx::query_scalar("SELECT m.title FROM issues i JOIN milestones m ON m.id = i.milestone_id WHERE i.id = $1")
486
            .bind(iid)
487
            .fetch_one(&fresh.app.db)
488
            .await
489
            .unwrap();
490
    assert_eq!(milestone.as_deref(), Some("1.0"));
491
    assert_eq!(db::issues::assignees(&fresh.app.db, iid).await.unwrap(), vec!["andy"]);
492
    // The proposal, its review and its timeline.
493
    let p2 = db::proposals::get(&fresh.app.db, r2.id, p.number).await.unwrap().expect("the proposal");
494
    assert_eq!((p2.title.as_str(), p2.author.as_str(), p2.author_id), ("Double-buffer the sprites", "bob", None));
495
    let reviews: Vec<(String, String)> = sqlx::query_as("SELECT author, verdict FROM proposal_reviews WHERE proposal_id = $1")
496
        .bind(p2.id)
497
        .fetch_all(&fresh.app.db)
498
        .await
499
        .unwrap();
500
    assert_eq!(reviews, vec![("andy".to_string(), "approve".to_string())]);
501
    assert!(db::proposals::events(&fresh.app.db, p2.id).await.unwrap().iter().any(|e| e.event == "opened"));
502
    // The release and its asset's bytes.
503
    let rel = db::releases::get(&fresh.app.db, r2.id, "v1.0").await.unwrap().expect("the release");
504
    let assets = db::releases::assets(&fresh.app.db, rel.id).await.unwrap();
505
    assert_eq!(assets.len(), 1);
506
    assert_eq!(std::fs::read(fresh.app.config.data.join("assets").join(&hash[..2]).join(&hash)).unwrap(), asset);
507
    // Search finds what came back.
508
    let hits = db::notify::search(&fresh.app.db, "flicker", Some(r2.id), 10).await.unwrap();
509
    assert!(hits.iter().any(|h| h.kind == "issue" && h.target_id == iid), "{:?}", hits.iter().map(|h| &h.title).collect::<Vec<_>>());
510
    // New issues number on from where the old hub was.
511
    let next = db::issues::create(
512
        &fresh.app.db,
513
        db::issues::NewIssue { repo_id: r2.id, title: "After", body: "", author_id: Some(andy2.id), author: "andy" },
514
    )
515
    .await
516
    .unwrap();
517
    assert!(next.number > p.number, "{} after {}", next.number, p.number);
518
}
519
520
/// docs/backups.md §5 (BP3): a repository left out is in no run; one with
521
/// its own schedule makes its owner's backup run at least that often, and
522
/// alone when its owner's is off; a run by hand takes all but those left
523
/// out.
524
#[tokio::test(flavor = "multi_thread")]
525
async fn repositories_choose_their_own_backup() {
526
    let store = tempfile::tempdir().unwrap();
527
    let h = hub(store.path()).await;
528
    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
529
    for n in ["game", "notes", "secret"] {
530
        create(&h.app, "andy", n, hub_api::Visibility::Public).await;
531
        let work = tempfile::tempdir().unwrap();
532
        push(&format!("{}/andy/{n}", h.base), work.path(), &[("README.md", n)]);
533
    }
534
    let id = |n: &'static str| {
535
        let db = h.app.db.clone();
536
        async move { db::repos::get(&db, "andy", n).await.unwrap().unwrap().id }
537
    };
538
    db::repos::set_backup(&h.app.db, id("secret").await, "off").await.unwrap();
539
    let mut s = Setting::off("andy", "user");
540
    s.every = "daily".into();
541
    s.target = store.path().display().to_string();
542
    backups::save(&h.app.db, &s).await.unwrap();
543
    let names = |at: String| {
544
        let dest = Directory { root: store.path().join("andy") };
545
        let mut v = dest.list(&format!("runs/{at}/records")).unwrap();
546
        v.sort();
547
        v
548
    };
549
    let first = backups::run(&h.app, "andy", "schedule").await.unwrap();
550
    assert_eq!((first.status.as_str(), first.repos), ("ok", 2), "{}", first.detail);
551
    assert_eq!(names(first.at.clone()), vec!["game.json", "notes.json"], "the one left out is not in it");
552
    // Daily, and it just ran: not due, until a repository asks for hourly.
553
    assert!(backups::due(&h.app.db).await.unwrap().is_empty());
554
    db::repos::set_backup(&h.app.db, id("game").await, "hourly").await.unwrap();
555
    sqlx::query("UPDATE backup_runs SET started_ms = started_ms - 2 * 3600 * 1000").execute(&h.app.db).await.unwrap();
556
    assert_eq!(backups::due(&h.app.db).await.unwrap().len(), 1, "hourly for the game");
557
    // The owner's schedule off: only the game, which asks for its own.
558
    s.every = "off".into();
559
    backups::save(&h.app.db, &s).await.unwrap();
560
    assert_eq!(backups::due(&h.app.db).await.unwrap().len(), 1);
561
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
562
    let second = backups::run(&h.app, "andy", "schedule").await.unwrap();
563
    assert_eq!(names(second.at.clone()), vec!["game.json"]);
564
    // By hand: everything but the one left out.
565
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
566
    let third = backups::run(&h.app, "andy", "now").await.unwrap();
567
    assert_eq!(names(third.at.clone()), vec!["game.json", "notes.json"]);
568
    // Admin › Backups counts what the next scheduled run covers.
569
    let (owners, uncovered) = backups::overview(&h.app.db).await.unwrap();
570
    assert_eq!((owners[0].repos, uncovered.len()), (1, 0));
571
}
572
573
/// AWS STS and S3, as much as backups use (path style, under `/s3`): an
574
/// AssumeRoleWithWebIdentity that admits `backup:` subjects with audience
575
/// `sts.amazonaws.com`, and buckets (`OWNER-backup`) that check every
576
/// request's Signature Version 4 and admit only their owner's session.
577
async fn fake_aws() -> (String, Shared2) {
578
    use axum::extract::{Request, State};
579
    use axum::http::StatusCode;
580
    use axum::response::IntoResponse;
581
    let state: Shared2 = Default::default();
582
    async fn handle(State(g): State<Shared2>, req: Request) -> axum::response::Response {
583
        let (parts, body) = req.into_parts();
584
        let bytes = http_body_util::BodyExt::collect(body).await.unwrap().to_bytes();
585
        let path = parts.uri.path().to_string();
586
        let header = |h: &str| parts.headers.get(h).and_then(|v| v.to_str().ok()).unwrap_or("").to_string();
587
        if path == "/" && parts.method == "POST" {
588
            let form: Vec<(String, String)> = String::from_utf8_lossy(&bytes)
589
                .split('&')
590
                .filter_map(|kv| kv.split_once('=').map(|(k, v)| (k.to_string(), decode(&v.replace('+', " ")))))
591
                .collect();
592
            let field = |k: &str| form.iter().find(|(f, _)| f == k).map(|(_, v)| v.clone()).unwrap_or_default();
593
            let jwt = field("WebIdentityToken");
594
            let claims: serde_json::Value = serde_json::from_slice(&base64url(jwt.split('.').nth(1).unwrap())).unwrap();
595
            let (sub, aud) = (claims["sub"].as_str().unwrap().to_string(), claims["aud"].as_str().unwrap().to_string());
596
            g.lock().unwrap().exchanged.push((sub.clone(), aud.clone()));
597
            if field("Action") != "AssumeRoleWithWebIdentity" || !sub.starts_with("backup:") || aud != "sts.amazonaws.com" {
598
                return (StatusCode::FORBIDDEN, "<ErrorResponse><Error><Code>AccessDenied</Code><Message>Not authorized to perform sts:AssumeRoleWithWebIdentity</Message></Error></ErrorResponse>").into_response();
599
            }
600
            let owner = sub.trim_start_matches("backup:");
601
            return format!(
602
                "<AssumeRoleWithWebIdentityResponse><AssumeRoleWithWebIdentityResult><Credentials>\
603
                 <AccessKeyId>ASIA{owner}</AccessKeyId><SecretAccessKey>secret/{owner}</SecretAccessKey>\
604
                 <SessionToken>session-{owner}</SessionToken><Expiration>2030-01-01T00:00:00Z</Expiration>\
605
                 </Credentials></AssumeRoleWithWebIdentityResult></AssumeRoleWithWebIdentityResponse>"
606
            )
607
            .into_response();
608
        }
609
        // /s3/{bucket}/{key…}
610
        let rest = path.strip_prefix("/s3/").unwrap_or("");
611
        let (bucket, key) = rest.split_once('/').unwrap_or((rest, ""));
612
        let (bucket, key) = (bucket.to_string(), decode(key));
613
        let owner = bucket.trim_end_matches("-backup").to_string();
614
        // Check the signature as AWS would, with the session's secret.
615
        let session = header("x-amz-security-token");
616
        let who = session.trim_start_matches("session-").to_string();
617
        let query: Vec<(String, String)> = parts
618
            .uri
619
            .query()
620
            .unwrap_or("")
621
            .split('&')
622
            .filter(|kv| !kv.is_empty())
623
            .map(|kv| kv.split_once('=').map(|(k, v)| (decode(k), decode(v))).unwrap_or((decode(kv), String::new())))
624
            .collect();
625
        let signed = vec![
626
            ("host".to_string(), header("host")),
627
            ("x-amz-content-sha256".to_string(), header("x-amz-content-sha256")),
628
            ("x-amz-date".to_string(), header("x-amz-date")),
629
            ("x-amz-security-token".to_string(), session.clone()),
630
        ];
631
        let id = format!("ASIA{who}");
632
        let secret = format!("secret/{who}");
633
        let want = oxygenhub::backups::sigv4(
634
            parts.method.as_str(),
635
            &path,
636
            &query,
637
            &signed,
638
            &header("x-amz-content-sha256"),
639
            (&id, &secret),
640
            "eu-west-2",
641
            "s3",
642
            &header("x-amz-date"),
643
        );
644
        use sha2::Digest;
645
        let body_hash = hex::encode(sha2::Sha256::digest(&bytes));
646
        if header("authorization") != want || header("x-amz-content-sha256") != body_hash {
647
            return (
648
                StatusCode::FORBIDDEN,
649
                "<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>",
650
            )
651
                .into_response();
652
        }
653
        if who != owner {
654
            return (StatusCode::FORBIDDEN, "<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>").into_response();
655
        }
656
        let mut g = g.lock().unwrap();
657
        match (parts.method.as_str(), key.is_empty()) {
658
            ("PUT", false) if !parts.headers.contains_key("content-length") => {
659
                (StatusCode::LENGTH_REQUIRED, "<Error><Code>MissingContentLength</Code></Error>").into_response()
660
            }
661
            ("PUT", false) => {
662
                g.objects.insert((bucket, key), bytes.to_vec());
663
                StatusCode::OK.into_response()
664
            }
665
            ("GET", false) | ("HEAD", false) => match g.objects.get(&(bucket, key)) {
666
                Some(b) => b.clone().into_response(),
667
                None => (StatusCode::NOT_FOUND, "<Error><Code>NoSuchKey</Code></Error>").into_response(),
668
            },
669
            ("DELETE", false) => {
670
                g.objects.remove(&(bucket, key));
671
                StatusCode::NO_CONTENT.into_response()
672
            }
673
            ("GET", true) => {
674
                let prefix = query.iter().find(|(k, _)| k == "prefix").map(|(_, v)| v.clone()).unwrap_or_default();
675
                let delimited = query.iter().any(|(k, _)| k == "delimiter");
676
                let (mut keys, mut prefixes) = (String::new(), std::collections::BTreeSet::new());
677
                for (b, n) in g.objects.keys() {
678
                    if *b != bucket || !n.starts_with(&prefix) {
679
                        continue;
680
                    }
681
                    match n[prefix.len()..].find('/') {
682
                        Some(i) if delimited => {
683
                            prefixes.insert(format!("{prefix}{}", &n[prefix.len()..prefix.len() + i + 1]));
684
                        }
685
                        _ => keys.push_str(&format!("<Contents><Key>{n}</Key></Contents>")),
686
                    }
687
                }
688
                let common: String = prefixes.iter().map(|p| format!("<CommonPrefixes><Prefix>{p}</Prefix></CommonPrefixes>")).collect();
689
                format!("<ListBucketResult><Prefix>{prefix}</Prefix><IsTruncated>false</IsTruncated>{keys}{common}</ListBucketResult>")
690
                    .into_response()
691
            }
692
            _ => StatusCode::METHOD_NOT_ALLOWED.into_response(),
693
        }
694
    }
695
    let app = axum::Router::new().fallback(handle).with_state(state.clone());
696
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
697
    let port = listener.local_addr().unwrap().port();
698
    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
699
    (format!("http://127.0.0.1:{port}"), state)
700
}
701
702
async fn hub_aws(backups: &Path, aws: &str) -> Hub {
703
    hermetic();
704
    let dir = tempfile::tempdir().unwrap();
705
    let mut config = Config::load(dir.path()).unwrap();
706
    // The first account's setup code (oxygenhub#245).
707
    config.registration.setup_code = "test-setup".into();
708
    config.auth.push_tokens = vec!["site".into()];
709
    config.backups.directories = vec![backups.to_path_buf()];
710
    config.backups.aws_sts_url = aws.to_string();
711
    config.backups.s3_url = format!("{aws}/s3/{{bucket}}");
712
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
713
    let port = listener.local_addr().unwrap().port();
714
    config.server.base_url = format!("http://127.0.0.1:{port}");
715
    let app = oxygenhub::build(config).await.unwrap();
716
    let router = oxygenhub::router(app.clone());
717
    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
718
    Hub { app, base: format!("http://127.0.0.1:{port}"), _dir: dir }
719
}
720
721
const ROLE: &str = "arn:aws:iam::123456789012:role/oxygenhub-backup";
722
723
fn s3(owner: &str, bucket: &str) -> Setting {
724
    let mut s = Setting::off(owner, "user");
725
    s.every = "hourly".into();
726
    s.destination = "s3".into();
727
    s.target = bucket.into();
728
    s.provider = ROLE.into();
729
    s.region = "eu-west-2".into();
730
    s.prefix = "oxygen hub".into();
731
    s
732
}
733
734
/// docs/backups.md §3 (BP4): an S3 bucket, keylessly: the hub's own token
735
/// for AWS STS, then signed requests; a second run sends nothing; the
736
/// probe leaves the bucket as it was; a fresh hub restores the same
737
/// repository from it; another owner's session is refused, saying what the
738
/// role needs.
739
#[tokio::test(flavor = "multi_thread")]
740
async fn an_s3_backup_needs_no_key_and_restores_the_same_repository() {
741
    let (aws, seen) = fake_aws().await;
742
    let store = tempfile::tempdir().unwrap();
743
    let h = hub_aws(store.path(), &aws).await;
744
    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
745
    db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();
746
    create(&h.app, "andy", "demo", hub_api::Visibility::Public).await;
747
    let work = tempfile::tempdir().unwrap();
748
    push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n"), ("src/a b+c.rs", "fn main() {}\n")]);
749
    std::fs::write(h.app.repos.dir(&RepoName::parse("andy/demo").unwrap()).join(".oxy/empty-marker"), b"").unwrap();
750
751
    backups::save(&h.app.db, &s3("andy", "andy-backup")).await.unwrap();
752
    let first = backups::run(&h.app, "andy", "now").await.unwrap();
753
    assert_eq!((first.status.as_str(), first.repos), ("ok", 1), "{}", first.detail);
754
    {
755
        let g = seen.lock().unwrap();
756
        assert!(
757
            g.objects.keys().any(|(b, n)| b == "andy-backup" && n == &format!("oxygen hub/runs/{}/DONE", first.at)),
758
            "under the prefix"
759
        );
760
        assert_eq!(g.exchanged.first().map(|(s, a)| (s.as_str(), a.as_str())), Some(("backup:andy", "sts.amazonaws.com")));
761
    }
762
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
763
    let second = backups::run(&h.app, "andy", "now").await.unwrap();
764
    assert_eq!((second.status.as_str(), second.sent), ("ok", 0), "{}", second.detail);
765
    let before = seen.lock().unwrap().objects.len();
766
    let dest = backups::destination(&h.app, &s3("andy", "andy-backup")).unwrap();
767
    tokio::task::spawn_blocking(move || backups::probe(dest.as_ref())).await.unwrap().unwrap();
768
    assert_eq!(seen.lock().unwrap().objects.len(), before, "the probe cleans up");
769
770
    let empty = tempfile::tempdir().unwrap();
771
    let fresh = hub_aws(empty.path(), &aws).await;
772
    let dest: Arc<dyn Destination> = backups::destination(&fresh.app, &s3("andy", "andy-backup")).unwrap().into();
773
    assert_eq!(backups::restore_into_hub(&fresh.app, dest, None).await.unwrap(), vec!["andy/demo (repos)"]);
774
    let name = RepoName::parse("andy/demo").unwrap();
775
    assert_eq!(files(&fresh.app.repos.dir(&name)), files(&h.app.repos.dir(&name)), "byte for byte");
776
777
    backups::save(&h.app.db, &s3("bob", "andy-backup")).await.unwrap();
778
    let refused = backups::run(&h.app, "bob", "now").await.unwrap();
779
    assert_eq!(refused.status, "failed");
780
    assert!(refused.detail.contains("s3:PutObject") && refused.detail.contains(ROLE), "{}", refused.detail);
781
}
782
783
/// One API call with a bearer token: the status and the JSON answer.
784
async fn api(
785
    app: &Shared,
786
    method: &str,
787
    path: &str,
788
    token: &str,
789
    body: Option<serde_json::Value>,
790
) -> (axum::http::StatusCode, serde_json::Value) {
791
    use axum::body::Body;
792
    use http_body_util::BodyExt;
793
    use tower::ServiceExt;
794
    let mut req =
795
        axum::http::Request::builder().method(method).uri(format!("/api/v1{path}")).header("authorization", format!("Bearer {token}"));
796
    let body = match body {
797
        Some(b) => {
798
            req = req.header("content-type", "application/json");
799
            Body::from(b.to_string())
800
        }
801
        None => Body::empty(),
802
    };
803
    let res = oxygenhub::router(app.clone()).oneshot(req.body(body).unwrap()).await.unwrap();
804
    let st = res.status();
805
    let bytes = res.into_body().collect().await.unwrap().to_bytes();
806
    (st, serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null))
807
}
808
809
/// oxygenhub#179: the API does what the Backups pages do, for the caller's
810
/// own repositories and an organisation it owns, with repo:admin: the
811
/// setting (checked as the page checks it), TEST, back up now, the runs.
812
#[tokio::test(flavor = "multi_thread")]
813
async fn backups_are_set_up_and_run_through_the_api() {
814
    let store = tempfile::tempdir().unwrap();
815
    let h = hub(store.path()).await;
816
    let andy = db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", false).await.unwrap();
817
    let bob = db::accounts::create_account(&h.app.db, "bob", "[email protected]", "x", "salt", false).await.unwrap();
818
    db::accounts::create_org(&h.app.db, "club", "Club", andy.id).await.unwrap();
819
    create(&h.app, "andy", "demo", hub_api::Visibility::Public).await;
820
    create(&h.app, "club", "game", hub_api::Visibility::Private).await;
821
    let token = |id: i64, scopes: &'static str| {
822
        let db = h.app.db.clone();
823
        async move {
824
            let (t, hash) = oxygenhub::auth::new_token();
825
            db::accounts::create_token(&db, id, "t", &hash, scopes, "", None).await.unwrap();
826
            t
827
        }
828
    };
829
    let (andy_t, bob_t, weak) = (token(andy.id, "repo:admin").await, token(bob.id, "repo:admin").await, token(andy.id, "repo:read").await);
830
    let dir = store.path().display().to_string();
831
832
    let (st, empty) = api(&h.app, "GET", "/user/backups", &andy_t, None).await;
833
    assert_eq!(st, 200);
834
    assert!(empty["setting"].is_null() && empty["runs"].as_array().unwrap().is_empty(), "{empty}");
835
    let (st, _) = api(&h.app, "POST", "/user/backups/run", &andy_t, None).await;
836
    assert_eq!(st, 409, "nothing saved yet");
837
    let (st, _) =
838
        api(&h.app, "PUT", "/user/backups", &weak, Some(serde_json::json!({ "every": "hourly", "destination": "dir", "dir": dir }))).await;
839
    assert_eq!(st, 403, "repo:admin only");
840
    let (st, bad) = api(
841
        &h.app,
842
        "PUT",
843
        "/user/backups",
844
        &andy_t,
845
        Some(serde_json::json!({ "every": "hourly", "destination": "gcs", "bucket": "andy-backup", "provider": "nope" })),
846
    )
847
    .await;
848
    assert_eq!(st, 400, "checked as the page checks it: {bad}");
849
    let (st, saved) = api(
850
        &h.app,
851
        "PUT",
852
        "/user/backups",
853
        &andy_t,
854
        Some(serde_json::json!({ "every": "hourly", "destination": "dir", "dir": dir, "keep_days": 30 })),
855
    )
856
    .await;
857
    assert_eq!(st, 200, "{saved}");
858
    assert_eq!((saved["setting"]["every"].as_str(), saved["place"].as_str()), (Some("hourly"), Some(format!("{dir}/andy").as_str())));
859
    let (st, probe) = api(&h.app, "POST", "/user/backups/test", &andy_t, None).await;
860
    assert_eq!((st.as_u16(), probe["ok"].as_bool()), (200, Some(true)), "{probe}");
861
    let (st, started) = api(&h.app, "POST", "/user/backups/run", &andy_t, None).await;
862
    assert_eq!(st, 202, "{started}");
863
    let mut done = serde_json::Value::Null;
864
    for _ in 0..200 {
865
        let (_, runs) = api(&h.app, "GET", "/user/backups/runs", &andy_t, None).await;
866
        if runs[0]["status"].as_str().is_some_and(|s| s != "running") {
867
            done = runs[0].clone();
868
            break;
869
        }
870
        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
871
    }
872
    assert_eq!((done["status"].as_str(), done["repos"].as_i64(), done["trigger"].as_str()), (Some("ok"), Some(1), Some("now")), "{done}");
873
    // A whole run after a failed one says it works again; the day's
874
    // summary tells the owner what ran (#183).
875
    sqlx::query("INSERT INTO backup_runs (owner, started_ms, finished_ms, status, at, trigger, detail) VALUES ('andy', 1, 2, 'failed', 'x', 'schedule', 'GCS refused')")
876
        .execute(&h.app.db)
877
        .await
878
        .unwrap();
879
    tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
880
    let again = backups::run(&h.app, "andy", "now").await.unwrap();
881
    assert_eq!(again.status, "ok");
882
    let told = |db: oxygenhub::db::Pool| async move { db::notify::inbox(&db, andy.id, 20).await.unwrap() };
883
    assert!(told(h.app.db.clone()).await.iter().any(|n| n.kind == "backup.recovered" && n.title.contains("works again")));
884
    assert_eq!(backups::daily_summary(&h.app, db::now_ms(), true).await.unwrap(), 1);
885
    let summary = told(h.app.db.clone()).await.into_iter().find(|n| n.kind == "backup.summary").expect("a summary");
886
    assert!(summary.title.contains("Backups of andy today") && summary.title.contains("runs"), "{}", summary.title);
887
888
    // An organisation's, by its owner; another account sees no such thing.
889
    let (st, _) =
890
        api(&h.app, "PUT", "/orgs/club/backups", &bob_t, Some(serde_json::json!({ "every": "hourly", "destination": "dir", "dir": dir })))
891
            .await;
892
    assert_eq!(st, 404);
893
    let (st, org) = api(
894
        &h.app,
895
        "PUT",
896
        "/orgs/club/backups",
897
        &andy_t,
898
        Some(serde_json::json!({ "every": "daily", "destination": "dir", "dir": dir, "keep_days": 90 })),
899
    )
900
    .await;
901
    assert_eq!(st, 200, "{org}");
902
    assert_eq!((org["setting"]["kind"].as_str(), org["setting"]["keep_days"].as_i64()), (Some("org"), Some(90)));
903
    let (st, _) = api(&h.app, "GET", "/orgs/club/backups", &bob_t, None).await;
904
    assert_eq!(st, 404);
905
}