OXYGENHUB
# cb7b3cd45dc0 oxygenhub / crates / hub / tests / roles.rs
7.6 KiB · Rust · last changed Secret scanning: scan existing history, remove an allowed fingerprint 16 days ago source blame history raw
1
//! Roles (docs/roles.md, #272): a web-only hub starts none of the worker's
2
//! background tasks, a worker-only hub serves no pages, and on PostgreSQL
3
//! two hubs on one database run each singleton task once between them.
4
5
use axum::body::Body;
6
use axum::http::{header, Request, StatusCode};
7
use http_body_util::BodyExt;
8
use oxygenhub::config::Config;
9
use oxygenhub::Shared;
10
use std::path::Path;
11
use std::time::{Duration, Instant};
12
use tower::ServiceExt;
13
14
/// The worker's singleton tasks that run for as long as the process does.
15
const WORKER_SINGLETONS: [&str; 8] = [
16
    "backups.scheduler",
17
    "diffusion.reconciler",
18
    "flow.placement",
19
    "flow.reaper",
20
    "flow.scheduler",
21
    "hooks.worker",
22
    "maintenance.nightly",
23
    "secrets.history_scan",
24
];
25
26
fn hermetic() {
27
    static ONCE: std::sync::Once = std::sync::Once::new();
28
    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();
29
    ONCE.call_once(|| {
30
        let d = tempfile::tempdir().unwrap();
31
        std::env::set_var("OXY_HOME", d.path());
32
        DIR.set(d).ok();
33
    });
34
}
35
36
fn config(dir: &Path, roles: &[&str]) -> Config {
37
    let mut config = Config::load(dir).unwrap();
38
    config.server.roles = roles.iter().map(|r| r.to_string()).collect();
39
    config.registration.setup_code = "test-setup".into();
40
    config
41
}
42
43
async fn get(app: &Shared, path: &str, token: Option<&str>) -> (StatusCode, String) {
44
    let mut req = Request::get(path);
45
    if let Some(t) = token {
46
        req = req.header(header::AUTHORIZATION, format!("Bearer {t}"));
47
    }
48
    let res = oxygenhub::router(app.clone()).oneshot(req.body(Body::empty()).unwrap()).await.unwrap();
49
    let status = res.status();
50
    let body = res.into_body().collect().await.unwrap().to_bytes();
51
    (status, String::from_utf8_lossy(&body).into_owned())
52
}
53
54
async fn health(app: &Shared) -> serde_json::Value {
55
    let (status, body) = get(app, "/health", None).await;
56
    assert_eq!(status, StatusCode::OK, "{body}");
57
    serde_json::from_str(&body).unwrap()
58
}
59
60
/// Wait up to `limit` for `ok`, checking every 100 ms.
61
async fn until(limit: Duration, mut ok: impl FnMut() -> bool) -> bool {
62
    let start = Instant::now();
63
    while start.elapsed() < limit {
64
        if ok() {
65
            return true;
66
        }
67
        tokio::time::sleep(Duration::from_millis(100)).await;
68
    }
69
    ok()
70
}
71
72
#[tokio::test(flavor = "multi_thread")]
73
async fn a_web_only_hub_starts_none_of_the_workers_tasks() {
74
    hermetic();
75
    let dir = tempfile::tempdir().unwrap();
76
    let app = oxygenhub::build(config(dir.path(), &["web"])).await.unwrap();
77
    // Only the transparency log's checkpointer, which goes with the log's
78
    // appends in the web role.
79
    assert_eq!(app.leader.tasks(), ["tlog.checkpointer"]);
80
    assert!(until(Duration::from_secs(5), || app.leader.leading() == ["tlog.checkpointer"]).await, "{:?}", app.leader.leading());
81
    let h = health(&app).await;
82
    assert_eq!(h["roles"], serde_json::json!(["web"]));
83
    assert_eq!(h["leading"], serde_json::json!(["tlog.checkpointer"]));
84
    // It serves pages.
85
    let (status, body) = get(&app, "/explore", None).await;
86
    assert_eq!(status, StatusCode::OK, "{body}");
87
}
88
89
#[tokio::test(flavor = "multi_thread")]
90
async fn a_worker_only_hub_serves_its_health_and_metrics_and_no_pages() {
91
    hermetic();
92
    let dir = tempfile::tempdir().unwrap();
93
    let mut cfg = config(dir.path(), &["worker"]);
94
    cfg.server.metrics_token = "m".into();
95
    let app = oxygenhub::build(cfg).await.unwrap();
96
    let mut tasks = WORKER_SINGLETONS.to_vec();
97
    tasks.extend(["diffusion.backfill", "images.learn_archs"]);
98
    tasks.sort();
99
    assert_eq!(app.leader.tasks(), tasks, "every worker task, and not the log's checkpointer");
100
    assert!(
101
        until(Duration::from_secs(5), || WORKER_SINGLETONS.iter().all(|n| app.leader.leading().contains(n))).await,
102
        "{:?}",
103
        app.leader.leading()
104
    );
105
    let h = health(&app).await;
106
    assert_eq!(h["roles"], serde_json::json!(["worker"]));
107
    assert!(h["leading"].as_array().unwrap().iter().any(|n| n == "flow.scheduler"), "{h}");
108
    for path in ["/", "/explore", "/login", "/api/v1/version", "/andy/demo", "/log/checkpoint"] {
109
        assert_eq!(get(&app, path, None).await.0, StatusCode::NOT_FOUND, "{path} is the web role's");
110
    }
111
    let (status, body) = get(&app, "/metrics", Some("m")).await;
112
    assert_eq!(status, StatusCode::OK, "{body}");
113
    // It reads the transparency log but never writes it.
114
    assert!(app.tlog.append(&[]).is_err());
115
}
116
117
#[tokio::test(flavor = "multi_thread")]
118
async fn a_hub_takes_every_role_by_default() {
119
    hermetic();
120
    let dir = tempfile::tempdir().unwrap();
121
    let app = oxygenhub::build(config(dir.path(), &[])).await.unwrap();
122
    assert_eq!(app.leader.tasks().len(), 11, "{:?}", app.leader.tasks());
123
    assert_eq!(health(&app).await["roles"], serde_json::json!(["web", "worker"]));
124
    assert_eq!(get(&app, "/explore", None).await.0, StatusCode::OK);
125
    let mut bad = config(dir.path(), &["web", "cron"]);
126
    bad.server.roles.push(String::new());
127
    let e = oxygenhub::build(bad).await.err().expect("an unknown role is refused").to_string();
128
    assert!(e.contains("unknown role \"cron\""), "{e}");
129
}
130
131
/// Two hubs on one PostgreSQL database (one schema): exactly one leads
132
/// each singleton; when its connection drops it stops them, and when it
133
/// stops the other takes every one over within seconds.
134
#[cfg(feature = "postgres")]
135
#[tokio::test(flavor = "multi_thread")]
136
async fn two_hubs_on_one_database_run_each_singleton_once_and_fail_over() {
137
    hermetic();
138
    let (da, dbb) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
139
    let schema = format!("hub_roles_{}", da.path().file_name().unwrap().to_string_lossy());
140
    let pg = |dir: &Path| {
141
        let mut c = config(dir, &[]);
142
        c.database.schema = schema.clone();
143
        c
144
    };
145
    let a = oxygenhub::build(pg(da.path())).await.unwrap();
146
    let b = oxygenhub::build(pg(dbb.path())).await.unwrap();
147
    let mut singletons = WORKER_SINGLETONS.to_vec();
148
    singletons.push("tlog.checkpointer");
149
    let once = |a: &Shared, b: &Shared| {
150
        let (la, lb) = (a.leader.leading(), b.leader.leading());
151
        singletons.iter().all(|n| la.contains(n) != lb.contains(n))
152
    };
153
    assert!(until(Duration::from_secs(10), || once(&a, &b)).await, "a {:?}, b {:?}", a.leader.leading(), b.leader.leading());
154
155
    // A's connection is ended, as when the network drops: its locks free,
156
    // it stops what it led, and between them each runs once again.
157
    let pid = a.leader.session_pid().await.expect("a has a session");
158
    let ended: bool = sqlx::query_scalar("SELECT pg_terminate_backend($1)").bind(pid).fetch_one(&b.db).await.unwrap();
159
    assert!(ended);
160
    let start = Instant::now();
161
    loop {
162
        let reconnected = a.leader.session_pid().await.is_some_and(|p| p != pid);
163
        if reconnected && once(&a, &b) {
164
            break;
165
        }
166
        assert!(start.elapsed() < Duration::from_secs(15), "a {:?}, b {:?}", a.leader.leading(), b.leader.leading());
167
        tokio::time::sleep(Duration::from_millis(100)).await;
168
    }
169
170
    // The hub leading the scheduler stops; the other takes everything over.
171
    let (gone, left) = if a.leader.leading().contains(&"flow.scheduler") { (&a, &b) } else { (&b, &a) };
172
    gone.leader.resign().await;
173
    let start = Instant::now();
174
    assert!(
175
        until(Duration::from_secs(10), || singletons.iter().all(|n| left.leader.leading().contains(n))).await,
176
        "left {:?}",
177
        left.leader.leading()
178
    );
179
    eprintln!("took over in {:.1} s", start.elapsed().as_secs_f64());
180
    assert!(gone.leader.leading().is_empty(), "{:?}", gone.leader.leading());
181
    assert_eq!(health(left).await["leading"].as_array().unwrap().len(), left.leader.leading().len());
182
}