OXYGENHUB
# 356183b71fab oxygenhub / crates / hub / tests / roles.rs
8.8 KiB · Rust · last changed One auto-land and one Diffusion bump at a time across hub processes (closes #275) 15 days ago source blame history raw
1
//! Roles (docs/roles.md, #272): a web-only hub starts none of the worker's
2
//! background tasks, a worker-only hub serves no pages, and on PostgreSQL
3
//! two hubs on one database run each singleton task once between them.
4
5
use axum::body::Body;
6
use axum::http::{header, Request, StatusCode};
7
use http_body_util::BodyExt;
8
use oxygenhub::config::Config;
9
use oxygenhub::Shared;
10
use std::path::Path;
11
use std::time::{Duration, Instant};
12
use tower::ServiceExt;
13
14
/// The worker's singleton tasks that run for as long as the process does.
15
const WORKER_SINGLETONS: [&str; 8] = [
16
    "backups.scheduler",
17
    "diffusion.reconciler",
18
    "flow.placement",
19
    "flow.reaper",
20
    "flow.scheduler",
21
    "hooks.worker",
22
    "maintenance.nightly",
23
    "secrets.history_scan",
24
];
25
26
fn hermetic() {
27
    static ONCE: std::sync::Once = std::sync::Once::new();
28
    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();
29
    ONCE.call_once(|| {
30
        let d = tempfile::tempdir().unwrap();
31
        std::env::set_var("OXY_HOME", d.path());
32
        DIR.set(d).ok();
33
    });
34
}
35
36
fn config(dir: &Path, roles: &[&str]) -> Config {
37
    let mut config = Config::load(dir).unwrap();
38
    config.server.roles = roles.iter().map(|r| r.to_string()).collect();
39
    config.registration.setup_code = "test-setup".into();
40
    config
41
}
42
43
async fn get(app: &Shared, path: &str, token: Option<&str>) -> (StatusCode, String) {
44
    let mut req = Request::get(path);
45
    if let Some(t) = token {
46
        req = req.header(header::AUTHORIZATION, format!("Bearer {t}"));
47
    }
48
    let res = oxygenhub::router(app.clone()).oneshot(req.body(Body::empty()).unwrap()).await.unwrap();
49
    let status = res.status();
50
    let body = res.into_body().collect().await.unwrap().to_bytes();
51
    (status, String::from_utf8_lossy(&body).into_owned())
52
}
53
54
async fn health(app: &Shared) -> serde_json::Value {
55
    let (status, body) = get(app, "/health", None).await;
56
    assert_eq!(status, StatusCode::OK, "{body}");
57
    serde_json::from_str(&body).unwrap()
58
}
59
60
/// Wait up to `limit` for `ok`, checking every 100 ms.
61
async fn until(limit: Duration, mut ok: impl FnMut() -> bool) -> bool {
62
    let start = Instant::now();
63
    while start.elapsed() < limit {
64
        if ok() {
65
            return true;
66
        }
67
        tokio::time::sleep(Duration::from_millis(100)).await;
68
    }
69
    ok()
70
}
71
72
#[tokio::test(flavor = "multi_thread")]
73
async fn a_web_only_hub_starts_none_of_the_workers_tasks() {
74
    hermetic();
75
    let dir = tempfile::tempdir().unwrap();
76
    let app = oxygenhub::build(config(dir.path(), &["web"])).await.unwrap();
77
    // Only the transparency log's checkpointer, which goes with the log's
78
    // appends in the web role.
79
    assert_eq!(app.leader.tasks(), ["tlog.checkpointer"]);
80
    assert!(until(Duration::from_secs(5), || app.leader.leading() == ["tlog.checkpointer"]).await, "{:?}", app.leader.leading());
81
    let h = health(&app).await;
82
    assert_eq!(h["roles"], serde_json::json!(["web"]));
83
    assert_eq!(h["leading"], serde_json::json!(["tlog.checkpointer"]));
84
    // It serves pages.
85
    let (status, body) = get(&app, "/explore", None).await;
86
    assert_eq!(status, StatusCode::OK, "{body}");
87
}
88
89
#[tokio::test(flavor = "multi_thread")]
90
async fn a_worker_only_hub_serves_its_health_and_metrics_and_no_pages() {
91
    hermetic();
92
    let dir = tempfile::tempdir().unwrap();
93
    let mut cfg = config(dir.path(), &["worker"]);
94
    cfg.server.metrics_token = "m".into();
95
    let app = oxygenhub::build(cfg).await.unwrap();
96
    let mut tasks = WORKER_SINGLETONS.to_vec();
97
    tasks.extend(["diffusion.backfill", "images.learn_archs"]);
98
    tasks.sort();
99
    assert_eq!(app.leader.tasks(), tasks, "every worker task, and not the log's checkpointer");
100
    assert!(
101
        until(Duration::from_secs(5), || WORKER_SINGLETONS.iter().all(|n| app.leader.leading().contains(n))).await,
102
        "{:?}",
103
        app.leader.leading()
104
    );
105
    let h = health(&app).await;
106
    assert_eq!(h["roles"], serde_json::json!(["worker"]));
107
    assert!(h["leading"].as_array().unwrap().iter().any(|n| n == "flow.scheduler"), "{h}");
108
    for path in ["/", "/explore", "/login", "/api/v1/version", "/andy/demo", "/log/checkpoint"] {
109
        assert_eq!(get(&app, path, None).await.0, StatusCode::NOT_FOUND, "{path} is the web role's");
110
    }
111
    let (status, body) = get(&app, "/metrics", Some("m")).await;
112
    assert_eq!(status, StatusCode::OK, "{body}");
113
    // It reads the transparency log but never writes it.
114
    assert!(app.tlog.append(&[]).is_err());
115
}
116
117
/// oxygenhub#275: a named lock is held by one holder at a time across the
118
/// database's sessions (on SQLite, the process); other names are free.
119
#[tokio::test(flavor = "multi_thread")]
120
async fn an_exclusive_lock_has_one_holder_at_a_time() {
121
    hermetic();
122
    let dir = tempfile::tempdir().unwrap();
123
    let app = oxygenhub::build(config(dir.path(), &[])).await.unwrap();
124
    let held = oxygenhub::leader::exclusive(&app.db, "autoland:7").await.unwrap();
125
    let (tx, mut rx) = tokio::sync::oneshot::channel();
126
    let db = app.db.clone();
127
    tokio::spawn(async move {
128
        let g = oxygenhub::leader::exclusive(&db, "autoland:7").await.unwrap();
129
        let _ = tx.send(());
130
        drop(g);
131
    });
132
    tokio::time::sleep(Duration::from_millis(300)).await;
133
    assert!(rx.try_recv().is_err(), "the second holder waits");
134
    let other = tokio::time::timeout(Duration::from_secs(2), oxygenhub::leader::exclusive(&app.db, "autoland:8")).await;
135
    assert!(other.is_ok(), "another name is free");
136
    drop(other);
137
    drop(held);
138
    assert!(tokio::time::timeout(Duration::from_secs(5), rx).await.is_ok(), "and takes it once it is let go");
139
}
140
141
#[tokio::test(flavor = "multi_thread")]
142
async fn a_hub_takes_every_role_by_default() {
143
    hermetic();
144
    let dir = tempfile::tempdir().unwrap();
145
    let app = oxygenhub::build(config(dir.path(), &[])).await.unwrap();
146
    assert_eq!(app.leader.tasks().len(), 11, "{:?}", app.leader.tasks());
147
    assert_eq!(health(&app).await["roles"], serde_json::json!(["web", "worker"]));
148
    assert_eq!(get(&app, "/explore", None).await.0, StatusCode::OK);
149
    let mut bad = config(dir.path(), &["web", "cron"]);
150
    bad.server.roles.push(String::new());
151
    let e = oxygenhub::build(bad).await.err().expect("an unknown role is refused").to_string();
152
    assert!(e.contains("unknown role \"cron\""), "{e}");
153
}
154
155
/// Two hubs on one PostgreSQL database (one schema): exactly one leads
156
/// each singleton; when its connection drops it stops them, and when it
157
/// stops the other takes every one over within seconds.
158
#[cfg(feature = "postgres")]
159
#[tokio::test(flavor = "multi_thread")]
160
async fn two_hubs_on_one_database_run_each_singleton_once_and_fail_over() {
161
    hermetic();
162
    let (da, dbb) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
163
    let schema = format!("hub_roles_{}", da.path().file_name().unwrap().to_string_lossy());
164
    let pg = |dir: &Path| {
165
        let mut c = config(dir, &[]);
166
        c.database.schema = schema.clone();
167
        c
168
    };
169
    let a = oxygenhub::build(pg(da.path())).await.unwrap();
170
    let b = oxygenhub::build(pg(dbb.path())).await.unwrap();
171
    let mut singletons = WORKER_SINGLETONS.to_vec();
172
    singletons.push("tlog.checkpointer");
173
    let once = |a: &Shared, b: &Shared| {
174
        let (la, lb) = (a.leader.leading(), b.leader.leading());
175
        singletons.iter().all(|n| la.contains(n) != lb.contains(n))
176
    };
177
    assert!(until(Duration::from_secs(10), || once(&a, &b)).await, "a {:?}, b {:?}", a.leader.leading(), b.leader.leading());
178
179
    // A's connection is ended, as when the network drops: its locks free,
180
    // it stops what it led, and between them each runs once again.
181
    let pid = a.leader.session_pid().await.expect("a has a session");
182
    let ended: bool = sqlx::query_scalar("SELECT pg_terminate_backend($1)").bind(pid).fetch_one(&b.db).await.unwrap();
183
    assert!(ended);
184
    let start = Instant::now();
185
    loop {
186
        let reconnected = a.leader.session_pid().await.is_some_and(|p| p != pid);
187
        if reconnected && once(&a, &b) {
188
            break;
189
        }
190
        assert!(start.elapsed() < Duration::from_secs(15), "a {:?}, b {:?}", a.leader.leading(), b.leader.leading());
191
        tokio::time::sleep(Duration::from_millis(100)).await;
192
    }
193
194
    // The hub leading the scheduler stops; the other takes everything over.
195
    let (gone, left) = if a.leader.leading().contains(&"flow.scheduler") { (&a, &b) } else { (&b, &a) };
196
    gone.leader.resign().await;
197
    let start = Instant::now();
198
    assert!(
199
        until(Duration::from_secs(10), || singletons.iter().all(|n| left.leader.leading().contains(n))).await,
200
        "left {:?}",
201
        left.leader.leading()
202
    );
203
    eprintln!("took over in {:.1} s", start.elapsed().as_secs_f64());
204
    assert!(gone.leader.leading().is_empty(), "{:?}", gone.leader.leading());
205
    assert_eq!(health(left).await["leading"].as_array().unwrap().len(), left.leader.leading().len());
206
}