OXYGENHUB
◆ v0.9.8 oxygenhub / crates / hub / tests / bus.rs
13.5 KiB · Rust · last changed Carry events between the hub's processes on an event bus 16 days ago source blame history raw
33d98147 Andy McCall 1
//! The event bus (docs/roles.md, #273): a hub hears its own events once;
2
//! on PostgreSQL, two hubs on one database hear each other's: a push made
3
//! through a web process reaches live pages and the webhook worker of
4
//! another process, and a process whose listening connection is ended
5
//! listens again on a new one.
6
7
use oxygenhub::bus::{Event, Topic};
8
use oxygenhub::config::Config;
9
use std::path::Path;
10
use std::time::Duration;
11
12
fn hermetic() {
13
    static ONCE: std::sync::Once = std::sync::Once::new();
14
    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();
15
    ONCE.call_once(|| {
16
        let d = tempfile::tempdir().unwrap();
17
        std::env::set_var("OXY_HOME", d.path());
18
        DIR.set(d).ok();
19
    });
20
}
21
22
fn config(dir: &Path, roles: &[&str]) -> Config {
23
    let mut config = Config::load(dir).unwrap();
24
    config.server.roles = roles.iter().map(|r| r.to_string()).collect();
25
    config.registration.setup_code = "test-setup".into();
26
    config
27
}
28
29
/// The next event on `rx` that `want` picks, within `limit`.
30
async fn heard<T: Clone>(rx: &mut tokio::sync::broadcast::Receiver<T>, limit: Duration, want: impl Fn(&T) -> bool) -> Option<T> {
31
    tokio::time::timeout(limit, async {
32
        loop {
33
            match rx.recv().await {
34
                Ok(e) if want(&e) => return Some(e),
35
                Ok(_) | Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
36
                Err(_) => return None,
37
            }
38
        }
39
    })
40
    .await
41
    .ok()
42
    .flatten()
43
}
44
45
#[tokio::test(flavor = "multi_thread")]
46
async fn a_hub_hears_its_own_events_once() {
47
    hermetic();
48
    let dir = tempfile::tempdir().unwrap();
49
    let app = oxygenhub::build(config(dir.path(), &[])).await.unwrap();
50
    let mut live = app.bus.subscribe(Topic::Live);
51
    let mut deliveries = app.bus.subscribe(Topic::Deliveries);
52
    app.bus.live("repo:1", "push");
53
    app.bus.delivery_waiting();
54
    let second = Duration::from_secs(1);
55
    assert_eq!(heard(&mut live, second, |_| true).await, Some(Event::Live { topic: "repo:1".into(), kind: "push".into() }));
56
    assert_eq!(heard(&mut deliveries, second, |_| true).await, Some(Event::DeliveryWaiting));
57
    // On PostgreSQL the event also went out through the database and came
58
    // back to this process's listener, which skips its own.
59
    tokio::time::sleep(Duration::from_millis(500)).await;
60
    assert!(live.try_recv().is_err(), "heard once");
61
    assert!(deliveries.try_recv().is_err(), "heard once");
62
    assert_eq!(app.bus.live.last_id(), 1);
63
}
64
65
#[cfg(feature = "postgres")]
66
mod postgres {
67
    use super::*;
68
    use axum::body::Body;
69
    use axum::http::{header, Request, StatusCode};
70
    use axum::routing::post;
71
    use http_body_util::BodyExt;
72
    use oxy_core::repo::Repo;
73
    use oxygenhub::Shared;
74
    use std::sync::{Arc, Mutex};
75
    use tower::ServiceExt;
76
77
    fn pg(dir: &Path, schema: &str, roles: &[&str]) -> Config {
78
        let mut c = config(dir, roles);
79
        c.database.schema = schema.to_string();
80
        c
81
    }
82
83
    async fn call(
84
        app: &Shared,
85
        method: &str,
86
        path: &str,
87
        auth: (Option<&str>, Option<&str>),
88
        body: Option<(&str, String)>,
89
    ) -> (StatusCode, String, Option<String>) {
90
        let mut req = Request::builder().method(method).uri(path);
91
        if let Some(b) = auth.0 {
92
            req = req.header(header::AUTHORIZATION, format!("Bearer {b}"));
93
        }
94
        if let Some(c) = auth.1 {
95
            req = req.header(header::COOKIE, format!("oxyhub={c}"));
96
        }
97
        let body = match body {
98
            Some((kind, b)) => {
99
                req = req.header(header::CONTENT_TYPE, kind);
100
                Body::from(b)
101
            }
102
            None => Body::empty(),
103
        };
104
        let res = oxygenhub::router(app.clone()).oneshot(req.body(body).unwrap()).await.unwrap();
105
        let status = res.status();
106
        let cookie = res
107
            .headers()
108
            .get(header::SET_COOKIE)
109
            .and_then(|v| v.to_str().ok())
110
            .and_then(|v| v.split(';').next())
111
            .and_then(|kv| kv.split_once('='))
112
            .map(|(_, v)| v.to_string());
113
        let bytes = res.into_body().collect().await.unwrap().to_bytes();
114
        (status, String::from_utf8_lossy(&bytes).into_owned(), cookie)
115
    }
116
117
    /// A webhook receiver that records the events it gets.
118
    async fn receiver() -> (String, Arc<Mutex<Vec<String>>>) {
119
        let seen: Arc<Mutex<Vec<String>>> = Arc::default();
120
        let s2 = seen.clone();
121
        let router = axum::Router::new().route(
122
            "/hook",
123
            post(move |headers: header::HeaderMap| {
124
                let s2 = s2.clone();
125
                async move {
126
                    let event = headers.get("x-oxyhub-event").and_then(|v| v.to_str().ok()).unwrap_or("").to_string();
127
                    s2.lock().unwrap().push(event);
128
                    StatusCode::OK
129
                }
130
            }),
131
        );
132
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
133
        let port = listener.local_addr().unwrap().port();
134
        tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
135
        (format!("http://127.0.0.1:{port}/hook"), seen)
136
    }
137
138
    /// A push through a web-only hub: a live page's subscriber on a
139
    /// worker-only hub hears it, and that hub's webhook worker, nudged over
140
    /// the bus, delivers its webhook at once rather than on its
141
    /// once-a-minute pass. The web hub hears its own push once.
142
    #[tokio::test(flavor = "multi_thread")]
143
    async fn a_push_through_one_hub_reaches_the_other_hubs_pages_and_worker() {
144
        hermetic();
145
        let (da, dbb) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
146
        let schema = format!("hub_bus_{}", da.path().file_name().unwrap().to_string_lossy());
147
        let mut ca = pg(da.path(), &schema, &["web"]);
148
        ca.auth.push_tokens = vec!["site".into()];
149
        ca.registration.mode = "open".into();
150
        ca.outbound.allow = vec!["http://127.0.0.1".into()];
151
        let tcp = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
152
        let base = format!("http://127.0.0.1:{}", tcp.local_addr().unwrap().port());
153
        ca.server.base_url = base.clone();
154
        let a = oxygenhub::build(ca).await.unwrap();
155
        let router = oxygenhub::router(a.clone());
156
        tokio::spawn(async move { axum::serve(tcp, router).await.unwrap() });
157
        let mut cb = pg(dbb.path(), &schema, &["worker"]);
158
        cb.outbound.allow = vec!["http://127.0.0.1".into()];
159
        let b = oxygenhub::build(cb).await.unwrap();
160
161
        // An account, an admin token, a repository and its webhook, on A.
162
        let form = "application/x-www-form-urlencoded";
163
        let (_, _, cookie) = call(
164
            &a,
165
            "POST",
166
            "/register",
167
            (None, None),
168
            Some((form, "username=andy&email=a%40example.com&password=correct-horse-battery&setup=test-setup".into())),
169
        )
170
        .await;
171
        let andy = cookie.expect("signed in");
172
        let (_, page, _) = call(&a, "GET", "/settings/tokens", (None, Some(&andy)), None).await;
173
        let at = page.find("name=\"_csrf\" value=\"").unwrap() + "name=\"_csrf\" value=\"".len();
174
        let csrf = page[at..].split('"').next().unwrap().to_string();
175
        let (_, page, _) =
176
            call(&a, "POST", "/settings/tokens", (None, Some(&andy)), Some((form, format!("_csrf={csrf}&name=t&scopes=repo%3Aadmin"))))
177
                .await;
178
        let token = format!("oxyhub_{}", page.split("oxyhub_").nth(1).unwrap().split('<').next().unwrap().trim());
179
        let json = "application/json";
180
        let (st, body, _) = call(
181
            &a,
182
            "POST",
183
            "/api/v1/repos",
184
            (Some(&token), None),
185
            Some((json, r#"{"owner":"andy","name":"demo","visibility":"public"}"#.into())),
186
        )
187
        .await;
188
        assert_eq!(st, StatusCode::CREATED, "{body}");
189
        let (hook, seen) = receiver().await;
190
        let (st, body, _) = call(
191
            &a,
192
            "POST",
193
            "/api/v1/repos/andy/demo/hooks",
194
            (Some(&token), None),
195
            Some((json, format!(r#"{{"url":"{hook}","secret":"s","events":"push"}}"#))),
196
        )
197
        .await;
198
        assert_eq!(st, StatusCode::CREATED, "{body}");
199
        let repo = oxygenhub::db::repos::get(&a.db, "andy", "demo").await.unwrap().unwrap();
200
        let topic = oxygenhub::events::topic::repo(repo.id);
201
202
        // B's worker leads the webhook worker and has made its first pass.
203
        let start = std::time::Instant::now();
204
        while !b.leader.leading().contains(&"hooks.worker") {
205
            assert!(start.elapsed() < Duration::from_secs(10), "{:?}", b.leader.leading());
206
            tokio::time::sleep(Duration::from_millis(100)).await;
207
        }
208
        tokio::time::sleep(Duration::from_millis(500)).await;
209
210
        let mut b_live = b.bus.live.subscribe();
211
        let work = tempfile::tempdir().unwrap();
212
        let mut client = Repo::init(work.path()).unwrap();
213
        client.config.set("remote.origin.url", &format!("{base}/andy/demo")).unwrap();
214
        client.config.set("remote.origin.token", "site").unwrap();
215
        client.save_config().unwrap();
216
        std::fs::write(work.path().join("a.txt"), b"a\n").unwrap();
217
        client.snapshot("t").unwrap();
218
        client.commit("base", &[]).unwrap();
219
        assert!(client.push("origin", false).unwrap().iter().all(|r| r.ok));
220
221
        let push = heard(&mut b_live, Duration::from_secs(10), |e| e.topic == topic).await.expect("B hears A's push");
222
        assert_eq!(push.kind, "push");
223
        let start = std::time::Instant::now();
224
        while seen.lock().unwrap().is_empty() {
225
            assert!(start.elapsed() < Duration::from_secs(10), "B's worker was not nudged");
226
            tokio::time::sleep(Duration::from_millis(100)).await;
227
        }
228
        assert_eq!(*seen.lock().unwrap(), ["push"]);
229
        eprintln!("delivered by the other process in {:.1} s", start.elapsed().as_secs_f64());
230
231
        // A heard its own push once, not again from the database.
232
        tokio::time::sleep(Duration::from_millis(500)).await;
233
        let oxygenhub::events::Replay::Events(on_a) = a.bus.live.since(0, std::slice::from_ref(&topic)) else { panic!() };
234
        assert_eq!(on_a.iter().filter(|e| e.kind == "push").count(), 1, "{on_a:?}");
235
    }
236
237
    /// Proposal events and resolver rooms' choices cross over; a process
238
    /// whose listening connection is ended listens again, looks for
239
    /// deliveries it may have missed, and hears the other again.
240
    #[tokio::test(flavor = "multi_thread")]
241
    async fn two_hubs_hear_each_other_and_listen_again_after_losing_the_connection() {
242
        hermetic();
243
        let (da, dbb) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
244
        let schema = format!("hub_bus_{}", da.path().file_name().unwrap().to_string_lossy());
245
        let a = oxygenhub::build(pg(da.path(), &schema, &[])).await.unwrap();
246
        let b = oxygenhub::build(pg(dbb.path(), &schema, &[])).await.unwrap();
247
        // A hub in another schema of the same database hears neither.
248
        let dc = tempfile::tempdir().unwrap();
249
        let c = oxygenhub::build(pg(dc.path(), &format!("{schema}_other"), &[])).await.unwrap();
250
        let mut c_live = c.bus.subscribe(Topic::Live);
251
252
        let mut b_proposals = b.bus.subscribe(Topic::Proposals);
253
        let check = hub_api::ProposalEvent { number: 0, kind: "check".into(), check: None };
254
        a.bus.proposal(7, check.clone());
255
        let got = heard(&mut b_proposals, Duration::from_secs(5), |_| true).await;
256
        assert_eq!(got, Some(Event::Proposal { repo: 7, event: check }));
257
258
        // A choice made on A reaches the same room's viewers on B.
259
        let key = oxygenhub::live::RoomKey { repo_id: 1, bookmark: "main".into(), path: "a.txt".into() };
260
        let (mut b_room, _) = b.rooms.join(key.clone(), "tab-b", "bea");
261
        let choice = oxygenhub::live::Choice { choice: "side2".into(), text: String::new(), by: "andy".into() };
262
        let event = oxygenhub::live::RoomEvent::Choice { region: "1".into(), choice: choice.clone(), client: "tab-a".into() };
263
        a.bus.publish(Event::Room { room: key.clone(), event });
264
        let got = heard(&mut b_room, Duration::from_secs(5), |e| matches!(e, oxygenhub::live::LiveEvent::Choice { .. })).await;
265
        assert_eq!(got, Some(oxygenhub::live::LiveEvent::Choice { region: "1".into(), choice, client: "tab-a".into() }));
266
267
        // B's listening connection is ended, as when the network drops.
268
        let pid = b.bus.listener_pid().expect("b listens");
269
        let mut b_deliveries = b.bus.subscribe(Topic::Deliveries);
270
        let ended: bool = sqlx::query_scalar("SELECT pg_terminate_backend($1)").bind(pid).fetch_one(&a.db).await.unwrap();
271
        assert!(ended);
272
        let start = std::time::Instant::now();
273
        while b.bus.listener_pid().is_none_or(|p| p == pid) {
274
            assert!(start.elapsed() < Duration::from_secs(15), "b did not listen again");
275
            tokio::time::sleep(Duration::from_millis(100)).await;
276
        }
277
        eprintln!("listening again in {:.1} s", start.elapsed().as_secs_f64());
278
        // It looks now for deliveries a missed nudge would have announced.
279
        assert_eq!(heard(&mut b_deliveries, Duration::from_secs(5), |_| true).await, Some(Event::DeliveryWaiting));
280
        let mut b_live = b.bus.subscribe(Topic::Live);
281
        a.bus.live("repo:9", "push");
282
        let got = heard(&mut b_live, Duration::from_secs(5), |_| true).await;
283
        assert_eq!(got, Some(Event::Live { topic: "repo:9".into(), kind: "push".into() }));
284
        tokio::time::sleep(Duration::from_millis(300)).await;
285
        assert!(c_live.try_recv().is_err(), "another schema's hub hears nothing");
286
    }
287
}