OXYGENHUB
# 80642fde09cc oxygenhub / crates / hub / src / shutdown.rs
3.8 KiB · Rust · last changed A stop no longer waits on blocking work past the drain 19 days ago source blame history raw
1
//! Stopping without cutting anyone off (oxygenhub#233). On SIGTERM (a
2
//! `systemctl restart`) or Ctrl-C the hub stops taking connections, ends
3
//! its event streams (browsers reconnect by themselves), and gives the
4
//! requests in flight up to `DRAIN` to finish. Given a listening socket by
5
//! systemd (socket activation, docs/deploy.md), it serves that rather than
6
//! binding: the socket outlives the restart, so a request that arrives
7
//! between the old process and the new one waits in its queue instead of
8
//! being refused.
9
10
use std::sync::OnceLock;
11
use tokio::sync::watch;
12
13
/// How long the requests in flight get to finish.
14
pub const DRAIN: std::time::Duration = std::time::Duration::from_secs(30);
15
16
/// How long, after the drain, work still running on the runtime's
17
/// blocking threads (a slow request's repository work, a bucket call) may
18
/// hold the process. Dropping the runtime would wait for it however long
19
/// it takes: on 2026-10-17 a restart stood a minute past "stopped", until
20
/// systemd killed it.
21
pub const EXIT_GRACE: std::time::Duration = std::time::Duration::from_secs(1);
22
23
/// End the runtime once `serve` has returned, cutting off what still runs.
24
pub fn finish(rt: tokio::runtime::Runtime) {
25
    rt.shutdown_timeout(EXIT_GRACE);
26
}
27
28
fn sender() -> &'static watch::Sender<bool> {
29
    static STOP: OnceLock<watch::Sender<bool>> = OnceLock::new();
30
    STOP.get_or_init(|| watch::channel(false).0)
31
}
32
33
/// Begin stopping.
34
pub fn begin() {
35
    sender().send_replace(true);
36
}
37
38
pub fn is_stopping() -> bool {
39
    *sender().borrow()
40
}
41
42
/// Done when the hub begins stopping: for event streams to end on, and for
43
/// the servers' graceful shutdown.
44
pub async fn stopping() {
45
    let mut rx = sender().subscribe();
46
    let _ = rx.wait_for(|s| *s).await;
47
}
48
49
/// Wait for SIGTERM or Ctrl-C, then begin stopping. Installed before the
50
/// servers start, so a signal never kills the process outright.
51
pub fn on_signal() -> anyhow::Result<impl std::future::Future<Output = ()>> {
52
    #[cfg(unix)]
53
    let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
54
    Ok(async move {
55
        #[cfg(unix)]
56
        tokio::select! {
57
            _ = term.recv() => tracing::info!("SIGTERM: stopping (requests in flight may take {} s)", DRAIN.as_secs()),
58
            _ = tokio::signal::ctrl_c() => tracing::info!("stopping"),
59
        }
60
        #[cfg(not(unix))]
61
        {
62
            let _ = tokio::signal::ctrl_c().await;
63
            tracing::info!("stopping");
64
        }
65
        begin();
66
    })
67
}
68
69
/// The listening socket systemd passed (LISTEN_FDS, first descriptor 3),
70
/// when it passed one to this process.
71
pub fn systemd_listener() -> Option<std::net::TcpListener> {
72
    #[cfg(unix)]
73
    {
74
        use std::os::fd::FromRawFd;
75
        let pid: u32 = std::env::var("LISTEN_PID").ok()?.parse().ok()?;
76
        let fds: u32 = std::env::var("LISTEN_FDS").ok()?.parse().ok()?;
77
        if pid != std::process::id() || fds < 1 {
78
            return None;
79
        }
80
        // SAFETY: systemd hands this process descriptor 3 as an open,
81
        // listening socket (sd_listen_fds(3)); nothing else owns it.
82
        let listener = unsafe { std::net::TcpListener::from_raw_fd(3) };
83
        listener.set_nonblocking(true).ok()?;
84
        Some(listener)
85
    }
86
    #[cfg(not(unix))]
87
    None
88
}
89
90
#[cfg(test)]
91
mod tests {
92
    /// Blocking work still running when the hub stops does not hold the
93
    /// process: the runtime ends within its grace.
94
    #[test]
95
    fn finishing_does_not_wait_for_blocking_work() {
96
        let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap();
97
        rt.spawn_blocking(|| std::thread::sleep(std::time::Duration::from_secs(30)));
98
        let started = std::time::Instant::now();
99
        super::finish(rt);
100
        assert!(started.elapsed() < super::EXIT_GRACE * 3, "{:?}", started.elapsed());
101
    }
102
}