OXYGENHUB
# a46d15e72101 oxygenhub / crates / hub / src / swarm.rs
6.5 KiB · Rust · last changed Jetstream: the swarm's tracker and peer tickets 25 days ago source blame history raw
1
//! The swarm's tracker (oxygenhub#144, oxygenflow docs/jetstream.md §8.1):
2
//! runners that share images tell the hub where peers reach them with each
3
//! claim; a runner fetching an image announces it and is given the peers
4
//! in its site that hold or are fetching that image, and a ticket, signed
5
//! with the hub key, that those peers accept.
6
//!
7
//! What the tracker knows lives in memory: runners repeat their address
8
//! with every claim and fetchers announce every `INTERVAL_S`, so a
9
//! restarted hub knows the swarm again within seconds.
10
11
use crate::api::ApiError;
12
use crate::db;
13
use crate::Shared;
14
use axum::extract::State;
15
use axum::http::{HeaderMap, StatusCode};
16
use axum::Json;
17
use flow_core::api::{Announce, Announced, SwarmAddr, SwarmPeer};
18
use jet::swarm::{Ticket, TICKET_LIFE_MS};
19
use std::collections::HashMap;
20
use std::sync::Mutex;
21
22
/// How often a fetcher announces again.
23
pub const INTERVAL_S: u64 = 15;
24
/// A runner whose address is older than this is not offered as a peer.
25
const ADDR_FRESH_MS: i64 = 2 * 60 * 1000;
26
/// A fetcher that has not announced for this long is no longer fetching.
27
const FETCHING_FRESH_MS: i64 = 3 * INTERVAL_S as i64 * 1000;
28
/// A ticket is made again once it has less than this to run.
29
const TICKET_RENEW_MS: u64 = 5 * 60 * 1000;
30
31
#[derive(Default)]
32
pub struct Tracker {
33
    /// Runner id to its swarm address and when it last said so.
34
    addrs: Mutex<HashMap<i64, (SwarmAddr, i64)>>,
35
    /// Image to the runners fetching it, with when each last announced.
36
    fetching: Mutex<HashMap<String, HashMap<i64, i64>>>,
37
    /// Runner id to its current ticket and when that expires.
38
    tickets: Mutex<HashMap<i64, (String, u64)>>,
39
}
40
41
impl Tracker {
42
    /// What a runner's claim said: where peers reach it, or that it is not
43
    /// in a swarm.
44
    pub fn claimed(&self, runner: i64, swarm: Option<&SwarmAddr>) {
45
        let mut addrs = self.addrs.lock().unwrap_or_else(|p| p.into_inner());
46
        match swarm.filter(|s| plausible(&s.addr)) {
47
            Some(s) => {
48
                addrs.insert(runner, (s.clone(), db::now_ms()));
49
            }
50
            None => {
51
                addrs.remove(&runner);
52
            }
53
        }
54
    }
55
56
    /// Where peers reach `runner`, if it is in a swarm now.
57
    pub fn addr(&self, runner: i64) -> Option<SwarmAddr> {
58
        let addrs = self.addrs.lock().unwrap_or_else(|p| p.into_inner());
59
        addrs.get(&runner).filter(|(_, seen)| db::now_ms() - seen < ADDR_FRESH_MS).map(|(a, _)| a.clone())
60
    }
61
62
    fn fetch(&self, image: &str, runner: i64, state: &str) {
63
        let mut fetching = self.fetching.lock().unwrap_or_else(|p| p.into_inner());
64
        let now = db::now_ms();
65
        for who in fetching.values_mut() {
66
            who.retain(|_, seen| now - *seen < FETCHING_FRESH_MS);
67
        }
68
        fetching.retain(|_, who| !who.is_empty());
69
        if state == "fetching" {
70
            fetching.entry(image.to_string()).or_default().insert(runner, now);
71
        } else if let Some(who) = fetching.get_mut(image) {
72
            who.remove(&runner);
73
        }
74
    }
75
76
    fn fetchers(&self, image: &str) -> Vec<i64> {
77
        let fetching = self.fetching.lock().unwrap_or_else(|p| p.into_inner());
78
        fetching.get(image).map(|who| who.keys().copied().collect()).unwrap_or_default()
79
    }
80
}
81
82
/// An address a peer could dial: `host:port`, nothing else.
83
fn plausible(addr: &str) -> bool {
84
    addr.len() <= 255 && addr.rsplit_once(':').is_some_and(|(host, port)| !host.is_empty() && port.parse::<u16>().is_ok_and(|p| p > 0))
85
}
86
87
/// `POST /flow/swarm/announce`: a runner starting, continuing or ending a
88
/// fetch. The peers it may ask, in its site, and its ticket.
89
pub async fn announce(State(app): State<Shared>, headers: HeaderMap, Json(a): Json<Announce>) -> Result<Json<Announced>, ApiError> {
90
    let me = crate::flow::runner_auth(&app, &headers).await?;
91
    if !jet::images::is_hash(&a.image) || !matches!(a.state.as_str(), "fetching" | "done" | "stopped") {
92
        return Err(ApiError(StatusCode::BAD_REQUEST, "announce an image's hash with state fetching, done or stopped".into()));
93
    }
94
    let image = a.image.to_ascii_lowercase();
95
    let Some(mine) = app.swarm.addr(me.id) else {
96
        return Err(ApiError(
97
            StatusCode::CONFLICT,
98
            "this runner has not said where peers reach it: give it a [runner.swarm] section and let it claim".into(),
99
        ));
100
    };
101
    app.swarm.fetch(&image, me.id, &a.state);
102
    let fetchers = app.swarm.fetchers(&image);
103
    let mut peers = Vec::new();
104
    if a.state == "fetching" {
105
        for r in db::runners::list(&app.db).await? {
106
            if r.id == me.id || r.state == "disabled" || !r.online() {
107
                continue;
108
            }
109
            let Some(addr) = app.swarm.addr(r.id).filter(|s| s.site == mine.site) else { continue };
110
            let complete = r.image_list().is_some_and(|held| held.iter().any(|h| h.eq_ignore_ascii_case(&image)));
111
            if complete || fetchers.contains(&r.id) {
112
                peers.push(SwarmPeer { runner: r.id, name: r.name.clone(), addr: addr.addr, complete });
113
            }
114
        }
115
    }
116
    let ticket = ticket(&app, &me, &mine.site)?;
117
    Ok(Json(Announced { ticket, peers, interval_s: INTERVAL_S }))
118
}
119
120
/// The runner's ticket, made again when it nears its end or its site
121
/// changed.
122
fn ticket(app: &Shared, me: &db::runners::Runner, site: &str) -> Result<String, ApiError> {
123
    let now = db::now_ms() as u64;
124
    let mut tickets = app.swarm.tickets.lock().unwrap_or_else(|p| p.into_inner());
125
    if let Some((t, until)) = tickets.get(&me.id) {
126
        if *until > now + TICKET_RENEW_MS && Ticket::open(t, &app.hub_key.identity, now).is_ok_and(|t| t.site == site) {
127
            return Ok(t.clone());
128
        }
129
    }
130
    let (key, secret) = crate::keys::load_secret(&app.config.keys_dir())
131
        .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("the hub key: {e}")))?;
132
    let expires_ms = now + TICKET_LIFE_MS;
133
    let t = Ticket { hub: app.hub_key.key.to_hex(), runner: me.id, name: me.name.clone(), site: site.to_string(), expires_ms }
134
        .seal(&key, &secret);
135
    tickets.insert(me.id, (t.clone(), expires_ms));
136
    Ok(t)
137
}
138
139
#[cfg(test)]
140
mod tests {
141
    use super::*;
142
143
    #[test]
144
    fn only_host_and_port_addresses_are_kept() {
145
        assert!(plausible("10.0.0.5:7401"));
146
        assert!(plausible("[::1]:7401"));
147
        assert!(plausible("runner.lan:7401"));
148
        assert!(!plausible("10.0.0.5"));
149
        assert!(!plausible(":7401"));
150
        assert!(!plausible("10.0.0.5:0"));
151
        assert!(!plausible("10.0.0.5:99999"));
152
    }
153
}