OXYGENHUB
◆ v0.9.24 oxygenhub / crates / hub / src / swarm.rs
7.6 KiB · Rust · last changed Peer-held images: the hub keeps runners' manifests of images it lacks and places their jobs where a peer in the site holds them (closes #342; pins OxygenFlow #94) 9 days ago source blame history raw
a46d15e7 Andy McCall ↶ 1
//! The swarm's tracker (oxygenhub#144, oxygenflow docs/jetstream.md §8.1):
2
//! runners that share images tell the hub where peers reach them with each
3
//! claim; a runner fetching an image announces it and is given the peers
6adcd68b Andy McCall ↶ 4
//! in its site that hold or are fetching that image, each with the
5
//! fingerprint of its TLS certificate, and a ticket, signed with the hub
6
//! key, that those peers accept.
a46d15e7 Andy McCall ↶ 7
//!
8
//! What the tracker knows lives in memory: runners repeat their address
9
//! with every claim and fetchers announce every `INTERVAL_S`, so a
10
//! restarted hub knows the swarm again within seconds.
11
12
use crate::api::ApiError;
13
use crate::db;
14
use crate::Shared;
15
use axum::extract::State;
16
use axum::http::{HeaderMap, StatusCode};
17
use axum::Json;
18
use flow_core::api::{Announce, Announced, SwarmAddr, SwarmPeer};
19
use jet::swarm::{Ticket, TICKET_LIFE_MS};
20
use std::collections::HashMap;
21
use std::sync::Mutex;
22
23
/// How often a fetcher announces again.
24
pub const INTERVAL_S: u64 = 15;
25
/// A runner whose address is older than this is not offered as a peer.
26
const ADDR_FRESH_MS: i64 = 2 * 60 * 1000;
27
/// A fetcher that has not announced for this long is no longer fetching.
28
const FETCHING_FRESH_MS: i64 = 3 * INTERVAL_S as i64 * 1000;
29
/// A ticket is made again once it has less than this to run.
30
const TICKET_RENEW_MS: u64 = 5 * 60 * 1000;
31
32
#[derive(Default)]
33
pub struct Tracker {
34
    /// Runner id to its swarm address and when it last said so.
35
    addrs: Mutex<HashMap<i64, (SwarmAddr, i64)>>,
36
    /// Image to the runners fetching it, with when each last announced.
37
    fetching: Mutex<HashMap<String, HashMap<i64, i64>>>,
38
    /// Runner id to its current ticket and when that expires.
39
    tickets: Mutex<HashMap<i64, (String, u64)>>,
40
}
41
42
impl Tracker {
43
    /// What a runner's claim said: where peers reach it, or that it is not
44
    /// in a swarm.
45
    pub fn claimed(&self, runner: i64, swarm: Option<&SwarmAddr>) {
46
        let mut addrs = self.addrs.lock().unwrap_or_else(|p| p.into_inner());
6adcd68b Andy McCall ↶ 47
        // A peer is trusted by its certificate, so one without is not in
48
        // the swarm (an older flow).
49
        match swarm.filter(|s| plausible(&s.addr) && jet::images::is_hash(&s.cert)) {
a46d15e7 Andy McCall ↶ 50
            Some(s) => {
51
                addrs.insert(runner, (s.clone(), db::now_ms()));
52
            }
53
            None => {
54
                addrs.remove(&runner);
55
            }
56
        }
57
    }
58
59
    /// Where peers reach `runner`, if it is in a swarm now.
60
    pub fn addr(&self, runner: i64) -> Option<SwarmAddr> {
61
        let addrs = self.addrs.lock().unwrap_or_else(|p| p.into_inner());
62
        addrs.get(&runner).filter(|(_, seen)| db::now_ms() - seen < ADDR_FRESH_MS).map(|(a, _)| a.clone())
63
    }
64
65
    fn fetch(&self, image: &str, runner: i64, state: &str) {
66
        let mut fetching = self.fetching.lock().unwrap_or_else(|p| p.into_inner());
67
        let now = db::now_ms();
68
        for who in fetching.values_mut() {
69
            who.retain(|_, seen| now - *seen < FETCHING_FRESH_MS);
70
        }
71
        fetching.retain(|_, who| !who.is_empty());
72
        if state == "fetching" {
73
            fetching.entry(image.to_string()).or_default().insert(runner, now);
74
        } else if let Some(who) = fetching.get_mut(image) {
75
            who.remove(&runner);
76
        }
77
    }
78
6ca0b971 Andy McCall ↶ 79
    /// Who is fetching which image now, by image.
80
    pub fn fetching_now(&self) -> HashMap<String, Vec<i64>> {
81
        let fetching = self.fetching.lock().unwrap_or_else(|p| p.into_inner());
82
        let now = db::now_ms();
83
        fetching
84
            .iter()
85
            .map(|(image, who)| {
86
                (image.clone(), who.iter().filter(|(_, seen)| now - **seen < FETCHING_FRESH_MS).map(|(r, _)| *r).collect::<Vec<_>>())
87
            })
88
            .filter(|(_, who)| !who.is_empty())
89
            .collect()
90
    }
91
a46d15e7 Andy McCall ↶ 92
    fn fetchers(&self, image: &str) -> Vec<i64> {
93
        let fetching = self.fetching.lock().unwrap_or_else(|p| p.into_inner());
94
        fetching.get(image).map(|who| who.keys().copied().collect()).unwrap_or_default()
95
    }
96
}
97
98
/// An address a peer could dial: `host:port`, nothing else.
99
fn plausible(addr: &str) -> bool {
100
    addr.len() <= 255 && addr.rsplit_once(':').is_some_and(|(host, port)| !host.is_empty() && port.parse::<u16>().is_ok_and(|p| p > 0))
101
}
102
103
/// `POST /flow/swarm/announce`: a runner starting, continuing or ending a
104
/// fetch. The peers it may ask, in its site, and its ticket.
105
pub async fn announce(State(app): State<Shared>, headers: HeaderMap, Json(a): Json<Announce>) -> Result<Json<Announced>, ApiError> {
106
    let me = crate::flow::runner_auth(&app, &headers).await?;
107
    if !jet::images::is_hash(&a.image) || !matches!(a.state.as_str(), "fetching" | "done" | "stopped") {
108
        return Err(ApiError(StatusCode::BAD_REQUEST, "announce an image's hash with state fetching, done or stopped".into()));
109
    }
110
    let image = a.image.to_ascii_lowercase();
111
    let Some(mine) = app.swarm.addr(me.id) else {
112
        return Err(ApiError(
113
            StatusCode::CONFLICT,
114
            "this runner has not said where peers reach it: give it a [runner.swarm] section and let it claim".into(),
115
        ));
116
    };
117
    app.swarm.fetch(&image, me.id, &a.state);
118
    let fetchers = app.swarm.fetchers(&image);
119
    let mut peers = Vec::new();
120
    if a.state == "fetching" {
121
        for r in db::runners::list(&app.db).await? {
122
            if r.id == me.id || r.state == "disabled" || !r.online() {
123
                continue;
124
            }
125
            let Some(addr) = app.swarm.addr(r.id).filter(|s| s.site == mine.site) else { continue };
126
            let complete = r.image_list().is_some_and(|held| held.iter().any(|h| h.eq_ignore_ascii_case(&image)));
127
            if complete || fetchers.contains(&r.id) {
6adcd68b Andy McCall ↶ 128
                peers.push(SwarmPeer { runner: r.id, name: r.name.clone(), addr: addr.addr, cert: addr.cert, complete });
a46d15e7 Andy McCall ↶ 129
            }
130
        }
131
    }
132
    let ticket = ticket(&app, &me, &mine.site)?;
951aed50 Andy McCall ↶ 133
    // An image only runners hold (oxygenflow#94): the hub has its manifest
134
    // and none of its chunks, so the fetcher asks peers alone.
135
    let seed = Some(
136
        sqlx::query_scalar::<_, String>("SELECT source FROM jet_images WHERE hash = $1")
137
            .bind(&image)
138
            .fetch_optional(&app.db)
139
            .await?
140
            .is_none_or(|s| s != "peer"),
141
    );
142
    Ok(Json(Announced { ticket, peers, interval_s: INTERVAL_S, seed }))
a46d15e7 Andy McCall ↶ 143
}
144
145
/// The runner's ticket, made again when it nears its end or its site
146
/// changed.
147
fn ticket(app: &Shared, me: &db::runners::Runner, site: &str) -> Result<String, ApiError> {
148
    let now = db::now_ms() as u64;
149
    let mut tickets = app.swarm.tickets.lock().unwrap_or_else(|p| p.into_inner());
150
    if let Some((t, until)) = tickets.get(&me.id) {
151
        if *until > now + TICKET_RENEW_MS && Ticket::open(t, &app.hub_key.identity, now).is_ok_and(|t| t.site == site) {
152
            return Ok(t.clone());
153
        }
154
    }
155
    let (key, secret) = crate::keys::load_secret(&app.config.keys_dir())
156
        .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("the hub key: {e}")))?;
157
    let expires_ms = now + TICKET_LIFE_MS;
158
    let t = Ticket { hub: app.hub_key.key.to_hex(), runner: me.id, name: me.name.clone(), site: site.to_string(), expires_ms }
159
        .seal(&key, &secret);
160
    tickets.insert(me.id, (t.clone(), expires_ms));
161
    Ok(t)
162
}
163
164
#[cfg(test)]
165
mod tests {
166
    use super::*;
167
168
    #[test]
169
    fn only_host_and_port_addresses_are_kept() {
170
        assert!(plausible("10.0.0.5:7401"));
171
        assert!(plausible("[::1]:7401"));
172
        assert!(plausible("runner.lan:7401"));
173
        assert!(!plausible("10.0.0.5"));
174
        assert!(!plausible(":7401"));
175
        assert!(!plausible("10.0.0.5:0"));
176
        assert!(!plausible("10.0.0.5:99999"));
177
    }
178
}