OXYGENHUB

Coverage / crates/hub/src/runner_updates.rs 97.9%

237 of 242 lines run, 102 of 118 functions · run #507 on main · run missed

1
//! Runner updates (oxygenhub#322, OxygenFlow docs/jetstream.md §8.2):
2
//! which build each runner should run, under the policy the site, its
3
//! owner, its test groups and the runner itself set, the nearest setting
4
//! winning. Off unless some level turns updates on.
5
6
use crate::db::runner_updates::{self as rdb, Group, Policy};
7
use crate::db::runners::Runner;
8
use crate::db::{self, Pool};
9
use flow_core::api::{UpdateOffer, UpdatePolicy};
10
use std::collections::{BTreeSet, HashMap};
11
12
/// The site setting naming the repository whose releases carry runner
13
/// builds (`owner/name`), each an asset `flow-OS-ARCH`.
14
pub const SETTING_REPO: &str = "runner_updates_repo";
15
16
/// A runner's place, 0 to 99, for percentage groups: a stable hash of its
17
/// id, so a group's share is the same machines every rollout, and a
18
/// smaller share sits inside a larger one.
192519×
pub fn bucket(runner_id: i64) -> i64 {
202519×
    let h = blake3::hash(format!("runner {runner_id}").as_bytes());
212519×
    let mut first = [0u8; 8];
222519×
    first.copy_from_slice(&h.as_bytes()[..8]);
232519×
    (u64::from_le_bytes(first) % 100) as i64
242519×
}
25
26
/// Whether a group may hold the runner at all: a site group holds any, an
27
/// owner's group only that owner's runners.
28649×
pub fn in_reach(g: &Group, r: &Runner) -> bool {
29649×
    g.owner.is_empty() || g.owner == r.scope_owner
30649×
}
31
32
/// Every level's settings and the groups, read once.
33
#[derive(Clone, Debug, Default)]
34
pub struct Levels {
35
    pub policies: HashMap<(String, String), Policy>,
36
    pub groups: Vec<Group>,
37
    /// Chosen members, as (group, runner).
38
    pub chosen: BTreeSet<(i64, i64)>,
39
}
40
41
impl Levels {
4262×
    pub async fn load(pool: &Pool) -> sqlx::Result<Levels> {
4384×
        let policies = rdb::policies(pool).await?.into_iter().map(|p| ((p.scope.clone(), p.key.clone()), p)).collect();
4462×
        Ok(Levels { policies, groups: rdb::groups(pool).await?, chosen: rdb::members(pool).await?.into_iter().collect() })
4562×
    }
46
47649×
    pub fn member(&self, g: &Group, r: &Runner) -> bool {
48649×
        in_reach(g, r) && (self.chosen.contains(&(g.id, r.id)) || g.percent.is_some_and(|p| bucket(r.id) < p))
49649×
    }
50
51
    /// The runner's groups, nearest first: its owner's, then the site's.
52629×
    pub fn groups_of(&self, r: &Runner) -> Vec<&Group> {
53629×
        let mut gs: Vec<&Group> = self.groups.iter().filter(|g| self.member(g, r)).collect();
54629×
        gs.sort_by_key(|g| (g.owner.is_empty(), g.name.clone()));
55629×
        gs
56629×
    }
57
58203×
    fn get(&self, scope: &str, key: &str) -> Option<&Policy> {
59203×
        self.policies.get(&(scope.to_string(), key.to_string()))
60203×
    }
61
62
    /// The levels that apply to `r`, nearest first, each with its name.
6372×
    fn chain(&self, r: &Runner) -> Vec<(String, &Policy)> {
6472×
        let mut out = vec![];
6572×
        if let Some(p) = self.get("runner", &r.id.to_string()) {
6620×
            out.push(("runner".to_string(), p));
6752×
        }
6872×
        for g in self.groups_of(r) {
6935×
            if let Some(p) = self.get("group", &g.id.to_string()) {
7027×
                out.push((format!("group {}", g.name), p));
7127×
            }
72
        }
7372×
        if !r.scope_owner.is_empty() {
7424×
            if let Some(p) = self.get("owner", &r.scope_owner) {
7522×
                out.push((format!("owner {}", r.scope_owner), p));
7622×
            }
7748×
        }
7872×
        if let Some(p) = self.get("site", "") {
7938×
            out.push(("site".to_string(), p));
8038×
        }
8172×
        out
8272×
    }
83
84
    /// The policy for `r`: each setting from the nearest level that sets it.
8572×
    pub fn resolve(&self, r: &Runner) -> UpdatePolicy {
8672×
        let chain = self.chain(r);
87288×
        fn pick<T>(chain: &[(String, &Policy)], f: impl Fn(&Policy) -> Option<T>) -> (Option<T>, String) {
88338×
            chain.iter().find_map(|(name, p)| f(p).map(|v| (Some(v), name.clone()))).unwrap_or((None, "default".into()))
89288×
        }
9072×
        let (updates, u_from) = pick(&chain, |p| p.updates);
9183×
        let (channel, c_from) = pick(&chain, |p| p.channel.clone().filter(|c| !c.is_empty()));
9291×
        let (window, w_from) = pick(&chain, |p| Some((p.window_from? as u8, p.window_to? as u8)));
9372×
        let (dry_run, d_from) = pick(&chain, |p| p.dry_run);
94
        UpdatePolicy {
9572×
            updates: updates.unwrap_or(false),
9672×
            channel: channel.unwrap_or_else(|| "stable".into()),
9772×
            window,
9872×
            dry_run: dry_run.unwrap_or(false),
9972×
            from: vec![u_from, c_from, w_from, d_from],
100
        }
10172×
    }
102
}
103
104
/// Whether hour `h` (UTC) falls in `[from, to)`, which may wrap past midnight.
10517×
pub fn in_window((from, to): (u8, u8), h: u8) -> bool {
10617×
    if from <= to {
10714×
        from <= h && h < to
108
    } else {
1093×
        h >= from || h < to
110
    }
11117×
}
112
113
/// Parse `02-05` (hours, UTC) as a window; blank is none.
11415×
pub fn parse_window(s: &str) -> Result<Option<(u8, u8)>, String> {
11515×
    let s = s.trim();
11615×
    if s.is_empty() {
1178×
        return Ok(None);
1187×
    }
1197×
    let (a, b) = s.split_once('-').ok_or("a window is two hours, from-to, e.g. 02-05")?;
12010×
    let hour = |x: &str| x.trim().trim_end_matches(":00").parse::<u8>().ok().filter(|h| *h < 24);
1215×
    match (hour(a), hour(b)) {
1224×
        (Some(a), Some(b)) if a != b => Ok(Some((a, b))),
1232×
        _ => Err("a window is two different hours from 0 to 23, from-to, e.g. 02-05".into()),
124
    }
12515×
}
126
127
/// A channel as written: `stable`, `canary`, or a version.
12811×
pub fn valid_channel(c: &str) -> bool {
12915×
    matches!(c, "stable" | "canary") || (!c.is_empty() && c.split('.').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())))
13011×
}
131
132
/// The version in a runner's or a tag's words: `0.3.1 (build)` → `0.3.1`,
133
/// `v0.3.1` → `0.3.1`.
13455×
pub fn version_of(s: &str) -> String {
13555×
    s.split_whitespace().next().unwrap_or("").trim_start_matches('v').to_string()
13655×
}
137
13816×
fn older(a: &str, b: &str) -> bool {
13996×
    let parts = |v: &str| v.split('.').map(|p| p.parse::<u64>().unwrap_or(0)).collect::<Vec<_>>();
14016×
    parts(a) < parts(b)
14116×
}
142
143
/// The version a runner's report says rolled back: `rolled back from flow
144
/// 0.4.0 to 0.3.1: …` (the runner's words, oxygenflow spec §11.5.3) → `0.4.0`.
1454×
pub fn rolled_back_version(note: &str) -> Option<String> {
1464×
    let v = note.strip_prefix("rolled back from flow ")?.split_whitespace().next()?;
1476×
    v.split('.').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())).then(|| v.to_string())
1484×
}
149
150
/// The release asset a runner on `os` and `arch` runs: `flow-linux-x86_64`,
151
/// `flow-windows-x86_64.exe`.
15218×
pub fn asset_for(os: &str, arch: &str) -> String {
15318×
    format!("flow-{os}-{arch}{}", if os == "windows" { ".exe" } else { "" })
15418×
}
155
156
/// The build `r` should install under `policy` at hour `hour` (UTC), or
157
/// why there is none.
15868×
pub async fn offer(pool: &Pool, r: &Runner, policy: &UpdatePolicy, hour: u8) -> sqlx::Result<(Option<UpdateOffer>, String)> {
15968×
    if !policy.updates {
16018×
        return Ok((None, format!("updates are off ({})", policy.from[0])));
16150×
    }
16250×
    let Some(spec) = db::accounts::setting(pool, SETTING_REPO).await?.filter(|s| !s.trim().is_empty()) else {
16316×
        return Ok((None, "no runner releases repository is set (Admin › Runners › Updates)".into()));
164
    };
16534×
    let Some((owner, name)) = spec.trim().split_once('/') else {
1660×
        return Ok((None, format!("the runner releases repository {spec:?} is not owner/name")));
167
    };
16834×
    let Some(repo) = db::repos::get(pool, owner, name).await? else {
1690×
        return Ok((None, format!("the runner releases repository {spec} does not exist")));
170
    };
17134×
    let releases = db::releases::list(pool, repo.id).await?;
17234×
    let release = match policy.channel.as_str() {
17334×
        "stable" => releases.iter().find(|x| !x.prerelease),
17415×
        "canary" => releases.first(),
17515×
        v => releases.iter().find(|x| version_of(&x.tag) == v),
176
    };
17734×
    let Some(release) = release else {
17815×
        return Ok((None, format!("{spec} has no release for the {} channel", policy.channel)));
179
    };
18019×
    let (want, have) = (version_of(&release.tag), version_of(&r.version));
18119×
    if want == have {
1820×
        return Ok((None, format!("up to date: flow {have} ({} channel)", policy.channel)));
18319×
    }
18419×
    if let Some(h) = rdb::held(pool, &want).await? {
1853×
        return Ok((
1863×
            None,
1873×
            format!("flow {want} is held: it rolled back on {} ({}); lift the hold on Admin › Runners › Updates", h.runner, h.why),
1883×
        ));
18916×
    }
19016×
    let going_back = older(&want, &have);
19116×
    if going_back && matches!(policy.channel.as_str(), "stable" | "canary") {
1920×
        return Ok((None, format!("runs flow {have}, newer than the {} channel's {want}", policy.channel)));
19316×
    }
19416×
    let file = asset_for(&r.os, &r.arch);
19516×
    let assets = db::releases::assets(pool, release.id).await?;
19616×
    let Some(a) = assets.iter().find(|a| a.name == file) else {
1970×
        return Ok((None, format!("{spec} {} has no {file}", release.tag)));
198
    };
19916×
    if let Some(w) = policy.window.filter(|w| !in_window(*w, hour)) {
20011×
        return Ok((None, format!("flow {want} waits for its window, {:02}:00–{:02}:00 UTC", w.0, w.1)));
2015×
    }
2025×
    let note = format!(
203
        "flow {want} ({} channel){}",
204
        policy.channel,
2055×
        if policy.dry_run { ": dry run, check it and say what would happen" } else { "" }
206
    );
2075×
    Ok((
2085×
        Some(UpdateOffer {
2095×
            version: want,
2105×
            tag: release.tag.clone(),
2115×
            asset: a.name.clone(),
2125×
            url: format!("/{owner}/{name}/asset/{}/{}", a.id, a.name),
2135×
            blake3: a.hash.clone(),
2145×
            size: a.size as u64,
2155×
            older: going_back,
2165×
        }),
2175×
        note,
2185×
    ))
21968×
}
220
221
#[cfg(test)]
222
mod tests {
223
    use super::*;
224
225505×
    fn runner(id: i64, owner: &str) -> Runner {
226505×
        Runner { id, scope_owner: owner.into(), version: "0.3.1 (build)".into(), ..Runner::default() }
227505×
    }
228
229
    #[test]
2301×
    fn a_smaller_share_sits_inside_a_larger_one_and_is_about_its_size() {
2311000×
        let ten: Vec<i64> = (1..=1000).filter(|id| bucket(*id) < 10).collect();
2321000×
        let fifty: Vec<i64> = (1..=1000).filter(|id| bucket(*id) < 50).collect();
23396×
        assert!(ten.iter().all(|id| fifty.contains(id)));
2341×
        assert!((60..=140).contains(&ten.len()), "{}", ten.len());
2351×
        assert_eq!(bucket(7), bucket(7), "stable");
2361×
    }
237
238
    #[test]
2391×
    fn the_nearest_level_wins_each_setting() {
2401×
        let mut l = Levels::default();
2414×
        let pol = |scope: &str, key: &str, f: &dyn Fn(&mut Policy)| {
2424×
            let mut p = Policy { scope: scope.into(), key: key.into(), ..Policy::default() };
2434×
            f(&mut p);
2444×
            ((scope.to_string(), key.to_string()), p)
2454×
        };
2461×
        l.policies.extend([
2471×
            pol("site", "", &|p| {
2481×
                p.updates = Some(false);
2491×
                p.channel = Some("stable".into());
2501×
            }),
2511×
            pol("owner", "acme", &|p| p.updates = Some(true)),
2521×
            pol("group", "1", &|p| p.channel = Some("canary".into())),
2531×
            pol("runner", "3", &|p| p.updates = Some(false)),
254
        ]);
2551×
        l.groups.push(Group { id: 1, owner: "acme".into(), name: "canary".into(), percent: None, created_ms: 0 });
2561×
        l.chosen.insert((1, 2));
2571×
        l.chosen.insert((1, 9));
258
2591×
        let site_runner = l.resolve(&runner(1, ""));
2601×
        assert!(!site_runner.updates, "off at the site");
2611×
        assert_eq!(site_runner.from, ["site", "site", "default", "default"]);
262
2631×
        let canary = l.resolve(&runner(2, "acme"));
2641×
        assert!(canary.updates && canary.channel == "canary");
2651×
        assert_eq!(canary.from[..2], ["owner acme".to_string(), "group canary".to_string()]);
266
2671×
        let held = l.resolve(&runner(3, "acme"));
2681×
        assert!(!held.updates, "the runner's own setting wins");
2691×
        assert_eq!(held.channel, "stable");
270
2711×
        assert!(l.groups_of(&runner(9, "other")).is_empty(), "an owner's group holds only its own runners");
2721×
        assert!(!Levels::default().resolve(&runner(4, "")).updates, "off unless chosen");
2731×
    }
274
275
    #[test]
2761×
    fn a_percentage_group_takes_its_share_of_the_runners_in_reach() {
2771×
        let mut l = Levels::default();
2781×
        l.groups.push(Group { id: 5, owner: String::new(), name: "farm 20%".into(), percent: Some(20), created_ms: 0 });
279500×
        let n = (1..=500).filter(|id| !l.groups_of(&runner(*id, "")).is_empty()).count();
2801×
        assert!((60..=140).contains(&n), "{n}");
2811×
    }
282
283
    #[test]
2841×
    fn windows_and_channels_read_as_written() {
2851×
        assert_eq!(parse_window("02-05"), Ok(Some((2, 5))));
2861×
        assert_eq!(parse_window(" 22:00-04:00 "), Ok(Some((22, 4))));
2871×
        assert_eq!(parse_window(""), Ok(None));
2881×
        assert!(parse_window("5").is_err() && parse_window("3-3").is_err() && parse_window("1-24").is_err());
2891×
        assert!(in_window((2, 5), 2) && in_window((2, 5), 4) && !in_window((2, 5), 5));
2901×
        assert!(in_window((22, 4), 23) && in_window((22, 4), 1) && !in_window((22, 4), 12));
2911×
        assert!(valid_channel("stable") && valid_channel("canary") && valid_channel("0.3.4"));
2921×
        assert!(!valid_channel("latest") && !valid_channel("0.3.") && !valid_channel(""));
2931×
        assert_eq!(version_of("v0.3.2"), "0.3.2");
2941×
        assert_eq!(version_of("0.3.1 (local build)"), "0.3.1");
2951×
        assert_eq!(asset_for("linux", "riscv64"), "flow-linux-riscv64");
2961×
        assert_eq!(asset_for("windows", "x86_64"), "flow-windows-x86_64.exe");
2971×
        assert_eq!(
2981×
            rolled_back_version("rolled back from flow 0.4.0 to 0.3.1: 3 starts without reaching the hub").as_deref(),
299
            Some("0.4.0")
300
        );
3011×
        assert_eq!(rolled_back_version("runs flow 0.4.0 (was 0.3.1); proven"), None);
3021×
    }
303
}