OXYGENHUB
◆ v0.9.7 oxygenhub / crates / hub / src / runner_updates.rs
13.0 KiB · Rust · last changed Runner updates: a build that rolls back on one runner is held 10 days ago source blame history raw
8501fd58 Andy McCall ↶ 1
//! Runner updates (oxygenhub#322, OxygenFlow docs/jetstream.md §8.2):
2
//! which build each runner should run, under the policy the site, its
3
//! owner, its test groups and the runner itself set, the nearest setting
4
//! winning. Off unless some level turns updates on.
5
6
use crate::db::runner_updates::{self as rdb, Group, Policy};
7
use crate::db::runners::Runner;
8
use crate::db::{self, Pool};
9
use flow_core::api::{UpdateOffer, UpdatePolicy};
10
use std::collections::{BTreeSet, HashMap};
11
12
/// The site setting naming the repository whose releases carry runner
13
/// builds (`owner/name`), each an asset `flow-OS-ARCH`.
14
pub const SETTING_REPO: &str = "runner_updates_repo";
15
16
/// A runner's place, 0 to 99, for percentage groups: a stable hash of its
17
/// id, so a group's share is the same machines every rollout, and a
18
/// smaller share sits inside a larger one.
19
pub fn bucket(runner_id: i64) -> i64 {
20
    let h = blake3::hash(format!("runner {runner_id}").as_bytes());
21
    let mut first = [0u8; 8];
22
    first.copy_from_slice(&h.as_bytes()[..8]);
23
    (u64::from_le_bytes(first) % 100) as i64
24
}
25
26
/// Whether a group may hold the runner at all: a site group holds any, an
27
/// owner's group only that owner's runners.
28
pub fn in_reach(g: &Group, r: &Runner) -> bool {
29
    g.owner.is_empty() || g.owner == r.scope_owner
30
}
31
32
/// Every level's settings and the groups, read once.
33
#[derive(Clone, Debug, Default)]
34
pub struct Levels {
35
    pub policies: HashMap<(String, String), Policy>,
36
    pub groups: Vec<Group>,
37
    /// Chosen members, as (group, runner).
38
    pub chosen: BTreeSet<(i64, i64)>,
39
}
40
41
impl Levels {
42
    pub async fn load(pool: &Pool) -> sqlx::Result<Levels> {
43
        let policies = rdb::policies(pool).await?.into_iter().map(|p| ((p.scope.clone(), p.key.clone()), p)).collect();
44
        Ok(Levels { policies, groups: rdb::groups(pool).await?, chosen: rdb::members(pool).await?.into_iter().collect() })
45
    }
46
47
    pub fn member(&self, g: &Group, r: &Runner) -> bool {
48
        in_reach(g, r) && (self.chosen.contains(&(g.id, r.id)) || g.percent.is_some_and(|p| bucket(r.id) < p))
49
    }
50
51
    /// The runner's groups, nearest first: its owner's, then the site's.
52
    pub fn groups_of(&self, r: &Runner) -> Vec<&Group> {
53
        let mut gs: Vec<&Group> = self.groups.iter().filter(|g| self.member(g, r)).collect();
54
        gs.sort_by_key(|g| (g.owner.is_empty(), g.name.clone()));
55
        gs
56
    }
57
58
    fn get(&self, scope: &str, key: &str) -> Option<&Policy> {
59
        self.policies.get(&(scope.to_string(), key.to_string()))
60
    }
61
62
    /// The levels that apply to `r`, nearest first, each with its name.
63
    fn chain(&self, r: &Runner) -> Vec<(String, &Policy)> {
64
        let mut out = vec![];
65
        if let Some(p) = self.get("runner", &r.id.to_string()) {
66
            out.push(("runner".to_string(), p));
67
        }
68
        for g in self.groups_of(r) {
69
            if let Some(p) = self.get("group", &g.id.to_string()) {
70
                out.push((format!("group {}", g.name), p));
71
            }
72
        }
73
        if !r.scope_owner.is_empty() {
74
            if let Some(p) = self.get("owner", &r.scope_owner) {
75
                out.push((format!("owner {}", r.scope_owner), p));
76
            }
77
        }
78
        if let Some(p) = self.get("site", "") {
79
            out.push(("site".to_string(), p));
80
        }
81
        out
82
    }
83
84
    /// The policy for `r`: each setting from the nearest level that sets it.
85
    pub fn resolve(&self, r: &Runner) -> UpdatePolicy {
86
        let chain = self.chain(r);
87
        fn pick<T>(chain: &[(String, &Policy)], f: impl Fn(&Policy) -> Option<T>) -> (Option<T>, String) {
88
            chain.iter().find_map(|(name, p)| f(p).map(|v| (Some(v), name.clone()))).unwrap_or((None, "default".into()))
89
        }
90
        let (updates, u_from) = pick(&chain, |p| p.updates);
91
        let (channel, c_from) = pick(&chain, |p| p.channel.clone().filter(|c| !c.is_empty()));
92
        let (window, w_from) = pick(&chain, |p| Some((p.window_from? as u8, p.window_to? as u8)));
93
        let (dry_run, d_from) = pick(&chain, |p| p.dry_run);
94
        UpdatePolicy {
95
            updates: updates.unwrap_or(false),
96
            channel: channel.unwrap_or_else(|| "stable".into()),
97
            window,
98
            dry_run: dry_run.unwrap_or(false),
99
            from: vec![u_from, c_from, w_from, d_from],
100
        }
101
    }
102
}
103
104
/// Whether hour `h` (UTC) falls in `[from, to)`, which may wrap past midnight.
105
pub fn in_window((from, to): (u8, u8), h: u8) -> bool {
106
    if from <= to {
107
        from <= h && h < to
108
    } else {
109
        h >= from || h < to
110
    }
111
}
112
113
/// Parse `02-05` (hours, UTC) as a window; blank is none.
114
pub fn parse_window(s: &str) -> Result<Option<(u8, u8)>, String> {
115
    let s = s.trim();
116
    if s.is_empty() {
117
        return Ok(None);
118
    }
119
    let (a, b) = s.split_once('-').ok_or("a window is two hours, from-to, e.g. 02-05")?;
120
    let hour = |x: &str| x.trim().trim_end_matches(":00").parse::<u8>().ok().filter(|h| *h < 24);
121
    match (hour(a), hour(b)) {
122
        (Some(a), Some(b)) if a != b => Ok(Some((a, b))),
123
        _ => Err("a window is two different hours from 0 to 23, from-to, e.g. 02-05".into()),
124
    }
125
}
126
127
/// A channel as written: `stable`, `canary`, or a version.
128
pub fn valid_channel(c: &str) -> bool {
129
    matches!(c, "stable" | "canary") || (!c.is_empty() && c.split('.').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())))
130
}
131
132
/// The version in a runner's or a tag's words: `0.3.1 (build)` → `0.3.1`,
133
/// `v0.3.1` → `0.3.1`.
134
pub fn version_of(s: &str) -> String {
135
    s.split_whitespace().next().unwrap_or("").trim_start_matches('v').to_string()
136
}
137
138
fn older(a: &str, b: &str) -> bool {
139
    let parts = |v: &str| v.split('.').map(|p| p.parse::<u64>().unwrap_or(0)).collect::<Vec<_>>();
140
    parts(a) < parts(b)
141
}
142
faf5246c Andy McCall ↶ 143
/// The version a runner's report says rolled back: `rolled back from flow
144
/// 0.4.0 to 0.3.1: …` (the runner's words, oxygenflow spec §11.5.3) → `0.4.0`.
145
pub fn rolled_back_version(note: &str) -> Option<String> {
146
    let v = note.strip_prefix("rolled back from flow ")?.split_whitespace().next()?;
147
    v.split('.').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())).then(|| v.to_string())
148
}
149
b6dfcb61 Andy McCall ↶ 150
/// The release asset a runner on `os` and `arch` runs: `flow-linux-x86_64`,
151
/// `flow-windows-x86_64.exe`.
152
pub fn asset_for(os: &str, arch: &str) -> String {
153
    format!("flow-{os}-{arch}{}", if os == "windows" { ".exe" } else { "" })
154
}
155
8501fd58 Andy McCall ↶ 156
/// The build `r` should install under `policy` at hour `hour` (UTC), or
157
/// why there is none.
158
pub async fn offer(pool: &Pool, r: &Runner, policy: &UpdatePolicy, hour: u8) -> sqlx::Result<(Option<UpdateOffer>, String)> {
159
    if !policy.updates {
160
        return Ok((None, format!("updates are off ({})", policy.from[0])));
161
    }
162
    let Some(spec) = db::accounts::setting(pool, SETTING_REPO).await?.filter(|s| !s.trim().is_empty()) else {
163
        return Ok((None, "no runner releases repository is set (Admin › Runners › Updates)".into()));
164
    };
165
    let Some((owner, name)) = spec.trim().split_once('/') else {
166
        return Ok((None, format!("the runner releases repository {spec:?} is not owner/name")));
167
    };
168
    let Some(repo) = db::repos::get(pool, owner, name).await? else {
169
        return Ok((None, format!("the runner releases repository {spec} does not exist")));
170
    };
171
    let releases = db::releases::list(pool, repo.id).await?;
172
    let release = match policy.channel.as_str() {
173
        "stable" => releases.iter().find(|x| !x.prerelease),
174
        "canary" => releases.first(),
175
        v => releases.iter().find(|x| version_of(&x.tag) == v),
176
    };
177
    let Some(release) = release else {
178
        return Ok((None, format!("{spec} has no release for the {} channel", policy.channel)));
179
    };
180
    let (want, have) = (version_of(&release.tag), version_of(&r.version));
181
    if want == have {
182
        return Ok((None, format!("up to date: flow {have} ({} channel)", policy.channel)));
faf5246c Andy McCall ↶ 183
    }
184
    if let Some(h) = rdb::held(pool, &want).await? {
185
        return Ok((
186
            None,
187
            format!("flow {want} is held: it rolled back on {} ({}); lift the hold on Admin › Runners › Updates", h.runner, h.why),
188
        ));
8501fd58 Andy McCall ↶ 189
    }
190
    let going_back = older(&want, &have);
191
    if going_back && matches!(policy.channel.as_str(), "stable" | "canary") {
192
        return Ok((None, format!("runs flow {have}, newer than the {} channel's {want}", policy.channel)));
193
    }
b6dfcb61 Andy McCall ↶ 194
    let file = asset_for(&r.os, &r.arch);
8501fd58 Andy McCall ↶ 195
    let assets = db::releases::assets(pool, release.id).await?;
196
    let Some(a) = assets.iter().find(|a| a.name == file) else {
197
        return Ok((None, format!("{spec} {} has no {file}", release.tag)));
198
    };
199
    if let Some(w) = policy.window.filter(|w| !in_window(*w, hour)) {
200
        return Ok((None, format!("flow {want} waits for its window, {:02}:00–{:02}:00 UTC", w.0, w.1)));
201
    }
202
    let note = format!(
203
        "flow {want} ({} channel){}",
204
        policy.channel,
205
        if policy.dry_run { ": dry run, check it and say what would happen" } else { "" }
206
    );
207
    Ok((
208
        Some(UpdateOffer {
209
            version: want,
210
            tag: release.tag.clone(),
211
            asset: a.name.clone(),
212
            url: format!("/{owner}/{name}/asset/{}/{}", a.id, a.name),
213
            blake3: a.hash.clone(),
214
            size: a.size as u64,
215
            older: going_back,
216
        }),
217
        note,
218
    ))
219
}
220
221
#[cfg(test)]
222
mod tests {
223
    use super::*;
224
225
    fn runner(id: i64, owner: &str) -> Runner {
226
        Runner { id, scope_owner: owner.into(), version: "0.3.1 (build)".into(), ..Runner::default() }
227
    }
228
229
    #[test]
230
    fn a_smaller_share_sits_inside_a_larger_one_and_is_about_its_size() {
231
        let ten: Vec<i64> = (1..=1000).filter(|id| bucket(*id) < 10).collect();
232
        let fifty: Vec<i64> = (1..=1000).filter(|id| bucket(*id) < 50).collect();
233
        assert!(ten.iter().all(|id| fifty.contains(id)));
234
        assert!((60..=140).contains(&ten.len()), "{}", ten.len());
235
        assert_eq!(bucket(7), bucket(7), "stable");
236
    }
237
238
    #[test]
239
    fn the_nearest_level_wins_each_setting() {
240
        let mut l = Levels::default();
241
        let pol = |scope: &str, key: &str, f: &dyn Fn(&mut Policy)| {
242
            let mut p = Policy { scope: scope.into(), key: key.into(), ..Policy::default() };
243
            f(&mut p);
244
            ((scope.to_string(), key.to_string()), p)
245
        };
246
        l.policies.extend([
247
            pol("site", "", &|p| {
248
                p.updates = Some(false);
249
                p.channel = Some("stable".into());
250
            }),
251
            pol("owner", "acme", &|p| p.updates = Some(true)),
252
            pol("group", "1", &|p| p.channel = Some("canary".into())),
253
            pol("runner", "3", &|p| p.updates = Some(false)),
254
        ]);
255
        l.groups.push(Group { id: 1, owner: "acme".into(), name: "canary".into(), percent: None, created_ms: 0 });
256
        l.chosen.insert((1, 2));
257
        l.chosen.insert((1, 9));
258
259
        let site_runner = l.resolve(&runner(1, ""));
260
        assert!(!site_runner.updates, "off at the site");
261
        assert_eq!(site_runner.from, ["site", "site", "default", "default"]);
262
263
        let canary = l.resolve(&runner(2, "acme"));
264
        assert!(canary.updates && canary.channel == "canary");
265
        assert_eq!(canary.from[..2], ["owner acme".to_string(), "group canary".to_string()]);
266
267
        let held = l.resolve(&runner(3, "acme"));
268
        assert!(!held.updates, "the runner's own setting wins");
269
        assert_eq!(held.channel, "stable");
270
271
        assert!(l.groups_of(&runner(9, "other")).is_empty(), "an owner's group holds only its own runners");
272
        assert!(!Levels::default().resolve(&runner(4, "")).updates, "off unless chosen");
273
    }
274
275
    #[test]
276
    fn a_percentage_group_takes_its_share_of_the_runners_in_reach() {
277
        let mut l = Levels::default();
278
        l.groups.push(Group { id: 5, owner: String::new(), name: "farm 20%".into(), percent: Some(20), created_ms: 0 });
279
        let n = (1..=500).filter(|id| !l.groups_of(&runner(*id, "")).is_empty()).count();
280
        assert!((60..=140).contains(&n), "{n}");
281
    }
282
283
    #[test]
284
    fn windows_and_channels_read_as_written() {
285
        assert_eq!(parse_window("02-05"), Ok(Some((2, 5))));
286
        assert_eq!(parse_window(" 22:00-04:00 "), Ok(Some((22, 4))));
287
        assert_eq!(parse_window(""), Ok(None));
288
        assert!(parse_window("5").is_err() && parse_window("3-3").is_err() && parse_window("1-24").is_err());
289
        assert!(in_window((2, 5), 2) && in_window((2, 5), 4) && !in_window((2, 5), 5));
290
        assert!(in_window((22, 4), 23) && in_window((22, 4), 1) && !in_window((22, 4), 12));
291
        assert!(valid_channel("stable") && valid_channel("canary") && valid_channel("0.3.4"));
292
        assert!(!valid_channel("latest") && !valid_channel("0.3.") && !valid_channel(""));
293
        assert_eq!(version_of("v0.3.2"), "0.3.2");
294
        assert_eq!(version_of("0.3.1 (local build)"), "0.3.1");
b6dfcb61 Andy McCall ↶ 295
        assert_eq!(asset_for("linux", "riscv64"), "flow-linux-riscv64");
296
        assert_eq!(asset_for("windows", "x86_64"), "flow-windows-x86_64.exe");
faf5246c Andy McCall ↶ 297
        assert_eq!(
298
            rolled_back_version("rolled back from flow 0.4.0 to 0.3.1: 3 starts without reaching the hub").as_deref(),
299
            Some("0.4.0")
300
        );
301
        assert_eq!(rolled_back_version("runs flow 0.4.0 (was 0.3.1); proven"), None);
8501fd58 Andy McCall ↶ 302
    }
303
}