OXYGENHUB
# fac6853c4e46 oxygenhub / crates / hub / src / runner_updates.rs
21.7 KiB · Rust · last changed Runner updates in waves: a later wave takes a version once the waves before it proved it; an error stops it (refs andymccall/oxygenflow#84) 8 days ago source blame history raw
1 
//! Runner updates (oxygenhub#322, OxygenFlow docs/jetstream.md §8.2):
2 
//! which build each runner should run, under the policy the site, its
3 
//! owner, its test groups and the runner itself set, the nearest setting
4 
//! winning. Off unless some level turns updates on.
5
6 
use crate::db::runner_updates::{self as rdb, Group, Policy, Soak};
7 
use crate::db::runners::Runner;
8 
use crate::db::{self, Pool};
9 
use flow_core::api::{UpdateOffer, UpdatePolicy};
10 
use std::collections::{BTreeSet, HashMap};
11
12 
/// The site setting naming the repository whose releases carry runner
13 
/// builds (`owner/name`), each an asset `flow-OS-ARCH`.
14 
pub const SETTING_REPO: &str = "runner_updates_repo";
15 
/// The site settings for proving a version in a wave (oxygenflow#84): the
16 
/// jobs each runner of the wave that took it must finish, or the hours
17 
/// since its first job there after which one job is enough.
18 
pub const SETTING_SOAK_JOBS: &str = "runner_updates_soak_jobs";
19 
pub const SETTING_SOAK_HOURS: &str = "runner_updates_soak_hours";
20 
pub const SOAK_JOBS: i64 = 5;
21 
pub const SOAK_HOURS: i64 = 24;
22
23 
/// A runner's place, 0 to 99, for percentage groups: a stable hash of its
24 
/// id, so a group's share is the same machines every rollout, and a
25 
/// smaller share sits inside a larger one.
26 
pub fn bucket(runner_id: i64) -> i64 {
27 
    let h = blake3::hash(format!("runner {runner_id}").as_bytes());
28 
    let mut first = [0u8; 8];
29 
    first.copy_from_slice(&h.as_bytes()[..8]);
30 
    (u64::from_le_bytes(first) % 100) as i64
31 
}
32
33 
/// Whether a group may hold the runner at all: a site group holds any, an
34 
/// owner's group only that owner's runners.
35 
pub fn in_reach(g: &Group, r: &Runner) -> bool {
36 
    g.owner.is_empty() || g.owner == r.scope_owner
37 
}
38
39 
/// Every level's settings and the groups, read once.
40 
#[derive(Clone, Debug, Default)]
41 
pub struct Levels {
42 
    pub policies: HashMap<(String, String), Policy>,
43 
    pub groups: Vec<Group>,
44 
    /// Chosen members, as (group, runner).
45 
    pub chosen: BTreeSet<(i64, i64)>,
46 
}
47
48 
impl Levels {
49 
    pub async fn load(pool: &Pool) -> sqlx::Result<Levels> {
50 
        let policies = rdb::policies(pool).await?.into_iter().map(|p| ((p.scope.clone(), p.key.clone()), p)).collect();
51 
        Ok(Levels { policies, groups: rdb::groups(pool).await?, chosen: rdb::members(pool).await?.into_iter().collect() })
52 
    }
53
54 
    pub fn member(&self, g: &Group, r: &Runner) -> bool {
55 
        in_reach(g, r) && (self.chosen.contains(&(g.id, r.id)) || g.percent.is_some_and(|p| bucket(r.id) < p))
56 
    }
57
58 
    /// The runner's groups, nearest first: its owner's, then the site's.
59 
    pub fn groups_of(&self, r: &Runner) -> Vec<&Group> {
60 
        let mut gs: Vec<&Group> = self.groups.iter().filter(|g| self.member(g, r)).collect();
61 
        gs.sort_by_key(|g| (g.owner.is_empty(), g.name.clone()));
62 
        gs
63 
    }
64
65 
    fn get(&self, scope: &str, key: &str) -> Option<&Policy> {
66 
        self.policies.get(&(scope.to_string(), key.to_string()))
67 
    }
68
69 
    /// The wave `r` takes a version in: the first of its groups' waves;
70 
    /// `None` after every wave.
71 
    pub fn wave_of(&self, r: &Runner) -> Option<i64> {
72 
        self.groups_of(r).iter().filter_map(|g| g.wave).min()
73 
    }
74
75 
    /// The waves before `r`'s, in order, each with its runners among `all`:
76 
    /// the waves of the groups that may hold `r` (the site's, and its
77 
    /// owner's), a runner counted in its own first wave.
78 
    pub fn waves_before<'a>(&self, r: &Runner, all: &'a [Runner]) -> Vec<(i64, Vec<&'a Runner>)> {
79 
        let mine = self.wave_of(r);
80 
        let mut waves: Vec<i64> =
81 
            self.groups.iter().filter(|g| in_reach(g, r)).filter_map(|g| g.wave).filter(|w| mine.is_none_or(|m| *w < m)).collect();
82 
        waves.sort();
83 
        waves.dedup();
84 
        waves
85 
            .into_iter()
86 
            .map(|w| {
87 
                let members = all
88 
                    .iter()
89 
                    .filter(|o| self.wave_of(o) == Some(w) && self.groups_of(o).iter().any(|g| g.wave == Some(w) && in_reach(g, r)))
90 
                    .collect();
91 
                (w, members)
92 
            })
93 
            .collect()
94 
    }
95
96 
    /// The levels that apply to `r`, nearest first, each with its name.
97 
    fn chain(&self, r: &Runner) -> Vec<(String, &Policy)> {
98 
        let mut out = vec![];
99 
        if let Some(p) = self.get("runner", &r.id.to_string()) {
100 
            out.push(("runner".to_string(), p));
101 
        }
102 
        for g in self.groups_of(r) {
103 
            if let Some(p) = self.get("group", &g.id.to_string()) {
104 
                out.push((format!("group {}", g.name), p));
105 
            }
106 
        }
107 
        if !r.scope_owner.is_empty() {
108 
            if let Some(p) = self.get("owner", &r.scope_owner) {
109 
                out.push((format!("owner {}", r.scope_owner), p));
110 
            }
111 
        }
112 
        if let Some(p) = self.get("site", "") {
113 
            out.push(("site".to_string(), p));
114 
        }
115 
        out
116 
    }
117
118 
    /// The policy for `r`: each setting from the nearest level that sets it.
119 
    pub fn resolve(&self, r: &Runner) -> UpdatePolicy {
120 
        let chain = self.chain(r);
121 
        fn pick<T>(chain: &[(String, &Policy)], f: impl Fn(&Policy) -> Option<T>) -> (Option<T>, String) {
122 
            chain.iter().find_map(|(name, p)| f(p).map(|v| (Some(v), name.clone()))).unwrap_or((None, "default".into()))
123 
        }
124 
        let (updates, u_from) = pick(&chain, |p| p.updates);
125 
        let (channel, c_from) = pick(&chain, |p| p.channel.clone().filter(|c| !c.is_empty()));
126 
        let (window, w_from) = pick(&chain, |p| Some((p.window_from? as u8, p.window_to? as u8)));
127 
        let (dry_run, d_from) = pick(&chain, |p| p.dry_run);
128 
        UpdatePolicy {
129 
            updates: updates.unwrap_or(false),
130 
            channel: channel.unwrap_or_else(|| "stable".into()),
131 
            window,
132 
            dry_run: dry_run.unwrap_or(false),
133 
            from: vec![u_from, c_from, w_from, d_from],
134 
        }
135 
    }
136 
}
137
138 
/// Whether hour `h` (UTC) falls in `[from, to)`, which may wrap past midnight.
139 
pub fn in_window((from, to): (u8, u8), h: u8) -> bool {
140 
    if from <= to {
141 
        from <= h && h < to
142 
    } else {
143 
        h >= from || h < to
144 
    }
145 
}
146
147 
/// Parse `02-05` (hours, UTC) as a window; blank is none.
148 
pub fn parse_window(s: &str) -> Result<Option<(u8, u8)>, String> {
149 
    let s = s.trim();
150 
    if s.is_empty() {
151 
        return Ok(None);
152 
    }
153 
    let (a, b) = s.split_once('-').ok_or("a window is two hours, from-to, e.g. 02-05")?;
154 
    let hour = |x: &str| x.trim().trim_end_matches(":00").parse::<u8>().ok().filter(|h| *h < 24);
155 
    match (hour(a), hour(b)) {
156 
        (Some(a), Some(b)) if a != b => Ok(Some((a, b))),
157 
        _ => Err("a window is two different hours from 0 to 23, from-to, e.g. 02-05".into()),
158 
    }
159 
}
160
161 
/// A channel as written: `stable`, `canary`, or a version.
162 
pub fn valid_channel(c: &str) -> bool {
163 
    matches!(c, "stable" | "canary") || (!c.is_empty() && c.split('.').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())))
164 
}
165
166 
/// The version in a runner's or a tag's words: `0.3.1 (build)` → `0.3.1`,
167 
/// `v0.3.1` → `0.3.1`.
168 
pub fn version_of(s: &str) -> String {
169 
    s.split_whitespace().next().unwrap_or("").trim_start_matches('v').to_string()
170 
}
171
172 
fn older(a: &str, b: &str) -> bool {
173 
    let parts = |v: &str| v.split('.').map(|p| p.parse::<u64>().unwrap_or(0)).collect::<Vec<_>>();
174 
    parts(a) < parts(b)
175 
}
176
177 
/// The version a runner's report says rolled back: `rolled back from flow
178 
/// 0.4.0 to 0.3.1: …` (the runner's words, oxygenflow spec §11.5.3) → `0.4.0`.
179 
pub fn rolled_back_version(note: &str) -> Option<String> {
180 
    let v = note.strip_prefix("rolled back from flow ")?.split_whitespace().next()?;
181 
    v.split('.').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())).then(|| v.to_string())
182 
}
183
184 
/// The release asset a runner on `os` and `arch` runs: `flow-linux-x86_64`,
185 
/// `flow-windows-x86_64.exe`.
186 
pub fn asset_for(os: &str, arch: &str) -> String {
187 
    format!("flow-{os}-{arch}{}", if os == "windows" { ".exe" } else { "" })
188 
}
189
190 
/// Where a wave stands with a version (oxygenflow#84).
191 
#[derive(Clone, Debug, PartialEq, Eq)]
192 
pub enum WaveState {
193 
    /// Every runner of the wave that took it has finished its soak jobs on
194 
    /// it, or the soak time has passed since its first job there; or the
195 
    /// wave has no runners.
196 
    Proved,
197 
    /// Not yet: `on` of `runners` run it, with `jobs` jobs between them.
198 
    Soaking { runners: usize, on: usize, jobs: i64 },
199 
    /// A job on it ended in error on this runner: the version stops here.
200 
    Stopped { runner_id: i64 },
201 
}
202
203 
/// Where a wave of `members` stands with a version, from each runner's
204 
/// [`Soak`] on it, at `now` (ms).
205 
pub fn wave_state(members: &[i64], soaks: &[Soak], soak_jobs: i64, soak_ms: i64, now: i64) -> WaveState {
206 
    let mine: Vec<&Soak> = soaks.iter().filter(|s| members.contains(&s.runner_id) && s.jobs > 0).collect();
207 
    if let Some(s) = mine.iter().find(|s| s.errors > 0) {
208 
        return WaveState::Stopped { runner_id: s.runner_id };
209 
    }
210 
    if members.is_empty() {
211 
        return WaveState::Proved;
212 
    }
213 
    let jobs = mine.iter().map(|s| s.jobs).sum();
214 
    let soaked = mine.iter().filter_map(|s| s.first_ms).min().is_some_and(|first| now - first >= soak_ms);
215 
    if !mine.is_empty() && (soaked || mine.iter().all(|s| s.jobs >= soak_jobs)) {
216 
        WaveState::Proved
217 
    } else {
218 
        WaveState::Soaking { runners: members.len(), on: mine.len(), jobs }
219 
    }
220 
}
221
222 
/// What the waves before `r`'s say about `version`: `Ok(None)` go ahead;
223 
/// `Ok(Some(why))` wait; `Err((runner, why))` it stopped on that runner.
224 
async fn wave_gate(
225 
    pool: &Pool,
226 
    levels: &Levels,
227 
    r: &Runner,
228 
    version: &str,
229 
    now: i64,
230 
) -> sqlx::Result<Result<Option<String>, (String, String)>> {
231 
    let all = db::runners::list(pool).await?;
232 
    let waves = levels.waves_before(r, &all);
233 
    if waves.is_empty() {
234 
        return Ok(Ok(None));
235 
    }
236 
    let setting = |v: Option<String>, d: i64| v.and_then(|s| s.trim().parse::<i64>().ok()).filter(|n| *n > 0).unwrap_or(d);
237 
    let soak_jobs = setting(db::accounts::setting(pool, SETTING_SOAK_JOBS).await?, SOAK_JOBS);
238 
    let soak_hours = setting(db::accounts::setting(pool, SETTING_SOAK_HOURS).await?, SOAK_HOURS);
239 
    let soaks = rdb::soaks(pool, version).await?;
240 
    for (w, members) in waves {
241 
        let ids: Vec<i64> = members.iter().map(|m| m.id).collect();
242 
        match wave_state(&ids, &soaks, soak_jobs, soak_hours * 3_600_000, now) {
243 
            WaveState::Proved => continue,
244 
            WaveState::Stopped { runner_id } => {
245 
                let name = members.iter().find(|m| m.id == runner_id).map(|m| m.name.clone()).unwrap_or_default();
246 
                let job = rdb::first_error(pool, version, runner_id).await?.unwrap_or_else(|| "a job".into());
247 
                return Ok(Err((name.clone(), format!("wave {w} stopped it on {name}: {job} ended in error"))));
248 
            }
249 
            WaveState::Soaking { runners, on, jobs } => {
250 
                return Ok(Ok(Some(format!(
251 
                    "flow {version} waits for wave {w} to prove it: {on} of {runners} runner{} on it, {jobs} job{} (each needs {soak_jobs} without an error, or {soak_hours} h)",
252 
                    if runners == 1 { "" } else { "s" },
253 
                    if jobs == 1 { "" } else { "s" },
254 
                ))));
255 
            }
256 
        }
257 
    }
258 
    Ok(Ok(None))
259 
}
260
261 
/// The build `r` should install under `policy` at hour `hour` (UTC), or
262 
/// why there is none. A version a wave stopped is held back from every
263 
/// runner when `act` (a runner asking; a page only says so).
264 
pub async fn offer(
265 
    pool: &Pool,
266 
    levels: &Levels,
267 
    r: &Runner,
268 
    policy: &UpdatePolicy,
269 
    hour: u8,
270 
    now: i64,
271 
    act: bool,
272 
) -> sqlx::Result<(Option<UpdateOffer>, String)> {
273 
    if !policy.updates {
274 
        return Ok((None, format!("updates are off ({})", policy.from[0])));
275 
    }
276 
    let Some(spec) = db::accounts::setting(pool, SETTING_REPO).await?.filter(|s| !s.trim().is_empty()) else {
277 
        return Ok((None, "no runner releases repository is set (Admin › Runners › Updates)".into()));
278 
    };
279 
    let Some((owner, name)) = spec.trim().split_once('/') else {
280 
        return Ok((None, format!("the runner releases repository {spec:?} is not owner/name")));
281 
    };
282 
    let Some(repo) = db::repos::get(pool, owner, name).await? else {
283 
        return Ok((None, format!("the runner releases repository {spec} does not exist")));
284 
    };
285 
    let releases = db::releases::list(pool, repo.id).await?;
286 
    let release = match policy.channel.as_str() {
287 
        "stable" => releases.iter().find(|x| !x.prerelease),
288 
        "canary" => releases.first(),
289 
        v => releases.iter().find(|x| version_of(&x.tag) == v),
290 
    };
291 
    let Some(release) = release else {
292 
        return Ok((None, format!("{spec} has no release for the {} channel", policy.channel)));
293 
    };
294 
    let (want, have) = (version_of(&release.tag), version_of(&r.version));
295 
    if want == have {
296 
        return Ok((None, format!("up to date: flow {have} ({} channel)", policy.channel)));
297 
    }
298 
    if let Some(h) = rdb::held(pool, &want).await? {
299 
        return Ok((None, format!("flow {want} is held: {}; lift the hold on Admin › Runners › Updates", h.why)));
300 
    }
301 
    let going_back = older(&want, &have);
302 
    if going_back && matches!(policy.channel.as_str(), "stable" | "canary") {
303 
        return Ok((None, format!("runs flow {have}, newer than the {} channel's {want}", policy.channel)));
304 
    }
305 
    // Waves (oxygenflow#84): a later wave takes a version once those before it proved it.
306 
    if !going_back {
307 
        match wave_gate(pool, levels, r, &want, now).await? {
308 
            Ok(None) => {}
309 
            Ok(Some(why)) => return Ok((None, why)),
310 
            Err((runner, why)) => {
311 
                if act {
312 
                    rdb::hold(pool, &want, &runner, &why).await?;
313 
                    db::accounts::audit(pool, None, "runner.update.held", &want, &why).await.ok();
314 
                }
315 
                return Ok((None, format!("flow {want} is held: {why}; lift the hold on Admin › Runners › Updates")));
316 
            }
317 
        }
318 
    }
319 
    let file = asset_for(&r.os, &r.arch);
320 
    let assets = db::releases::assets(pool, release.id).await?;
321 
    let Some(a) = assets.iter().find(|a| a.name == file) else {
322 
        return Ok((None, format!("{spec} {} has no {file}", release.tag)));
323 
    };
324 
    if let Some(w) = policy.window.filter(|w| !in_window(*w, hour)) {
325 
        return Ok((None, format!("flow {want} waits for its window, {:02}:00–{:02}:00 UTC", w.0, w.1)));
326 
    }
327 
    let note = format!(
328 
        "flow {want} ({} channel){}",
329 
        policy.channel,
330 
        if policy.dry_run { ": dry run, check it and say what would happen" } else { "" }
331 
    );
332 
    Ok((
333 
        Some(UpdateOffer {
334 
            version: want,
335 
            tag: release.tag.clone(),
336 
            asset: a.name.clone(),
337 
            url: format!("/{owner}/{name}/asset/{}/{}", a.id, a.name),
338 
            blake3: a.hash.clone(),
339 
            size: a.size as u64,
340 
            older: going_back,
341 
        }),
342 
        note,
343 
    ))
344 
}
345
346 
#[cfg(test)]
347 
mod tests {
348 
    use super::*;
349
350 
    fn runner(id: i64, owner: &str) -> Runner {
351 
        Runner { id, scope_owner: owner.into(), version: "0.3.1 (build)".into(), ..Runner::default() }
352 
    }
353
354 
    #[test]
355 
    fn a_smaller_share_sits_inside_a_larger_one_and_is_about_its_size() {
356 
        let ten: Vec<i64> = (1..=1000).filter(|id| bucket(*id) < 10).collect();
357 
        let fifty: Vec<i64> = (1..=1000).filter(|id| bucket(*id) < 50).collect();
358 
        assert!(ten.iter().all(|id| fifty.contains(id)));
359 
        assert!((60..=140).contains(&ten.len()), "{}", ten.len());
360 
        assert_eq!(bucket(7), bucket(7), "stable");
361 
    }
362
363 
    #[test]
364 
    fn the_nearest_level_wins_each_setting() {
365 
        let mut l = Levels::default();
366 
        let pol = |scope: &str, key: &str, f: &dyn Fn(&mut Policy)| {
367 
            let mut p = Policy { scope: scope.into(), key: key.into(), ..Policy::default() };
368 
            f(&mut p);
369 
            ((scope.to_string(), key.to_string()), p)
370 
        };
371 
        l.policies.extend([
372 
            pol("site", "", &|p| {
373 
                p.updates = Some(false);
374 
                p.channel = Some("stable".into());
375 
            }),
376 
            pol("owner", "acme", &|p| p.updates = Some(true)),
377 
            pol("group", "1", &|p| p.channel = Some("canary".into())),
378 
            pol("runner", "3", &|p| p.updates = Some(false)),
379 
        ]);
380 
        l.groups.push(Group { id: 1, owner: "acme".into(), name: "canary".into(), percent: None, created_ms: 0, wave: None });
381 
        l.chosen.insert((1, 2));
382 
        l.chosen.insert((1, 9));
383
384 
        let site_runner = l.resolve(&runner(1, ""));
385 
        assert!(!site_runner.updates, "off at the site");
386 
        assert_eq!(site_runner.from, ["site", "site", "default", "default"]);
387
388 
        let canary = l.resolve(&runner(2, "acme"));
389 
        assert!(canary.updates && canary.channel == "canary");
390 
        assert_eq!(canary.from[..2], ["owner acme".to_string(), "group canary".to_string()]);
391
392 
        let held = l.resolve(&runner(3, "acme"));
393 
        assert!(!held.updates, "the runner's own setting wins");
394 
        assert_eq!(held.channel, "stable");
395
396 
        assert!(l.groups_of(&runner(9, "other")).is_empty(), "an owner's group holds only its own runners");
397 
        assert!(!Levels::default().resolve(&runner(4, "")).updates, "off unless chosen");
398 
    }
399
400 
    #[test]
401 
    fn a_percentage_group_takes_its_share_of_the_runners_in_reach() {
402 
        let mut l = Levels::default();
403 
        l.groups.push(Group { id: 5, owner: String::new(), name: "farm 20%".into(), percent: Some(20), created_ms: 0, wave: None });
404 
        let n = (1..=500).filter(|id| !l.groups_of(&runner(*id, "")).is_empty()).count();
405 
        assert!((60..=140).contains(&n), "{n}");
406 
    }
407
408 
    #[test]
409 
    fn a_wave_proves_a_version_by_its_jobs_or_its_time_and_an_error_stops_it() {
410 
        let soak = |runner_id, jobs, errors, first_ms| Soak { runner_id, jobs, errors, first_ms: Some(first_ms) };
411 
        let hour = 3_600_000;
412 
        // Nobody has taken it yet: not proved.
413 
        assert_eq!(wave_state(&[1, 2], &[], 5, 24 * hour, 0), WaveState::Soaking { runners: 2, on: 0, jobs: 0 });
414 
        // One runner took it and ran 3 of its 5; the other has not updated.
415 
        let s = [soak(1, 3, 0, 0), soak(9, 50, 0, 0)];
416 
        assert_eq!(wave_state(&[1, 2], &s, 5, 24 * hour, hour), WaveState::Soaking { runners: 2, on: 1, jobs: 3 });
417 
        // Five each on every runner that took it proves it.
418 
        assert_eq!(wave_state(&[1, 2], &[soak(1, 5, 0, 0)], 5, 24 * hour, hour), WaveState::Proved);
419 
        // Or a day since the first job there, with no error.
420 
        assert_eq!(wave_state(&[1, 2], &[soak(1, 1, 0, 0)], 5, 24 * hour, 24 * hour), WaveState::Proved);
421 
        // An error anywhere in the wave stops it, however many passed.
422 
        assert_eq!(
423 
            wave_state(&[1, 2], &[soak(1, 40, 0, 0), soak(2, 2, 1, 0)], 5, 24 * hour, 48 * hour),
424 
            WaveState::Stopped { runner_id: 2 }
425 
        );
426 
        // Another wave's errors are not this one's.
427 
        assert_eq!(wave_state(&[1], &[soak(1, 5, 0, 0), soak(2, 2, 1, 0)], 5, 24 * hour, hour), WaveState::Proved);
428 
        // An empty wave holds nothing up.
429 
        assert_eq!(wave_state(&[], &[], 5, 24 * hour, 0), WaveState::Proved);
430 
    }
431
432 
    #[test]
433 
    fn a_runner_waits_for_the_waves_before_its_own() {
434 
        let mut l = Levels::default();
435 
        let group = |id, name: &str, wave| Group { id, owner: String::new(), name: name.into(), percent: None, created_ms: 0, wave };
436 
        l.groups.extend([group(1, "first", Some(1)), group(2, "second", Some(2)), group(3, "watchers", None)]);
437 
        l.chosen.extend([(1, 1), (2, 2), (2, 1), (3, 3)]);
438 
        let all: Vec<Runner> = (1..=4).map(|id| runner(id, "")).collect();
439 
        let ids = |ws: Vec<(i64, Vec<&Runner>)>| {
440 
            ws.into_iter().map(|(w, rs)| (w, rs.iter().map(|r| r.id).collect::<Vec<_>>())).collect::<Vec<_>>()
441 
        };
442 
        assert_eq!(l.wave_of(&all[0]), Some(1), "its first wave");
443 
        assert!(l.waves_before(&all[0], &all).is_empty(), "wave 1 goes first");
444 
        assert_eq!(ids(l.waves_before(&all[1], &all)), vec![(1, vec![1])]);
445 
        assert_eq!(l.wave_of(&all[2]), None, "a group with no wave");
446 
        assert_eq!(ids(l.waves_before(&all[2], &all)), vec![(1, vec![1]), (2, vec![2])], "in no wave: after every wave");
447 
        assert_eq!(ids(l.waves_before(&all[3], &all)), vec![(1, vec![1]), (2, vec![2])]);
448 
        // An owner's waves gate only that owner's runners.
449 
        l.groups.push(Group { id: 4, owner: "acme".into(), name: "acme first".into(), percent: None, created_ms: 0, wave: Some(1) });
450 
        l.chosen.insert((4, 5));
451 
        let mut all = all;
452 
        all.push(runner(5, "acme"));
453 
        all.push(runner(6, "acme"));
454 
        assert_eq!(ids(l.waves_before(&all[5], &all)), vec![(1, vec![1, 5]), (2, vec![2])]);
455 
        assert_eq!(ids(l.waves_before(&all[3], &all)), vec![(1, vec![1]), (2, vec![2])], "not another owner's");
456 
    }
457
458 
    #[test]
459 
    fn windows_and_channels_read_as_written() {
460 
        assert_eq!(parse_window("02-05"), Ok(Some((2, 5))));
461 
        assert_eq!(parse_window(" 22:00-04:00 "), Ok(Some((22, 4))));
462 
        assert_eq!(parse_window(""), Ok(None));
463 
        assert!(parse_window("5").is_err() && parse_window("3-3").is_err() && parse_window("1-24").is_err());
464 
        assert!(in_window((2, 5), 2) && in_window((2, 5), 4) && !in_window((2, 5), 5));
465 
        assert!(in_window((22, 4), 23) && in_window((22, 4), 1) && !in_window((22, 4), 12));
466 
        assert!(valid_channel("stable") && valid_channel("canary") && valid_channel("0.3.4"));
467 
        assert!(!valid_channel("latest") && !valid_channel("0.3.") && !valid_channel(""));
468 
        assert_eq!(version_of("v0.3.2"), "0.3.2");
469 
        assert_eq!(version_of("0.3.1 (local build)"), "0.3.1");
470 
        assert_eq!(asset_for("linux", "riscv64"), "flow-linux-riscv64");
471 
        assert_eq!(asset_for("windows", "x86_64"), "flow-windows-x86_64.exe");
472 
        assert_eq!(
473 
            rolled_back_version("rolled back from flow 0.4.0 to 0.3.1: 3 starts without reaching the hub").as_deref(),
474 
            Some("0.4.0")
475 
        );
476 
        assert_eq!(rolled_back_version("runs flow 0.4.0 (was 0.3.1); proven"), None);
477 
    }
478 
}