OXYGENHUB
# cc462b56fbbb oxygenflow / crates / flow / src / update.rs
16.3 KiB · Rust · last changed Runners ask the hub for their update and take it 10 days ago source blame history raw
85275e1e Andy McCall ↶ 1
//! Runner updates, the runner's side (docs/jetstream.md §8.2, #85): check
2
//! a new `flow` build, stage it beside the running one, swap it in keeping
3
//! the old one as `flow.prev`, and have the running runner leave with
4
//! notice so its unit starts the new build. The new build proves itself by
5
//! reaching the hub; three failed starts, or ten minutes without the hub,
6
//! put `flow.prev` back.
7
8
use anyhow::{bail, Context, Result};
9
use serde::{Deserialize, Serialize};
10
use std::path::{Path, PathBuf};
11
use std::time::Duration;
12
13
/// The staged build, beside the installed one.
14
pub const NEW: &str = "flow.new";
15
/// The build an update replaced, kept to go back to.
16
pub const PREV: &str = "flow.prev";
17
/// In the runner's home: the update not yet proven.
18
const PENDING: &str = "update.json";
19
/// In the runner's home: every update and rollback, a JSON line each.
20
const LOG: &str = "updates.log";
21
/// In the runner's home: the running runner's process id.
22
const PID: &str = "runner.pid";
23
/// Starts of an unproven build after which the previous one comes back.
24
pub const MAX_STARTS: u32 = 3;
25
/// How long a new build has to reach the hub.
26
pub const PROVE_WITHIN: Duration = Duration::from_secs(10 * 60);
27
28
/// An update swapped in and not yet proven.
29
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
30
pub struct Pending {
31
    pub from: String,
32
    pub to: String,
33
    /// The installed binary the update replaced.
34
    pub exe: PathBuf,
35
    pub at_ms: u64,
36
    /// How long the running runner had to finish what it held.
37
    pub notice_secs: u64,
38
    /// Starts of the new build so far.
39
    pub starts: u32,
40
}
41
42
/// What [`install`] did, or would do.
43
#[derive(Debug, PartialEq, Eq)]
44
pub struct Installed {
45
    pub from: String,
46
    pub to: String,
47
    pub exe: PathBuf,
48
    /// The running runner told to leave, by process id.
49
    pub told: Option<u32>,
50
}
51
52
/// The version a `flow --version` line names: `flow 0.3.1 (build)` → `0.3.1`.
53
pub fn parse_version(line: &str) -> Option<String> {
54
    let v = line.trim().strip_prefix("flow ")?.split_whitespace().next()?;
55
    v.split('.').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())).then(|| v.to_string())
56
}
57
58
/// Whether version `a` is older than `b`, part by part.
59
pub fn older(a: &str, b: &str) -> bool {
60
    let parts = |v: &str| v.split('.').map(|p| p.parse::<u64>().unwrap_or(0)).collect::<Vec<_>>();
61
    parts(a) < parts(b)
62
}
63
64
/// The ELF machine number of this build's architecture.
65
fn machine_here() -> Option<u16> {
66
    Some(match std::env::consts::ARCH {
67
        "x86_64" => 62,
68
        "aarch64" => 183,
69
        "riscv64" => 243,
70
        _ => return None,
71
    })
72
}
73
74
/// Refuse a file that is not a 64-bit little-endian ELF executable for this
75
/// machine: a build for another target would not start, and the rollback
76
/// would only notice after three tries.
77
pub fn check_target(bytes: &[u8]) -> Result<()> {
78
    if bytes.len() < 20 || &bytes[..4] != b"\x7fELF" {
79
        bail!("not an ELF executable");
80
    }
81
    if bytes[4] != 2 || bytes[5] != 1 {
82
        bail!("not a 64-bit little-endian build");
83
    }
84
    let machine = u16::from_le_bytes([bytes[18], bytes[19]]);
85
    match machine_here() {
86
        Some(m) if m == machine => Ok(()),
87
        Some(_) => bail!("a build for another machine (ELF machine {machine}); this runner is {}", std::env::consts::ARCH),
88
        None => bail!("updates are not supported on {}", std::env::consts::ARCH),
89
    }
90
}
91
92
/// Run `file args…` with a time limit, returning its standard output.
93
fn run(file: &Path, args: &[&str], limit: Duration) -> Result<String> {
94
    let mut child = std::process::Command::new(file)
95
        .args(args)
96
        .stdin(std::process::Stdio::null())
97
        .stdout(std::process::Stdio::piped())
98
        .stderr(std::process::Stdio::piped())
99
        .spawn()
100
        .with_context(|| format!("starting {}", file.display()))?;
101
    let start = std::time::Instant::now();
102
    loop {
103
        if child.try_wait()?.is_some() {
104
            break;
105
        }
106
        if start.elapsed() > limit {
107
            let _ = child.kill();
108
            let _ = child.wait();
109
            bail!("`{} {}` did not finish within {} s", file.display(), args.join(" "), limit.as_secs());
110
        }
111
        std::thread::sleep(Duration::from_millis(50));
112
    }
113
    let out = child.wait_with_output()?;
114
    if !out.status.success() {
115
        bail!("`{} {}` failed ({}): {}", file.display(), args.join(" "), out.status, String::from_utf8_lossy(&out.stderr).trim());
116
    }
117
    Ok(String::from_utf8_lossy(&out.stdout).into_owned())
118
}
119
120
/// Check a build as the runner would run it: this machine's target, it
121
/// starts and names its version, and it finds the tiers this home offers.
122
/// Returns its version.
123
pub fn check(file: &Path, home: &Path) -> Result<String> {
124
    let mut head = vec![0u8; 64];
125
    let n = std::io::Read::read(&mut std::fs::File::open(file).with_context(|| format!("opening {}", file.display()))?, &mut head)?;
126
    check_target(&head[..n])?;
127
    let limit = Duration::from_secs(30);
128
    let line = run(file, &["--version"], limit)?;
129
    let version = parse_version(&line).with_context(|| format!("{} does not say it is flow: {:?}", file.display(), line.trim()))?;
130
    let home_s = home.to_string_lossy();
131
    let tiers = run(file, &["--home", &home_s, "tiers"], limit)?;
132
    if !tiers.lines().any(|l| l.ends_with(": available")) {
133
        bail!("flow {version} finds no tier it can run here:\n{}", tiers.trim());
134
    }
135
    Ok(version)
136
}
137
138
fn now_ms() -> u64 {
139
    std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0)
140
}
141
142
fn log(home: &Path, event: &str, from: &str, to: &str, why: &str) {
143
    use std::io::Write;
144
    let line = serde_json::json!({"at_ms": now_ms(), "event": event, "from": from, "to": to, "why": why});
145
    if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(home.join(LOG)) {
146
        let _ = writeln!(f, "{line}");
147
    }
148
}
149
150
/// The update not yet proven, if any.
151
pub fn pending(home: &Path) -> Option<Pending> {
152
    serde_json::from_slice(&std::fs::read(home.join(PENDING)).ok()?).ok()
153
}
154
155
fn save(home: &Path, p: &Pending) -> Result<()> {
156
    let tmp = home.join(format!("{PENDING}.tmp"));
157
    std::fs::write(&tmp, serde_json::to_vec_pretty(p)?)?;
158
    std::fs::rename(&tmp, home.join(PENDING))?;
159
    Ok(())
160
}
161
162
/// Put `staged` in place of `exe`, keeping `exe` as [`PREV`] beside it.
163
/// `exe` is never missing: it is copied aside, then replaced in one rename.
164
pub fn swap(exe: &Path, staged: &Path) -> Result<()> {
165
    let prev = exe.with_file_name(PREV);
166
    let _ = std::fs::remove_file(&prev);
167
    if std::fs::hard_link(exe, &prev).is_err() {
168
        std::fs::copy(exe, &prev).with_context(|| format!("keeping {} as {}", exe.display(), prev.display()))?;
169
    }
170
    std::fs::rename(staged, exe).with_context(|| format!("putting {} in place of {}", staged.display(), exe.display()))?;
171
    Ok(())
172
}
173
174
/// Put [`PREV`] back in place of `exe`.
175
pub fn restore(exe: &Path) -> Result<()> {
176
    let prev = exe.with_file_name(PREV);
177
    if !prev.is_file() {
178
        bail!("{} is not there to go back to", prev.display());
179
    }
180
    std::fs::rename(&prev, exe).with_context(|| format!("putting {} back", prev.display()))?;
181
    Ok(())
182
}
183
184
/// Check `file`, stage it beside `exe`, and unless `dry_run` swap it in and
185
/// tell the running runner (if any) to leave within `notice`, finishing
186
/// what fits and handing back the rest, so its unit starts the new build.
187
pub fn install(home: &crate::home::Home, exe: &Path, file: &Path, notice: Duration, dry_run: bool, allow_older: bool) -> Result<Installed> {
188
    if home.config.runner.as_ref().and_then(|r| r.update.as_ref()).is_some_and(|u| !u.allow) {
189
        bail!("flow.toml says [runner.update] allow = false: this runner takes no updates");
190
    }
191
    if let Some(p) = pending(&home.root) {
192
        bail!("the update from {} to {} is not proven yet; wait for it (or for its rollback)", p.from, p.to);
193
    }
194
    let from = env!("CARGO_PKG_VERSION").to_string();
195
    let staged = exe.with_file_name(NEW);
196
    let _ = std::fs::remove_file(&staged);
197
    std::fs::copy(file, &staged).with_context(|| format!("staging {} as {}", file.display(), staged.display()))?;
198
    {
199
        use std::os::unix::fs::PermissionsExt;
200
        std::fs::set_permissions(&staged, std::fs::Permissions::from_mode(0o755))?;
201
    }
202
    let checked = check(&staged, &home.root).and_then(|to| {
203
        if older(&to, &from) && !allow_older {
204
            bail!("flow {to} is older than this runner's {from} (--allow-older to go back)");
205
        }
206
        Ok(to)
207
    });
208
    let to = match checked {
209
        Ok(v) => v,
210
        Err(e) => {
211
            let _ = std::fs::remove_file(&staged);
212
            return Err(e);
213
        }
214
    };
215
    if dry_run {
216
        let _ = std::fs::remove_file(&staged);
217
        return Ok(Installed { from, to, exe: exe.to_path_buf(), told: None });
218
    }
219
    swap(exe, &staged)?;
220
    let p =
221
        Pending { from: from.clone(), to: to.clone(), exe: exe.to_path_buf(), at_ms: now_ms(), notice_secs: notice.as_secs(), starts: 0 };
222
    save(&home.root, &p)?;
223
    log(&home.root, "installed", &from, &to, "flow runner update --from");
224
    let told = running(&home.root).filter(|pid| unsafe { libc::kill(*pid as libc::pid_t, libc::SIGUSR1) } == 0);
225
    Ok(Installed { from, to, exe: exe.to_path_buf(), told })
226
}
227
228
/// The running runner's process id, from its pid file, if that process is
229
/// still a `flow runner`: a stale file's pid may be another program's now,
230
/// and SIGUSR1 would end it. Builds before #85 write no pid file (and do
231
/// not listen for SIGUSR1), so they are never told.
232
fn running(home: &Path) -> Option<u32> {
233
    let pid: u32 = std::fs::read_to_string(home.join(PID)).ok()?.trim().parse().ok()?;
234
    let cmdline = std::fs::read(format!("/proc/{pid}/cmdline")).ok()?;
235
    is_runner(&cmdline).then_some(pid)
236
}
237
238
/// Whether a NUL-separated command line is `flow … runner` with no subcommand.
239
fn is_runner(cmdline: &[u8]) -> bool {
240
    let args: Vec<&[u8]> = cmdline.split(|b| *b == 0).filter(|a| !a.is_empty()).collect();
241
    let Some((argv0, rest)) = args.split_first() else { return false };
242
    let name = argv0.rsplit(|b| *b == b'/').next().unwrap_or(argv0);
243
    name.starts_with(b"flow") && rest.last() == Some(&&b"runner"[..])
244
}
245
246
/// Note this process as the home's runner, for `flow runner update` to tell.
247
pub fn note_runner(home: &Path) {
248
    let _ = std::fs::write(home.join(PID), std::process::id().to_string());
249
}
250
251
/// The runner stopped: forget its pid.
252
pub fn forget_runner(home: &Path) {
253
    if std::fs::read_to_string(home.join(PID)).is_ok_and(|p| p.trim() == std::process::id().to_string()) {
254
        let _ = std::fs::remove_file(home.join(PID));
255
    }
256
}
257
258
/// What a runner starting up should do about an update.
259
#[derive(Debug, PartialEq, Eq)]
260
pub enum Start {
261
    /// No update waits to be proven.
262
    Settled,
263
    /// The new build is on trial: prove it within [`PROVE_WITHIN`].
264
    Trial(Pending),
265
    /// It failed too often; the previous build is back and should be run.
266
    RolledBack(Pending),
267
}
268
269
/// Count this start of an unproven build; after [`MAX_STARTS`], put the
270
/// previous build back.
271
pub fn on_start(home: &Path) -> Result<Start> {
272
    let Some(mut p) = pending(home) else { return Ok(Start::Settled) };
273
    p.starts += 1;
274
    if p.starts > MAX_STARTS {
275
        roll_back(home, &p, &format!("{MAX_STARTS} starts without reaching the hub"))?;
276
        return Ok(Start::RolledBack(p));
277
    }
278
    save(home, &p)?;
279
    Ok(Start::Trial(p))
280
}
281
cedd1234 Andy McCall ↶ 282
/// In the runner's home: what to tell the hub once it answers.
283
const NOTE: &str = "update.note";
284
285
/// Keep `note` to tell the hub (a proof, a rollback) once it answers.
286
fn note(home: &Path, note: &str) {
287
    let _ = std::fs::write(home.join(NOTE), note);
288
}
289
290
/// What there is to tell the hub, once.
291
pub fn take_note(home: &Path) -> Option<String> {
292
    let n = std::fs::read_to_string(home.join(NOTE)).ok()?;
293
    let _ = std::fs::remove_file(home.join(NOTE));
294
    Some(n).filter(|n| !n.trim().is_empty())
295
}
296
85275e1e Andy McCall ↶ 297
/// The new build reached the hub: it stays.
298
pub fn proven(home: &Path) {
299
    if let Some(p) = pending(home) {
300
        let _ = std::fs::remove_file(home.join(PENDING));
301
        log(home, "proven", &p.from, &p.to, "reached the hub");
cedd1234 Andy McCall ↶ 302
        note(home, &format!("runs flow {} (was {}); proven", p.to, p.from));
85275e1e Andy McCall ↶ 303
        eprintln!("flow runner: flow {} proven (it reached the hub); flow {} stays as {PREV}", p.to, p.from);
304
    }
305
}
306
307
/// Put the previous build back and forget the update.
308
pub fn roll_back(home: &Path, p: &Pending, why: &str) -> Result<()> {
309
    restore(&p.exe)?;
310
    let _ = std::fs::remove_file(home.join(PENDING));
311
    log(home, "rolled back", &p.to, &p.from, why);
cedd1234 Andy McCall ↶ 312
    note(home, &format!("rolled back from flow {} to {}: {why}", p.to, p.from));
85275e1e Andy McCall ↶ 313
    eprintln!("flow runner: rolled back from flow {} to {} ({why})", p.to, p.from);
314
    Ok(())
315
}
316
317
#[cfg(test)]
318
mod tests {
319
    use super::*;
320
321
    #[test]
322
    fn versions_are_read_and_compared() {
323
        assert_eq!(parse_version("flow 0.3.1 (local build)\n").as_deref(), Some("0.3.1"));
324
        assert_eq!(parse_version("flow 0.10.0"), Some("0.10.0".into()));
325
        assert_eq!(parse_version("oxy 0.6.0"), None);
326
        assert_eq!(parse_version("flow banana"), None);
327
        assert!(older("0.3.1", "0.10.0"), "by number, not by text");
328
        assert!(!older("0.3.1", "0.3.1"));
329
        assert!(older("0.3", "0.3.1"));
330
    }
331
332
    #[test]
333
    fn only_a_build_for_this_machine_passes() {
334
        let me = std::fs::read(std::env::current_exe().unwrap()).unwrap();
335
        check_target(&me[..64]).unwrap();
336
        assert!(check_target(b"#!/bin/sh\necho flow 9.9.9\n").unwrap_err().to_string().contains("not an ELF"));
337
        let mut other = me[..64].to_vec();
338
        other[18..20].copy_from_slice(&(if std::env::consts::ARCH == "riscv64" { 62u16 } else { 243u16 }).to_le_bytes());
339
        assert!(check_target(&other).unwrap_err().to_string().contains("another machine"));
340
    }
341
342
    #[test]
343
    fn a_swap_keeps_the_old_build_and_three_failed_starts_bring_it_back() {
344
        let dir = tempfile::tempdir().unwrap();
345
        let (bin, home) = (dir.path().join("bin"), dir.path().join("home"));
346
        std::fs::create_dir_all(&bin).unwrap();
347
        std::fs::create_dir_all(&home).unwrap();
348
        let exe = bin.join("flow");
349
        std::fs::write(&exe, "old").unwrap();
350
        std::fs::write(bin.join(NEW), "new").unwrap();
351
        swap(&exe, &bin.join(NEW)).unwrap();
352
        assert_eq!(std::fs::read_to_string(&exe).unwrap(), "new");
353
        assert_eq!(std::fs::read_to_string(bin.join(PREV)).unwrap(), "old");
354
        assert!(!bin.join(NEW).exists());
355
356
        assert_eq!(on_start(&home).unwrap(), Start::Settled, "no update, nothing to do");
357
        let p = Pending { from: "0.3.1".into(), to: "0.3.2".into(), exe: exe.clone(), at_ms: 1, notice_secs: 600, starts: 0 };
358
        save(&home, &p).unwrap();
359
        for n in 1..=MAX_STARTS {
360
            assert_eq!(on_start(&home).unwrap(), Start::Trial(Pending { starts: n, ..p.clone() }));
361
        }
362
        assert!(matches!(on_start(&home).unwrap(), Start::RolledBack(_)));
363
        assert_eq!(std::fs::read_to_string(&exe).unwrap(), "old", "the previous build is back");
364
        assert!(pending(&home).is_none());
365
        let log = std::fs::read_to_string(home.join(LOG)).unwrap();
366
        assert!(log.contains("rolled back") && log.contains("0.3.2"), "{log}");
367
    }
368
369
    #[test]
370
    fn only_a_running_runner_is_told() {
371
        assert!(is_runner(b"/usr/local/bin/flow\0runner\0"));
372
        assert!(is_runner(b"flow\0--home\0/srv/flow\0runner\0"));
373
        assert!(!is_runner(b"flow\0runner\0update\0--from\0x\0"), "not the update command itself");
374
        assert!(!is_runner(b"/usr/bin/vim\0runner\0"));
375
        assert!(!is_runner(b""));
376
        let dir = tempfile::tempdir().unwrap();
377
        std::fs::write(dir.path().join(PID), std::process::id().to_string()).unwrap();
378
        assert_eq!(running(dir.path()), None, "this test is not a flow runner");
379
    }
380
381
    #[test]
382
    fn a_proven_build_stays() {
383
        let dir = tempfile::tempdir().unwrap();
384
        let exe = dir.path().join("flow");
385
        std::fs::write(&exe, "new").unwrap();
386
        let p = Pending { from: "0.3.1".into(), to: "0.3.2".into(), exe: exe.clone(), at_ms: 1, notice_secs: 0, starts: 1 };
387
        save(dir.path(), &p).unwrap();
388
        proven(dir.path());
389
        assert!(pending(dir.path()).is_none());
390
        assert_eq!(on_start(dir.path()).unwrap(), Start::Settled);
391
        assert!(std::fs::read_to_string(dir.path().join(LOG)).unwrap().contains("proven"));
392
    }
393
}