OXYGENHUB
# e0918b887850 oxygenflow / crates / flow / src / microvm.rs
6.9 KiB · Rust · last changed OxygenFlow M0: pipeline parser and canonical job graph, receipts, the runner API types, flow-init, the Firecracker tier, flow check, run, rootfs import and runner 1 month ago source blame history raw
1
//! Tier 2 (spec §4.3, docs/m0.md §2): one Firecracker microVM per job,
2
//! no network device, the rootfs read-only, the workspace read-write,
3
//! `flow-init` as PID 1. The host reads the framed console, enforces
4
//! the time limit, and writes the receipt's environment.
5
6
use crate::home::Home;
7
use anyhow::{bail, Context, Result};
8
use flow_core::api::Record;
9
use flow_core::{Environment, Job, Sandbox};
10
use std::io::{BufRead, BufReader, Read};
11
use std::path::Path;
12
use std::sync::mpsc;
13
use std::time::{Duration, Instant};
14
15
pub struct Outcome {
16
    /// None when the job never reported an exit (killed at the time limit, or the VM failed).
17
    pub exit_code: Option<i32>,
18
    pub duration_ms: u64,
19
    pub environment: Environment,
20
    pub timed_out: bool,
21
}
22
23
/// The receipt environment for a job in this tier.
24
pub fn environment(home: &Home, sandbox: &Sandbox, job: &Job) -> Result<Environment> {
25
    let cfg = home.config.microvm.as_ref().context("flow.toml has no [microvm] section")?;
26
    Ok(Environment {
27
        runtime: "microvm".into(),
28
        parts: vec![
29
            ("firecracker".into(), cfg.firecracker.blake3.clone()),
30
            ("kernel".into(), cfg.kernel.blake3.clone()),
31
            ("init".into(), cfg.init.blake3.clone()),
32
            ("memory_mib".into(), sandbox.memory_mib.to_string()),
33
            ("cpus".into(), sandbox.cpus.to_string()),
34
        ],
35
        rootfs: sandbox.rootfs.clone(),
36
        argv: job.exec.clone(),
37
        env: job.env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
38
    })
39
}
40
41
/// Boot, run, collect. `scratch` is an empty directory this call may fill.
42
pub fn run(
43
    home: &Home,
44
    sandbox: &Sandbox,
45
    job: &Job,
46
    workspace: &Path,
47
    scratch: &Path,
48
    timeout: Duration,
49
    sink: &mut dyn FnMut(Record),
50
) -> Result<Outcome> {
51
    let cfg = home.config.microvm.as_ref().context("flow.toml has no [microvm] section")?;
52
    let firecracker = home.verified(&home.resolve(&cfg.firecracker.path), &cfg.firecracker.blake3)?;
53
    let kernel = home.verified(&home.resolve(&cfg.kernel.path), &cfg.kernel.blake3)?;
54
    let init = home.verified(&home.resolve(&cfg.init.path), &cfg.init.blake3)?;
55
    let rootfs = home.rootfs(&sandbox.rootfs);
56
    if !rootfs.is_file() {
57
        bail!("rootfs {} is not on this runner (import it with `flow rootfs import`)", sandbox.rootfs);
58
    }
59
    let rootfs = home.verified(&rootfs, &sandbox.rootfs)?;
60
    let environment = environment(home, sandbox, job)?;
61
62
    let spec = serde_json::json!({ "argv": job.exec, "env": job.env.iter().collect::<Vec<_>>() });
63
    let init_bytes = std::fs::read(&init)?;
64
    let spec_bytes = serde_json::to_vec(&spec)?;
65
    let initrd = crate::cpio::newc(&[
66
        crate::cpio::Entry { name: "init", mode: 0o755, data: &init_bytes },
67
        crate::cpio::Entry { name: "job.json", mode: 0o644, data: &spec_bytes },
68
    ]);
69
    let initrd_path = scratch.join("initrd.cpio");
70
    std::fs::write(&initrd_path, initrd)?;
71
    let vm = serde_json::json!({
72
        "boot-source": {
73
            "kernel_image_path": kernel,
74
            "initrd_path": initrd_path,
75
            "boot_args": "console=ttyS0 reboot=k panic=-1 pci=off quiet loglevel=0 rdinit=/init",
76
        },
77
        "drives": [
78
            { "drive_id": "rootfs", "path_on_host": rootfs, "is_root_device": false, "is_read_only": true },
79
            { "drive_id": "workspace", "path_on_host": workspace, "is_root_device": false, "is_read_only": false },
80
        ],
81
        "machine-config": { "vcpu_count": sandbox.cpus, "mem_size_mib": sandbox.memory_mib },
82
    });
83
    let vm_path = scratch.join("vm.json");
84
    std::fs::write(&vm_path, serde_json::to_vec_pretty(&vm)?)?;
85
86
    let started = Instant::now();
87
    let mut child = std::process::Command::new(&firecracker)
88
        .arg("--no-api")
89
        .arg("--config-file")
90
        .arg(&vm_path)
91
        .args(["--level", "Error"])
92
        .current_dir(scratch)
93
        .stdin(std::process::Stdio::null())
94
        .stdout(std::process::Stdio::piped())
95
        .stderr(std::process::Stdio::piped())
96
        .spawn()
97
        .with_context(|| format!("start {}", firecracker.display()))?;
98
    let (tx, rx) = mpsc::channel::<(bool, Vec<u8>)>();
99
    let out = child.stdout.take().unwrap();
100
    let err = child.stderr.take().unwrap();
101
    let tx2 = tx.clone();
102
    let readers = [std::thread::spawn(move || lines(out, true, tx)), std::thread::spawn(move || lines(err, false, tx2))];
103
    let mut exit_code = None;
104
    let mut timed_out = false;
105
    let deadline = started + timeout;
106
    let t = |s: &Instant| s.elapsed().as_millis() as u64;
107
    loop {
108
        match rx.recv_timeout(Duration::from_millis(200)) {
109
            Ok((from_console, line)) => {
110
                let text = String::from_utf8_lossy(&line).trim_end_matches('\r').to_string();
111
                let rec = |stream: &str, text: &str| Record { t: t(&started), stream: stream.into(), text: text.into() };
112
                if !from_console {
113
                    sink(rec("console", &format!("firecracker: {text}")));
114
                } else if let Some(rest) = text.strip_prefix('\u{1e}') {
115
                    let (kind, body) = rest.split_at(rest.len().min(1));
116
                    let body = body.strip_prefix(' ').unwrap_or(body);
117
                    match kind {
118
                        "O" => sink(rec("out", body)),
119
                        "E" => sink(rec("err", body)),
120
                        "X" => exit_code = body.trim().parse().ok(),
121
                        "F" => sink(rec("flow", &format!("guest init failed: {body}"))),
122
                        _ => sink(rec("console", &text)),
123
                    }
124
                } else if !text.is_empty() {
125
                    sink(rec("console", &text));
126
                }
127
            }
128
            Err(mpsc::RecvTimeoutError::Timeout) => {}
129
            Err(mpsc::RecvTimeoutError::Disconnected) => break,
130
        }
131
        if Instant::now() >= deadline && !timed_out {
132
            timed_out = true;
133
            let _ = child.kill();
134
            sink(Record { t: t(&started), stream: "flow".into(), text: format!("stopped at the time limit of {} s", timeout.as_secs()) });
135
        }
136
    }
137
    let status = child.wait()?;
138
    for r in readers {
139
        let _ = r.join();
140
    }
141
    if exit_code.is_none() && !timed_out {
142
        sink(Record { t: t(&started), stream: "flow".into(), text: format!("the microVM ended without the job's exit code ({status})") });
143
    }
144
    Ok(Outcome { exit_code: if timed_out { None } else { exit_code }, duration_ms: t(&started), environment, timed_out })
145
}
146
147
fn lines(r: impl Read, console: bool, tx: mpsc::Sender<(bool, Vec<u8>)>) {
148
    let mut r = BufReader::new(r);
149
    let mut buf = Vec::new();
150
    loop {
151
        buf.clear();
152
        match r.read_until(b'\n', &mut buf) {
153
            Ok(0) | Err(_) => break,
154
            Ok(_) => {
155
                if buf.last() == Some(&b'\n') {
156
                    buf.pop();
157
                }
158
                if tx.send((console, buf.clone())).is_err() {
159
                    break;
160
                }
161
            }
162
        }
163
    }
164
}