OXYGENHUB
# c5b702e59c1f oxygenflow / crates / flow / src / microvm.rs
10.2 KiB · Rust · last changed Secrets mount as files: secret blocks, the claim's secrets, flow-init writes them 1 month ago source blame history raw
1
//! Tier 2 (spec §4.3, docs/m0.md §2): one Firecracker microVM per job,
2
//! no network device, the rootfs read-only, the workspace read-write,
3
//! `flow-init` as PID 1. The host reads the framed console, enforces
4
//! the time limit, and writes the receipt's environment.
5
6
use crate::home::Home;
7
use anyhow::{bail, Context, Result};
8
use flow_core::api::{Record, SecretInput};
9
use flow_core::{Environment, Job, MountSource, Sandbox};
10
use std::io::{BufRead, BufReader, Read};
11
use std::path::Path;
12
use std::sync::mpsc;
13
use std::time::{Duration, Instant};
14
15
pub struct Outcome {
16
    /// None when the job never reported an exit (killed at the time limit, or the VM failed).
17
    pub exit_code: Option<i32>,
18
    pub duration_ms: u64,
19
    pub environment: Environment,
20
    pub timed_out: bool,
21
}
22
23
/// The receipt environment for a job in this tier.
24
pub fn environment(home: &Home, sandbox: &Sandbox, job: &Job) -> Result<Environment> {
25
    let cfg = home.config.microvm.as_ref().context("flow.toml has no [microvm] section")?;
26
    Ok(Environment {
27
        runtime: "microvm".into(),
28
        parts: vec![
29
            ("firecracker".into(), cfg.firecracker.blake3.clone()),
30
            ("kernel".into(), cfg.kernel.blake3.clone()),
31
            ("init".into(), cfg.init.blake3.clone()),
32
            ("memory_mib".into(), sandbox.memory_mib.to_string()),
33
            ("cpus".into(), sandbox.cpus.to_string()),
34
        ]
35
        .into_iter()
36
        .chain(job.mounts.iter().map(|m| (format!("mount {}", m.target), format!("{} {} MiB", m.source.as_str(), m.size_mib))))
37
        .collect(),
38
        rootfs: sandbox.rootfs.clone(),
39
        argv: job.exec.clone(),
40
        env: job.env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
41
    })
42
}
43
44
/// Boot, run, collect. `scratch` is an empty directory this call may fill.
45
#[allow(clippy::too_many_arguments)]
46
pub fn run(
47
    home: &Home,
48
    sandbox: &Sandbox,
49
    job: &Job,
50
    secrets: &[SecretInput],
51
    workspace: &Path,
52
    scratch: &Path,
53
    timeout: Duration,
54
    sink: &mut dyn FnMut(Record),
55
) -> Result<Outcome> {
56
    let cfg = home.config.microvm.as_ref().context("flow.toml has no [microvm] section")?;
57
    let firecracker = home.verified(&home.resolve(&cfg.firecracker.path), &cfg.firecracker.blake3)?;
58
    let kernel = home.verified(&home.resolve(&cfg.kernel.path), &cfg.kernel.blake3)?;
59
    let init = home.verified(&home.resolve(&cfg.init.path), &cfg.init.blake3)?;
60
    let rootfs = home.rootfs(&sandbox.rootfs);
61
    if !rootfs.is_file() {
62
        bail!("rootfs {} is not on this runner (import it with `flow rootfs import`)", sandbox.rootfs);
63
    }
64
    let rootfs = home.verified(&rootfs, &sandbox.rootfs)?;
65
    let environment = environment(home, sandbox, job)?;
66
67
    let mut spec = guest_spec(job);
68
    spec["loopback"] = serde_json::Value::Bool(sandbox.network == flow_core::Network::Loopback);
69
    // Secrets ride in the guest's job file, which only flow-init reads,
70
    // outside the root the job is confined to; it writes each as a file.
71
    if !secrets.is_empty() {
72
        spec["secrets"] =
73
            serde_json::Value::Array(secrets.iter().map(|s| serde_json::json!({ "path": s.path, "value": s.value })).collect());
74
    }
75
    // Each `ephemeral://disk` mount is a fresh sparse ext4 disk in the
76
    // scratch directory, attached after the workspace (vdc, vdd, ...).
77
    let mut disks = Vec::new();
78
    for (i, m) in job.mounts.iter().filter(|m| m.source == MountSource::Disk).enumerate() {
79
        let path = scratch.join(format!("mount-{i}.ext4"));
80
        crate::workspace::empty_ext4(&path, m.size_mib, "scratch")?;
81
        disks.push(
82
            serde_json::json!({ "drive_id": format!("mount{i}"), "path_on_host": path, "is_root_device": false, "is_read_only": false }),
83
        );
84
    }
85
    let init_bytes = std::fs::read(&init)?;
86
    let spec_bytes = serde_json::to_vec(&spec)?;
87
    let initrd = crate::cpio::newc(&[
88
        crate::cpio::Entry { name: "init", mode: 0o755, data: &init_bytes },
89
        crate::cpio::Entry { name: "job.json", mode: 0o644, data: &spec_bytes },
90
    ]);
91
    let initrd_path = scratch.join("initrd.cpio");
92
    std::fs::write(&initrd_path, initrd)?;
93
    let drives: Vec<serde_json::Value> = [
94
        serde_json::json!({ "drive_id": "rootfs", "path_on_host": rootfs, "is_root_device": false, "is_read_only": true }),
95
        serde_json::json!({ "drive_id": "workspace", "path_on_host": workspace, "is_root_device": false, "is_read_only": false }),
96
    ]
97
    .into_iter()
98
    .chain(disks)
99
    .collect();
100
    let vm = serde_json::json!({
101
        "boot-source": {
102
            "kernel_image_path": kernel,
103
            "initrd_path": initrd_path,
104
            "boot_args": "console=ttyS0 reboot=k panic=-1 pci=off quiet loglevel=0 rdinit=/init",
105
        },
106
        "drives": drives,
107
        "machine-config": { "vcpu_count": sandbox.cpus, "mem_size_mib": sandbox.memory_mib },
108
    });
109
    let vm_path = scratch.join("vm.json");
110
    std::fs::write(&vm_path, serde_json::to_vec_pretty(&vm)?)?;
111
112
    let started = Instant::now();
113
    let mut child = std::process::Command::new(&firecracker)
114
        .arg("--no-api")
115
        .arg("--config-file")
116
        .arg(&vm_path)
117
        .args(["--level", "Error"])
118
        .current_dir(scratch)
119
        .stdin(std::process::Stdio::null())
120
        .stdout(std::process::Stdio::piped())
121
        .stderr(std::process::Stdio::piped())
122
        .spawn()
123
        .with_context(|| format!("start {}", firecracker.display()))?;
124
    let (tx, rx) = mpsc::channel::<(bool, Vec<u8>)>();
125
    let out = child.stdout.take().unwrap();
126
    let err = child.stderr.take().unwrap();
127
    let tx2 = tx.clone();
128
    let readers = [std::thread::spawn(move || lines(out, true, tx)), std::thread::spawn(move || lines(err, false, tx2))];
129
    let mut exit_code = None;
130
    let mut timed_out = false;
131
    let deadline = started + timeout;
132
    let t = |s: &Instant| s.elapsed().as_millis() as u64;
133
    loop {
134
        match rx.recv_timeout(Duration::from_millis(200)) {
135
            Ok((from_console, line)) => {
136
                let text = String::from_utf8_lossy(&line).trim_end_matches('\r').to_string();
137
                let rec = |stream: &str, text: &str| Record { t: t(&started), stream: stream.into(), text: text.into() };
138
                if !from_console {
139
                    sink(rec("console", &format!("firecracker: {text}")));
140
                } else if let Some(rest) = text.strip_prefix('\u{1e}') {
141
                    let (kind, body) = rest.split_at(rest.len().min(1));
142
                    let body = body.strip_prefix(' ').unwrap_or(body);
143
                    match kind {
144
                        "O" => sink(rec("out", body)),
145
                        "E" => sink(rec("err", body)),
146
                        "X" => exit_code = body.trim().parse().ok(),
147
                        "F" => sink(rec("flow", &format!("guest init failed: {body}"))),
148
                        _ => sink(rec("console", &text)),
149
                    }
150
                } else if !text.is_empty() {
151
                    sink(rec("console", &text));
152
                }
153
            }
154
            Err(mpsc::RecvTimeoutError::Timeout) => {}
155
            Err(mpsc::RecvTimeoutError::Disconnected) => break,
156
        }
157
        if Instant::now() >= deadline && !timed_out {
158
            timed_out = true;
159
            let _ = child.kill();
160
            sink(Record { t: t(&started), stream: "flow".into(), text: format!("stopped at the time limit of {} s", timeout.as_secs()) });
161
        }
162
    }
163
    let status = child.wait()?;
164
    for r in readers {
165
        let _ = r.join();
166
    }
167
    if exit_code.is_none() && !timed_out {
168
        sink(Record { t: t(&started), stream: "flow".into(), text: format!("the microVM ended without the job's exit code ({status})") });
169
    }
170
    Ok(Outcome { exit_code: if timed_out { None } else { exit_code }, duration_ms: t(&started), environment, timed_out })
171
}
172
173
/// What `flow-init` reads from `/job.json`: argv, env, and the mounts
174
/// with the device each disk mount is attached as (vdc onwards).
175
pub fn guest_spec(job: &Job) -> serde_json::Value {
176
    let mut next_disk = b'c';
177
    let mounts: Vec<serde_json::Value> = job
178
        .mounts
179
        .iter()
180
        .map(|m| match m.source {
181
            MountSource::Tmpfs => serde_json::json!({ "kind": "tmpfs", "target": m.target, "size_mib": m.size_mib }),
182
            MountSource::Disk => {
183
                let dev = format!("/dev/vd{}", next_disk as char);
184
                next_disk += 1;
185
                serde_json::json!({ "kind": "disk", "target": m.target, "size_mib": m.size_mib, "device": dev })
186
            }
187
        })
188
        .collect();
189
    serde_json::json!({ "argv": job.exec, "env": job.env.iter().collect::<Vec<_>>(), "mounts": mounts })
190
}
191
192
fn lines(r: impl Read, console: bool, tx: mpsc::Sender<(bool, Vec<u8>)>) {
193
    let mut r = BufReader::new(r);
194
    let mut buf = Vec::new();
195
    loop {
196
        buf.clear();
197
        match r.read_until(b'\n', &mut buf) {
198
            Ok(0) | Err(_) => break,
199
            Ok(_) => {
200
                if buf.last() == Some(&b'\n') {
201
                    buf.pop();
202
                }
203
                if tx.send((console, buf.clone())).is_err() {
204
                    break;
205
                }
206
            }
207
        }
208
    }
209
}
210
211
#[cfg(test)]
212
mod tests {
213
    use super::*;
214
    use flow_core::Mount;
215
216
    #[test]
217
    fn guest_spec_numbers_disks_after_the_workspace() {
218
        let mut job: Job = serde_json::from_value(serde_json::json!({
219
            "name": "t", "sandbox": "s", "scope": [], "depends_on": [], "env": {"A": "1"}, "exec": ["true"]
220
        }))
221
        .unwrap();
222
        assert_eq!(guest_spec(&job)["mounts"], serde_json::json!([]), "no mounts");
223
        job.mounts = vec![
224
            Mount { source: MountSource::Disk, target: "/src/target".into(), size_mib: 8192 },
225
            Mount { source: MountSource::Tmpfs, target: "/tmp/cache".into(), size_mib: 256 },
226
            Mount { source: MountSource::Disk, target: "/tmp/z".into(), size_mib: 64 },
227
        ];
228
        let spec = guest_spec(&job);
229
        assert_eq!(spec["mounts"][0], serde_json::json!({"kind": "disk", "target": "/src/target", "size_mib": 8192, "device": "/dev/vdc"}));
230
        assert_eq!(spec["mounts"][1], serde_json::json!({"kind": "tmpfs", "target": "/tmp/cache", "size_mib": 256}));
231
        assert_eq!(spec["mounts"][2]["device"], "/dev/vdd");
232
        assert_eq!(spec["env"], serde_json::json!([["A", "1"]]));
233
    }
234
}