OXYGENHUB

Coverage / crates/flow-init/src/main.rs 0.0%

0 of 244 lines run, 0 of 28 functions · run #203 on main · run missed

1
//! PID 1 of a job's microVM (docs/m0.md §2). It mounts the read-only
2
//! rootfs and the workspace, runs the job's argv as an unprivileged user
3
//! with exactly the job's environment, frames the output on the serial
4
//! console for the host, and powers off. It never decides anything the
5
//! host has not: the job comes from `/job.json` in the initramfs the host
6
//! built, and the host enforces the time limit.
7
//!
8
//! Frames, one per line on the console: `\x1eO ` stdout, `\x1eE ` stderr,
9
//! `\x1eX <code>` the exit code (128 + signal when killed), `\x1eF <text>`
10
//! a failure of init itself, `\x1eC <json>` a build cache pruned before
11
//! the guest powers off (oxygenflow#72): `{"target": ..., "pruned":
12
//! flow_init::prune::Pruned}`, sent before the exit code.
13
14
use flow_init::prune;
15
use serde::Deserialize;
16
use std::ffi::CString;
17
use std::io::{Read, Write};
18
use std::os::fd::FromRawFd;
19
use std::sync::{Arc, Mutex};
20
21
mod egress;
22
mod job;
23
use job::{dev_links, job_env, loopback_up, resolve_program, PATH};
24
25
#[derive(Deserialize)]
26
struct JobSpec {
27
    argv: Vec<String>,
28
    env: Vec<(String, String)>,
29
    #[serde(default)]
30
    mounts: Vec<MountSpec>,
31
    /// `network "loopback"`: bring up `lo` (the guest has no other device).
32
    #[serde(default)]
33
    loopback: bool,
34
    /// `network "egress"` (oxygenflow#55): `lo` comes up and a proxy on
35
    /// 127.0.0.1:3128 relays over vsock to the runner.
36
    #[serde(default)]
37
    egress: bool,
38
    /// Secrets to write before the job starts (spec §9): each at `path`
39
    /// under `/run/secrets`, 0400 and the job's user's, on a tmpfs.
40
    #[serde(default)]
41
    secrets: Vec<SecretSpec>,
42
}
43
44
#[derive(serde::Deserialize)]
45
struct SecretSpec {
46
    path: String,
47
    value: String,
48
}
49
50
/// An ephemeral mount from the host: a tmpfs of `size_mib`, or the disk at
51
/// `device`, mounted at `target` under the new root.
52
#[derive(serde::Deserialize)]
53
struct MountSpec {
54
    kind: String,
55
    target: String,
56
    size_mib: u32,
57
    #[serde(default)]
58
    device: Option<String>,
59
    /// A build cache's disk (oxygenflow#72): mounted `strictatime`, so
60
    /// what the job reads is seen, and after a passing job pruned to this
61
    /// percentage of its size before the guest powers off.
62
    #[serde(default)]
63
    prune: Option<u8>,
64
    /// Paths on the disk, relative to it, that pruning leaves: the job's
65
    /// outputs, which the runner reads after the guest stops.
66
    #[serde(default)]
67
    keep: Vec<String>,
68
}
69
700×
fn main() {
710×
    let console = Arc::new(Mutex::new(std::io::stdout()));
720×
    let code = match run(&console) {
730×
        Ok(code) => code,
740×
        Err(e) => {
750×
            frame(&console, b'F', e.as_bytes());
760×
            125
77
        }
78
    };
790×
    frame(&console, b'X', code.to_string().as_bytes());
800×
    unsafe {
810×
        libc::sync();
820×
        // Firecracker ends the VM when the guest reboots (reboot=k).
830×
        libc::reboot(libc::LINUX_REBOOT_CMD_RESTART);
840×
    }
850×
}
86
870×
fn frame(console: &Mutex<std::io::Stdout>, kind: u8, text: &[u8]) {
880×
    let mut out = console.lock().unwrap_or_else(|p| p.into_inner());
890×
    let mut line = Vec::with_capacity(text.len() + 4);
900×
    line.extend_from_slice(&[0x1e, kind, b' ']);
910×
    line.extend_from_slice(text);
920×
    line.push(b'\n');
930×
    let _ = out.write_all(&line);
940×
    let _ = out.flush();
950×
}
96
970×
fn cstr(s: &str) -> Result<CString, String> {
980×
    CString::new(s).map_err(|_| format!("{s:?} contains NUL"))
990×
}
100
1010×
fn chown(path: &str, uid: libc::uid_t, gid: libc::gid_t) -> Result<(), String> {
1020×
    let p = cstr(path)?;
1030×
    if unsafe { libc::chown(p.as_ptr(), uid, gid) } != 0 {
1040×
        return Err(format!("chown {path}: {}", std::io::Error::last_os_error()));
1050×
    }
1060×
    Ok(())
1070×
}
108
109
/// Create `path` and its missing parents, each owned by the job's user.
1100×
fn make_dirs_owned(path: &str, uid: libc::uid_t, gid: libc::gid_t) -> Result<(), String> {
1110×
    let mut missing = Vec::new();
1120×
    let mut p = std::path::Path::new(path);
1130×
    while !p.exists() {
1140×
        missing.push(p.to_path_buf());
1150×
        p = match p.parent() {
1160×
            Some(parent) => parent,
1170×
            None => break,
118
        };
119
    }
1200×
    for d in missing.iter().rev() {
1210×
        std::fs::create_dir(d).map_err(|e| format!("mkdir {}: {e}", d.display()))?;
1220×
        chown(&d.to_string_lossy(), uid, gid)?;
123
    }
1240×
    Ok(())
1250×
}
126
1270×
fn mount(src: &str, target: &str, fstype: &str, flags: libc::c_ulong, data: &str) -> Result<(), String> {
1280×
    let (s, t, f, d) = (cstr(src)?, cstr(target)?, cstr(fstype)?, cstr(data)?);
1290×
    let rc = unsafe { libc::mount(s.as_ptr(), t.as_ptr(), f.as_ptr(), flags, d.as_ptr() as *const libc::c_void) };
1300×
    if rc != 0 {
1310×
        return Err(format!("mount {src} on {target}: {}", std::io::Error::last_os_error()));
1320×
    }
1330×
    Ok(())
1340×
}
135
1360×
fn run(console: &Arc<Mutex<std::io::Stdout>>) -> Result<i32, String> {
1370×
    let job: JobSpec = serde_json::from_slice(&std::fs::read("/job.json").map_err(|e| format!("read /job.json: {e}"))?)
1380×
        .map_err(|e| format!("parse /job.json: {e}"))?;
1390×
    if job.argv.is_empty() {
1400×
        return Err("empty argv".into());
1410×
    }
1420×
    let nosuid = libc::MS_NOSUID | libc::MS_NODEV;
1430×
    std::fs::create_dir_all("/dev").ok();
1440×
    mount("devtmpfs", "/dev", "devtmpfs", libc::MS_NOSUID, "")?;
1450×
    std::fs::create_dir_all("/newroot").map_err(|e| e.to_string())?;
1460×
    mount("/dev/vda", "/newroot", "ext4", libc::MS_RDONLY | libc::MS_NODEV, "")?;
1470×
    mount("/dev/vdb", "/newroot/src", "ext4", nosuid, "")?;
1480×
    mount("tmpfs", "/newroot/tmp", "tmpfs", nosuid, "mode=1777")?;
1490×
    mount("proc", "/newroot/proc", "proc", nosuid | libc::MS_NOEXEC, "")?;
1500×
    mount("devtmpfs", "/newroot/dev", "devtmpfs", libc::MS_NOSUID, "")?;
1510×
    dev_links("/newroot/dev")?;
1520×
    if std::path::Path::new("/newroot/run").is_dir() {
1530×
        mount("tmpfs", "/newroot/run", "tmpfs", nosuid, "mode=755")?;
1540×
    }
155
    // The job runs as whoever owns the workspace (the host's builder), never root.
1560×
    let (uid, gid) = {
1570×
        let mut st: libc::stat = unsafe { std::mem::zeroed() };
1580×
        let p = cstr("/newroot/src")?;
1590×
        if unsafe { libc::stat(p.as_ptr(), &mut st) } != 0 {
1600×
            return Err("stat /src".into());
1610×
        }
1620×
        if st.st_uid == 0 {
1630×
            (1000, 1000)
164
        } else {
1650×
            (st.st_uid, st.st_gid)
166
        }
167
    };
168
    // Ephemeral mounts: mount points are made (and owned by the job's user)
169
    // under the writable /src or /tmp, then each is mounted and given to it.
1700×
    for m in &job.mounts {
1710×
        let target = format!("/newroot{}", m.target);
1720×
        make_dirs_owned(&target, uid, gid)?;
1730×
        match (m.kind.as_str(), &m.device) {
1740×
            ("tmpfs", _) => mount("tmpfs", &target, "tmpfs", nosuid, &format!("size={}m,uid={uid},gid={gid},mode=755", m.size_mib))?,
1750×
            ("disk", Some(dev)) => {
1760×
                let atime = if m.prune.is_some() { libc::MS_STRICTATIME | libc::MS_LAZYTIME } else { 0 };
1770×
                mount(dev, &target, "ext4", nosuid | atime, "")?;
1780×
                chown(&target, uid, gid)?;
179
            }
1800×
            (kind, _) => return Err(format!("unknown mount kind {kind}")),
181
        }
182
    }
183
    // Secrets: an in-memory mount of their own, only the job's user's,
184
    // each file 0400; the job file they came from is outside the new root.
1850×
    if !job.secrets.is_empty() {
1860×
        if !std::path::Path::new("/newroot/run").is_dir() {
1870×
            return Err("the image has no /run, where secrets are mounted".into());
1880×
        }
1890×
        let dir = "/newroot/run/secrets";
1900×
        std::fs::create_dir_all(dir).map_err(|e| format!("mkdir {dir}: {e}"))?;
1910×
        mount("tmpfs", dir, "tmpfs", nosuid | libc::MS_NOEXEC, &format!("size=1m,uid={uid},gid={gid},mode=700"))?;
1920×
        for s in &job.secrets {
1930×
            let Some(rel) = s.path.strip_prefix("/run/secrets/") else {
1940×
                return Err(format!("secret path {:?} is not under /run/secrets/", s.path));
195
            };
1960×
            let full = format!("{dir}/{rel}");
1970×
            if let Some(parent) = std::path::Path::new(&full).parent().and_then(|p| p.to_str()) {
1980×
                make_dirs_owned(parent, uid, gid)?;
1990×
            }
2000×
            std::fs::write(&full, s.value.as_bytes()).map_err(|e| format!("write {}: {e}", s.path))?;
2010×
            chown(&full, uid, gid)?;
2020×
            std::fs::set_permissions(&full, std::os::unix::fs::PermissionsExt::from_mode(0o400))
2030×
                .map_err(|e| format!("chmod {}: {e}", s.path))?;
204
        }
2050×
    }
2060×
    if job.loopback || job.egress {
2070×
        loopback_up()?;
2080×
    }
2090×
    if job.egress {
2100×
        egress::start()?;
2110×
    }
2120×
    let root = cstr("/newroot")?;
2130×
    if unsafe { libc::chroot(root.as_ptr()) } != 0 || std::env::set_current_dir("/src").is_err() {
2140×
        return Err(format!("enter the rootfs: {}", std::io::Error::last_os_error()));
2150×
    }
216
217
    // What the job reads or writes on a cache disk from now on is what
218
    // this run used.
2190×
    let started = job.mounts.iter().any(|m| m.prune.is_some()).then(prune::start_clock);
2200×
    let argv: Vec<CString> = job.argv.iter().map(|a| cstr(a)).collect::<Result<_, _>>()?;
221
    // glibc's execvpe searches the caller's PATH, and PID 1 has none of its
222
    // own, so the program is found here, in the job's PATH, before forking.
2230×
    let vars = job_env(std::fs::read_to_string("/etc/flow/env").ok().as_deref(), &job.env);
2240×
    let path = vars.get("PATH").map(String::as_str).unwrap_or(PATH);
2250×
    let program = cstr(&resolve_program(&job.argv[0], path).ok_or_else(|| format!("cannot run {:?}: not found in {path}", job.argv[0]))?)?;
2260×
    let env: Vec<CString> = vars.iter().map(|(k, v)| cstr(&format!("{k}={v}"))).collect::<Result<_, _>>()?;
2270×
    let mut out = [0; 2];
2280×
    let mut err = [0; 2];
2290×
    if unsafe { libc::pipe2(out.as_mut_ptr(), libc::O_CLOEXEC) } != 0 || unsafe { libc::pipe2(err.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
2300×
        return Err("pipe".into());
2310×
    }
2320×
    let pid = unsafe { libc::fork() };
2330×
    if pid < 0 {
2340×
        return Err("fork".into());
2350×
    }
2360×
    if pid == 0 {
237
        // Child: no way back to root, then the program itself (execvpe: no shell).
238
        unsafe {
2390×
            let null = libc::open(c"/dev/null".as_ptr(), libc::O_RDONLY);
2400×
            libc::dup2(null, 0);
2410×
            libc::dup2(out[1], 1);
2420×
            libc::dup2(err[1], 2);
2430×
            libc::setsid();
2440×
            if libc::setgroups(0, std::ptr::null()) != 0 || libc::setgid(gid) != 0 || libc::setuid(uid) != 0 {
2450×
                libc::_exit(126);
2460×
            }
2470×
            let mut a: Vec<*const libc::c_char> = argv.iter().map(|c| c.as_ptr()).collect();
2480×
            a.push(std::ptr::null());
2490×
            let mut e: Vec<*const libc::c_char> = env.iter().map(|c| c.as_ptr()).collect();
2500×
            e.push(std::ptr::null());
2510×
            libc::execve(program.as_ptr(), a.as_ptr(), e.as_ptr());
2520×
            let msg = format!("flow: cannot run {:?}: {}\n", job.argv[0], std::io::Error::last_os_error());
2530×
            libc::write(2, msg.as_ptr() as *const libc::c_void, msg.len());
2540×
            libc::_exit(127);
255
        }
2560×
    }
2570×
    unsafe {
2580×
        libc::close(out[1]);
2590×
        libc::close(err[1]);
2600×
    }
2610×
    let readers: Vec<_> = [(out[0], b'O'), (err[0], b'E')]
2620×
        .into_iter()
2630×
        .map(|(fd, kind)| {
2640×
            let console = console.clone();
2650×
            std::thread::spawn(move || pump(unsafe { std::fs::File::from_raw_fd(fd) }, kind, &console))
2660×
        })
2670×
        .collect();
2680×
    let mut status = 0;
269
    // Reap everything (we are PID 1) until the job itself exits.
270
    loop {
2710×
        let r = unsafe { libc::waitpid(-1, &mut status, 0) };
2720×
        if r == pid || r < 0 {
2730×
            break;
2740×
        }
275
    }
2760×
    for r in readers {
2770×
        let _ = r.join();
2780×
    }
2790×
    let code = if libc::WIFEXITED(status) { libc::WEXITSTATUS(status) } else { 128 + libc::WTERMSIG(status) };
280
    // A failing job's disks are thrown away: only a passing one's are
281
    // worth pruning.
2820×
    if let (Some(started), 0) = (started, code) {
2830×
        prune_caches(console, &job.mounts, started);
2840×
    }
2850×
    Ok(code)
2860×
}
287
288
/// Prune each cache disk and say so to the host. Whatever the job left
289
/// running stops first, so nothing changes the disks meanwhile.
2900×
fn prune_caches(console: &Mutex<std::io::Stdout>, mounts: &[MountSpec], started: std::time::SystemTime) {
291
    unsafe {
2920×
        libc::kill(-1, libc::SIGKILL);
2930×
        let mut status = 0;
2940×
        while libc::waitpid(-1, &mut status, 0) > 0 {}
295
    }
2960×
    for m in mounts {
2970×
        let Some(percent) = m.prune else { continue };
2980×
        let root = std::path::Path::new(&m.target);
2990×
        let pruned = match prune::usage(root) {
3000×
            Ok((used, size)) => {
3010×
                let mut p = prune::prune(root, started, &m.keep, size, used, percent).unwrap_or_else(|e| prune::Pruned {
3020×
                    size,
3030×
                    percent,
3040×
                    before: used,
3050×
                    after: used,
3060×
                    unpruned: Some(format!("pruning failed: {e}")),
3070×
                    ..Default::default()
3080×
                });
309
                // What is left as the file system counts it.
3100×
                if p.freed > 0 {
3110×
                    unsafe { libc::sync() };
3120×
                    if let Ok((now, _)) = prune::usage(root) {
3130×
                        p.after = now;
3140×
                    }
3150×
                }
3160×
                p
317
            }
3180×
            Err(e) => prune::Pruned { percent, unpruned: Some(format!("statvfs {}: {e}", m.target)), ..Default::default() },
319
        };
3200×
        let msg = serde_json::json!({ "target": m.target, "pruned": pruned });
3210×
        frame(console, b'C', msg.to_string().as_bytes());
322
    }
3230×
}
324
325
/// Frame a stream line by line; a long line is cut at 16 KiB.
3260×
fn pump(mut f: std::fs::File, kind: u8, console: &Mutex<std::io::Stdout>) {
3270×
    let mut buf = [0u8; 8192];
3280×
    let mut line: Vec<u8> = Vec::new();
329
    loop {
3300×
        let n = match f.read(&mut buf) {
3310×
            Ok(0) | Err(_) => break,
3320×
            Ok(n) => n,
333
        };
3340×
        for &b in &buf[..n] {
3350×
            if b == b'\n' || line.len() >= 16 * 1024 {
3360×
                frame(console, kind, &line);
3370×
                line.clear();
3380×
                if b == b'\n' {
3390×
                    continue;
3400×
                }
3410×
            }
3420×
            line.push(b);
343
        }
344
    }
3450×
    if !line.is_empty() {
3460×
        frame(console, kind, &line);
3470×
    }
3480×
}