OXYGENHUB
# 062c053ef8b1 oxygenflow / crates / flow-init / src / main.rs
8.8 KiB · Rust · last changed Ephemeral scratch mounts in M0: ephemeral://tmpfs and ephemeral://disk (a sparse disk per job), checked targets and sizes, in the canonical job and the receipt; flow-init mounts them for the job's user; make deploy-runner 1 month ago source blame history raw
e0918b88 Andy McCall 1
//! PID 1 of a job's microVM (docs/m0.md §2). It mounts the read-only
2
//! rootfs and the workspace, runs the job's argv as an unprivileged user
3
//! with exactly the job's environment, frames the output on the serial
4
//! console for the host, and powers off. It never decides anything the
5
//! host has not: the job comes from `/job.json` in the initramfs the host
6
//! built, and the host enforces the time limit.
7
//!
8
//! Frames, one per line on the console: `\x1eO ` stdout, `\x1eE ` stderr,
9
//! `\x1eX <code>` the exit code (128 + signal when killed), `\x1eF <text>`
10
//! a failure of init itself.
11
12
use serde::Deserialize;
13
use std::ffi::CString;
14
use std::io::{Read, Write};
15
use std::os::fd::FromRawFd;
16
use std::sync::{Arc, Mutex};
17
18
#[derive(Deserialize)]
19
struct JobSpec {
20
    argv: Vec<String>,
21
    env: Vec<(String, String)>,
fc10d9e8 Andy McCall ↶ 22
    #[serde(default)]
23
    mounts: Vec<MountSpec>,
24
}
25
26
/// An ephemeral mount from the host: a tmpfs of `size_mib`, or the disk at
27
/// `device`, mounted at `target` under the new root.
28
#[derive(serde::Deserialize)]
29
struct MountSpec {
30
    kind: String,
31
    target: String,
32
    size_mib: u32,
33
    #[serde(default)]
34
    device: Option<String>,
e0918b88 Andy McCall 35
}
36
37
const PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
38
39
fn main() {
40
    let console = Arc::new(Mutex::new(std::io::stdout()));
41
    let code = match run(&console) {
42
        Ok(code) => code,
43
        Err(e) => {
44
            frame(&console, b'F', e.as_bytes());
45
            125
46
        }
47
    };
48
    frame(&console, b'X', code.to_string().as_bytes());
49
    unsafe {
50
        libc::sync();
51
        // Firecracker ends the VM when the guest reboots (reboot=k).
52
        libc::reboot(libc::LINUX_REBOOT_CMD_RESTART);
53
    }
54
}
55
56
fn frame(console: &Mutex<std::io::Stdout>, kind: u8, text: &[u8]) {
57
    let mut out = console.lock().unwrap_or_else(|p| p.into_inner());
58
    let mut line = Vec::with_capacity(text.len() + 4);
59
    line.extend_from_slice(&[0x1e, kind, b' ']);
60
    line.extend_from_slice(text);
61
    line.push(b'\n');
62
    let _ = out.write_all(&line);
63
    let _ = out.flush();
64
}
65
66
fn cstr(s: &str) -> Result<CString, String> {
67
    CString::new(s).map_err(|_| format!("{s:?} contains NUL"))
68
}
69
fc10d9e8 Andy McCall ↶ 70
fn chown(path: &str, uid: libc::uid_t, gid: libc::gid_t) -> Result<(), String> {
71
    let p = cstr(path)?;
72
    if unsafe { libc::chown(p.as_ptr(), uid, gid) } != 0 {
73
        return Err(format!("chown {path}: {}", std::io::Error::last_os_error()));
74
    }
75
    Ok(())
76
}
77
78
/// Create `path` and its missing parents, each owned by the job's user.
79
fn make_dirs_owned(path: &str, uid: libc::uid_t, gid: libc::gid_t) -> Result<(), String> {
80
    let mut missing = Vec::new();
81
    let mut p = std::path::Path::new(path);
82
    while !p.exists() {
83
        missing.push(p.to_path_buf());
84
        p = match p.parent() {
85
            Some(parent) => parent,
86
            None => break,
87
        };
88
    }
89
    for d in missing.iter().rev() {
90
        std::fs::create_dir(d).map_err(|e| format!("mkdir {}: {e}", d.display()))?;
91
        chown(&d.to_string_lossy(), uid, gid)?;
92
    }
93
    Ok(())
94
}
95
e0918b88 Andy McCall 96
fn mount(src: &str, target: &str, fstype: &str, flags: libc::c_ulong, data: &str) -> Result<(), String> {
97
    let (s, t, f, d) = (cstr(src)?, cstr(target)?, cstr(fstype)?, cstr(data)?);
98
    let rc = unsafe { libc::mount(s.as_ptr(), t.as_ptr(), f.as_ptr(), flags, d.as_ptr() as *const libc::c_void) };
99
    if rc != 0 {
100
        return Err(format!("mount {src} on {target}: {}", std::io::Error::last_os_error()));
101
    }
102
    Ok(())
103
}
104
105
fn run(console: &Arc<Mutex<std::io::Stdout>>) -> Result<i32, String> {
106
    let job: JobSpec = serde_json::from_slice(&std::fs::read("/job.json").map_err(|e| format!("read /job.json: {e}"))?)
107
        .map_err(|e| format!("parse /job.json: {e}"))?;
108
    if job.argv.is_empty() {
109
        return Err("empty argv".into());
110
    }
111
    let nosuid = libc::MS_NOSUID | libc::MS_NODEV;
112
    std::fs::create_dir_all("/dev").ok();
113
    mount("devtmpfs", "/dev", "devtmpfs", libc::MS_NOSUID, "")?;
114
    std::fs::create_dir_all("/newroot").map_err(|e| e.to_string())?;
115
    mount("/dev/vda", "/newroot", "ext4", libc::MS_RDONLY | libc::MS_NODEV, "")?;
116
    mount("/dev/vdb", "/newroot/src", "ext4", nosuid, "")?;
117
    mount("tmpfs", "/newroot/tmp", "tmpfs", nosuid, "mode=1777")?;
118
    mount("proc", "/newroot/proc", "proc", nosuid | libc::MS_NOEXEC, "")?;
119
    mount("devtmpfs", "/newroot/dev", "devtmpfs", libc::MS_NOSUID, "")?;
120
    if std::path::Path::new("/newroot/run").is_dir() {
121
        mount("tmpfs", "/newroot/run", "tmpfs", nosuid, "mode=755")?;
122
    }
123
    // The job runs as whoever owns the workspace (the host's builder), never root.
124
    let (uid, gid) = {
125
        let mut st: libc::stat = unsafe { std::mem::zeroed() };
126
        let p = cstr("/newroot/src")?;
127
        if unsafe { libc::stat(p.as_ptr(), &mut st) } != 0 {
128
            return Err("stat /src".into());
129
        }
130
        if st.st_uid == 0 {
131
            (1000, 1000)
132
        } else {
133
            (st.st_uid, st.st_gid)
134
        }
135
    };
fc10d9e8 Andy McCall ↶ 136
    // Ephemeral mounts: mount points are made (and owned by the job's user)
137
    // under the writable /src or /tmp, then each is mounted and given to it.
138
    for m in &job.mounts {
139
        let target = format!("/newroot{}", m.target);
140
        make_dirs_owned(&target, uid, gid)?;
141
        match (m.kind.as_str(), &m.device) {
142
            ("tmpfs", _) => mount("tmpfs", &target, "tmpfs", nosuid, &format!("size={}m,uid={uid},gid={gid},mode=755", m.size_mib))?,
143
            ("disk", Some(dev)) => {
144
                mount(dev, &target, "ext4", nosuid, "")?;
145
                chown(&target, uid, gid)?;
146
            }
147
            (kind, _) => return Err(format!("unknown mount kind {kind}")),
148
        }
149
    }
e0918b88 Andy McCall 150
    let root = cstr("/newroot")?;
151
    if unsafe { libc::chroot(root.as_ptr()) } != 0 || std::env::set_current_dir("/src").is_err() {
152
        return Err(format!("enter the rootfs: {}", std::io::Error::last_os_error()));
153
    }
154
155
    let argv: Vec<CString> = job.argv.iter().map(|a| cstr(a)).collect::<Result<_, _>>()?;
156
    let mut env: Vec<CString> = vec![cstr(&format!("PATH={PATH}"))?, cstr("HOME=/tmp")?];
157
    for (k, v) in &job.env {
158
        env.push(cstr(&format!("{k}={v}"))?);
159
    }
160
    let mut out = [0; 2];
161
    let mut err = [0; 2];
162
    if unsafe { libc::pipe2(out.as_mut_ptr(), libc::O_CLOEXEC) } != 0 || unsafe { libc::pipe2(err.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
163
        return Err("pipe".into());
164
    }
165
    let pid = unsafe { libc::fork() };
166
    if pid < 0 {
167
        return Err("fork".into());
168
    }
169
    if pid == 0 {
170
        // Child: no way back to root, then the program itself (execvpe: no shell).
171
        unsafe {
172
            let null = libc::open(c"/dev/null".as_ptr(), libc::O_RDONLY);
173
            libc::dup2(null, 0);
174
            libc::dup2(out[1], 1);
175
            libc::dup2(err[1], 2);
176
            libc::setsid();
177
            if libc::setgroups(0, std::ptr::null()) != 0 || libc::setgid(gid) != 0 || libc::setuid(uid) != 0 {
178
                libc::_exit(126);
179
            }
180
            let mut a: Vec<*const libc::c_char> = argv.iter().map(|c| c.as_ptr()).collect();
181
            a.push(std::ptr::null());
182
            let mut e: Vec<*const libc::c_char> = env.iter().map(|c| c.as_ptr()).collect();
183
            e.push(std::ptr::null());
184
            libc::execvpe(a[0], a.as_ptr(), e.as_ptr());
185
            let msg = format!("flow: cannot run {:?}: {}\n", job.argv[0], std::io::Error::last_os_error());
186
            libc::write(2, msg.as_ptr() as *const libc::c_void, msg.len());
187
            libc::_exit(127);
188
        }
189
    }
190
    unsafe {
191
        libc::close(out[1]);
192
        libc::close(err[1]);
193
    }
194
    let readers: Vec<_> = [(out[0], b'O'), (err[0], b'E')]
195
        .into_iter()
196
        .map(|(fd, kind)| {
197
            let console = console.clone();
198
            std::thread::spawn(move || pump(unsafe { std::fs::File::from_raw_fd(fd) }, kind, &console))
199
        })
200
        .collect();
201
    let mut status = 0;
202
    // Reap everything (we are PID 1) until the job itself exits.
203
    loop {
204
        let r = unsafe { libc::waitpid(-1, &mut status, 0) };
205
        if r == pid || r < 0 {
206
            break;
207
        }
208
    }
209
    for r in readers {
210
        let _ = r.join();
211
    }
212
    Ok(if libc::WIFEXITED(status) { libc::WEXITSTATUS(status) } else { 128 + libc::WTERMSIG(status) })
213
}
214
215
/// Frame a stream line by line; a long line is cut at 16 KiB.
216
fn pump(mut f: std::fs::File, kind: u8, console: &Mutex<std::io::Stdout>) {
217
    let mut buf = [0u8; 8192];
218
    let mut line: Vec<u8> = Vec::new();
219
    loop {
220
        let n = match f.read(&mut buf) {
221
            Ok(0) | Err(_) => break,
222
            Ok(n) => n,
223
        };
224
        for &b in &buf[..n] {
225
            if b == b'\n' || line.len() >= 16 * 1024 {
226
                frame(console, kind, &line);
227
                line.clear();
228
                if b == b'\n' {
229
                    continue;
230
                }
231
            }
232
            line.push(b);
233
        }
234
    }
235
    if !line.is_empty() {
236
        frame(console, kind, &line);
237
    }
238
}