OXYGENHUB
# 1abadd4a2fa6 oxygenflow / crates / flow / src / outputs.rs
4.7 KiB · Rust · last changed A job's identity: identity blocks name an audience and a mount path 1 month ago source blame history raw
1
//! A job's artifacts (spec §5.2): read from its disks after the microVM
2
//! has stopped, with debugfs, so the runner needs no mount rights.
3
4
use anyhow::{bail, Context, Result};
5
use flow_core::{Job, MountSource};
6
use std::path::{Path, PathBuf};
7
8
/// Largest artifact a runner takes: 512 MiB.
9
pub const MAX_ARTIFACT_BYTES: u64 = 512 << 20;
10
11
/// Where guest path `path` lives on the host: the disk image, and the path
12
/// inside its file system. The workspace is `/src`; each `ephemeral://disk`
13
/// mount is `mount-N.ext4` in `scratch`, numbered as `microvm::run` does.
14
pub fn locate(job: &Job, workspace: &Path, scratch: &Path, path: &str) -> (PathBuf, String) {
15
    for (i, m) in job.mounts.iter().filter(|m| m.source == MountSource::Disk).enumerate() {
16
        if let Some(rest) = path.strip_prefix(&format!("{}/", m.target)) {
17
            return (scratch.join(format!("mount-{i}.ext4")), format!("/{rest}"));
18
        }
19
    }
20
    (workspace.to_path_buf(), path.strip_prefix("/src").unwrap_or(path).to_string())
21
}
22
23
fn debugfs(image: &Path, request: &str) -> Result<String> {
24
    let out = std::process::Command::new("debugfs").arg("-R").arg(request).arg(image).output().context("run debugfs (e2fsprogs)")?;
25
    if !out.status.success() {
26
        bail!("debugfs {request:?}: {}", String::from_utf8_lossy(&out.stderr).trim());
27
    }
28
    Ok(String::from_utf8_lossy(&out.stdout).into_owned())
29
}
30
31
/// Read regular file `inner` of ext4 image `image`, at most
32
/// [`MAX_ARTIFACT_BYTES`]. Paths come from a checked pipeline: absolute,
33
/// clean, and free of quotes and spaces.
34
pub fn read(image: &Path, inner: &str, scratch: &Path) -> Result<Vec<u8>> {
35
    let stat = debugfs(image, &format!("stat \"{inner}\""))?;
36
    if !stat.contains("Type: regular") {
37
        bail!("{inner} is not a regular file in the job's disk (it may be missing or a link)");
38
    }
39
    let size: u64 = stat
40
        .split("Size: ")
41
        .nth(1)
42
        .and_then(|s| s.split_whitespace().next())
43
        .and_then(|s| s.parse().ok())
44
        .context("debugfs gave no size")?;
45
    if size > MAX_ARTIFACT_BYTES {
46
        bail!("{inner} is {} MiB; artifacts are at most {} MiB", size >> 20, MAX_ARTIFACT_BYTES >> 20);
47
    }
48
    let dest = scratch.join("artifact.out");
49
    let _ = std::fs::remove_file(&dest);
50
    debugfs(image, &format!("dump \"{inner}\" \"{}\"", dest.display()))?;
51
    let bytes = std::fs::read(&dest).with_context(|| format!("{inner}: debugfs wrote nothing"))?;
52
    let _ = std::fs::remove_file(&dest);
53
    if bytes.len() as u64 != size {
54
        bail!("{inner}: read {} bytes of {size}", bytes.len());
55
    }
56
    Ok(bytes)
57
}
58
59
#[cfg(test)]
60
mod tests {
61
    use super::*;
62
    use flow_core::Mount;
63
64
    fn job() -> Job {
65
        Job {
66
            name: "build".into(),
67
            sandbox: "rust".into(),
68
            scope: vec![],
69
            depends_on: vec![],
70
            env: Default::default(),
71
            exec: vec!["make".into()],
72
            secrets: vec![],
73
            identities: vec![],
74
            mounts: vec![
75
                Mount { source: MountSource::Tmpfs, target: "/src/cache".into(), size_mib: 64 },
76
                Mount { source: MountSource::Disk, target: "/src/target".into(), size_mib: 1024 },
77
                Mount { source: MountSource::Disk, target: "/src/work".into(), size_mib: 1024 },
78
            ],
79
            outputs: vec![],
80
            runs_on: vec![],
81
            timeout_s: None,
82
            on: None,
83
        }
84
    }
85
86
    #[test]
87
    fn paths_are_found_on_the_disk_that_holds_them() {
88
        let (w, s) = (Path::new("/w.ext4"), Path::new("/scratch"));
89
        assert_eq!(locate(&job(), w, s, "/src/target/release/oxy"), (s.join("mount-0.ext4"), "/release/oxy".into()));
90
        assert_eq!(locate(&job(), w, s, "/src/work/a"), (s.join("mount-1.ext4"), "/a".into()));
91
        assert_eq!(locate(&job(), w, s, "/src/docs/a.1"), (w.to_path_buf(), "/docs/a.1".into()));
92
        assert_eq!(locate(&job(), w, s, "/src/targets/x"), (w.to_path_buf(), "/targets/x".into()), "a prefix is not a parent");
93
    }
94
95
    #[test]
96
    fn files_are_read_back_from_an_ext4_image() {
97
        let dir = tempfile::tempdir().unwrap();
98
        let tree = dir.path().join("tree");
99
        std::fs::create_dir_all(tree.join("release")).unwrap();
100
        let body: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect();
101
        std::fs::write(tree.join("release/tool"), &body).unwrap();
102
        std::os::unix::fs::symlink("tool", tree.join("release/link")).unwrap();
103
        let image = dir.path().join("disk.ext4");
104
        crate::workspace::image(&tree, &image).unwrap();
105
        assert_eq!(read(&image, "/release/tool", dir.path()).unwrap(), body);
106
        assert!(read(&image, "/release/missing", dir.path()).is_err());
107
        assert!(read(&image, "/release/link", dir.path()).unwrap_err().to_string().contains("not a regular file"));
108
    }
109
}