Coverage / crates/flow/src/outputs.rs 95.0%

pub fn locate(job: &Job, workspace: &Path, scratch: &Path, path: &str) -> (PathBuf, String) {

    locate_in(job, workspace, path, |i| scratch.join(format!("mount-{i}.ext4")))

}

pub fn locate_dir(job: &Job, workspace: &Path, scratch: &Path, path: &str) -> (PathBuf, String) {

    locate_in(job, workspace, path, |i| crate::syscall::disk_dir(scratch, i))

}

fn locate_in(job: &Job, workspace: &Path, path: &str, disk: impl Fn(usize) -> PathBuf) -> (PathBuf, String) {

    for (i, m) in job.mounts.iter().filter(|m| m.source == MountSource::Disk).enumerate() {

        if let Some(rest) = path.strip_prefix(&format!("{}/", m.target)) {

            return (disk(i), format!("/{rest}"));

        }

    (workspace.to_path_buf(), path.strip_prefix("/src").unwrap_or(path).to_string())

}

pub fn read_dir_file(base: &Path, inner: &str) -> Result<Vec<u8>> {

    let parts: Vec<&str> = inner.split('/').filter(|p| !p.is_empty()).collect();

    if parts.is_empty() || parts.iter().any(|p| *p == "." || *p == "..") {

        bail!("{inner} is not a clean path");

    }

    let c = |s: &str| CString::new(s).context("a path holds NUL");

    let mut dir = unsafe { libc::open(c(&base.to_string_lossy())?.as_ptr(), libc::O_DIRECTORY | libc::O_RDONLY | libc::O_CLOEXEC) };

    if dir < 0 {

    }

    let (last, dirs) = parts.split_last().unwrap();

    for d in dirs {

        let next = unsafe { libc::openat(dir, c(d)?.as_ptr(), libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_RDONLY | libc::O_CLOEXEC) };

        let err = std::io::Error::last_os_error();

        unsafe { libc::close(dir) };

        if next < 0 {

            bail!("{inner}: {d} is not a directory in the job's files ({err})");

        }

        dir = next;

    let fd = unsafe { libc::openat(dir, c(last)?.as_ptr(), libc::O_NOFOLLOW | libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NONBLOCK) };

    let err = std::io::Error::last_os_error();

    unsafe { libc::close(dir) };

    if fd < 0 {

        bail!("{inner} is not a regular file in the job's files ({err})");

    }

    let file = unsafe { std::fs::File::from_raw_fd(fd) };

    let meta = file.metadata()?;

    if !meta.is_file() {

        bail!("{inner} is not a regular file in the job's files (it may be missing or a link)");

    }

    if meta.len() > MAX_ARTIFACT_BYTES {

    }

    let mut bytes = Vec::with_capacity(meta.len() as usize);

    file.take(MAX_ARTIFACT_BYTES + 1).read_to_end(&mut bytes)?;

    if bytes.len() as u64 > MAX_ARTIFACT_BYTES {

    }

    Ok(bytes)

}

pub fn read_report(image: &Path, inner: &str, scratch: &Path) -> Result<Vec<(String, Vec<u8>)>> {

    let stat = debugfs(image, &format!("stat \"{inner}\""))?;

    if !stat.contains("Type: directory") {

        bail!("{inner} is not a directory in the job's disk: a report is a directory holding report.toml");

    }

    let dest = scratch.join("report.out");

    let _ = std::fs::remove_dir_all(&dest);

    std::fs::create_dir_all(&dest)?;

    debugfs(image, &format!("rdump \"{inner}\" \"{}\"", dest.display()))?;

    let base = inner.trim_end_matches('/').rsplit('/').next().unwrap_or_default();

    let files = read_tree(&dest.join(base));

    let _ = std::fs::remove_dir_all(&dest);

    files

}

pub fn read_report_dir(base: &Path, inner: &str) -> Result<Vec<(String, Vec<u8>)>> {

    let mut dir = base.to_path_buf();

    for part in inner.split('/').filter(|p| !p.is_empty()) {

        if part == "." || part == ".." {

            bail!("{inner} is not a clean path");

        }

        dir.push(part);

        let meta = std::fs::symlink_metadata(&dir).with_context(|| format!("{inner}: no such directory in the job's tree"))?;

        if !meta.is_dir() {

            bail!("{inner} is not a directory in the job's tree (a link is not followed): a report is a directory holding report.toml");

        }

    read_tree(&dir)

}

fn report_name_ok(n: &str) -> bool {

    !n.is_empty() && n != "." && n != ".." && n.len() <= 255 && n.bytes().all(|b| b.is_ascii_alphanumeric() || b"._-+".contains(&b))

}

pub fn read_tree(root: &Path) -> Result<Vec<(String, Vec<u8>)>> {

    let mut out: Vec<(String, Vec<u8>)> = Vec::new();

    let mut total = 0u64;

    let mut stack = vec![(root.to_path_buf(), String::new())];

    while let Some((dir, rel)) = stack.pop() {

        for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? {

            let entry = entry?;

            let name = entry.file_name().into_string().map_err(|_| anyhow::anyhow!("a report file name is not UTF-8"))?;

            let path = if rel.is_empty() { name.clone() } else { format!("{rel}/{name}") };

            if !report_name_ok(&name) {

                bail!("report file {path:?}: names may hold letters, digits, '.', '_', '-' and '+'");

            }

            let meta = std::fs::symlink_metadata(entry.path())?;

            if meta.is_dir() {

                stack.push((entry.path(), path));

            } else if meta.is_file() {

                if meta.len() > MAX_ARTIFACT_BYTES {

                }

                total += meta.len();

                if total > MAX_REPORT_BYTES {

                }

                if out.len() == MAX_REPORT_FILES {

                }

                out.push((path, std::fs::read(entry.path())?));

            }

    if !out.iter().any(|(p, _)| p == "report.toml") {

        bail!("the report directory has no report.toml at its top");

    }

    out.sort();

    Ok(out)

}

fn debugfs(image: &Path, request: &str) -> Result<String> {

    let out = std::process::Command::new("debugfs").arg("-R").arg(request).arg(image).output().context("run debugfs (e2fsprogs)")?;

    if !out.status.success() {

    }

    Ok(String::from_utf8_lossy(&out.stdout).into_owned())

}

pub fn read(image: &Path, inner: &str, scratch: &Path) -> Result<Vec<u8>> {

    let stat = debugfs(image, &format!("stat \"{inner}\""))?;

    if !stat.contains("Type: regular") {

        bail!("{inner} is not a regular file in the job's disk (it may be missing or a link)");

    }

    let size: u64 = stat

        .split("Size: ")

        .nth(1)

        .and_then(|s| s.split_whitespace().next())

        .and_then(|s| s.parse().ok())

        .context("debugfs gave no size")?;

    if size > MAX_ARTIFACT_BYTES {

    }

    let dest = scratch.join("artifact.out");

    let _ = std::fs::remove_file(&dest);

    debugfs(image, &format!("dump \"{inner}\" \"{}\"", dest.display()))?;

    let bytes = std::fs::read(&dest).with_context(|| format!("{inner}: debugfs wrote nothing"))?;

    let _ = std::fs::remove_file(&dest);

    if bytes.len() as u64 != size {

    }

    Ok(bytes)

}

    fn job() -> Job {

        Job {

            name: "build".into(),

            sandbox: "rust".into(),

            scope: vec![],

            depends_on: vec![],

            env: Default::default(),

            exec: vec!["make".into()],

            secrets: vec![],

            identities: vec![],

            mounts: vec![

                Mount { source: MountSource::Tmpfs, target: "/src/cache".into(), size_mib: 64, cache: None, key: vec![] },

                Mount { source: MountSource::Disk, target: "/src/target".into(), size_mib: 1024, cache: None, key: vec![] },

                Mount { source: MountSource::Disk, target: "/src/work".into(), size_mib: 1024, cache: None, key: vec![] },

            ],

            outputs: vec![],

            inputs: vec![],

            approval: None,

            environment: None,

            runs_on: vec![],

            timeout_s: None,

            on: None,

            priority: None,

            checkpoints: Vec::new(),

        }

    }

    fn paths_are_found_on_the_disk_that_holds_them() {

        let (w, s) = (Path::new("/w.ext4"), Path::new("/scratch"));

        assert_eq!(locate(&job(), w, s, "/src/target/release/oxy"), (s.join("mount-0.ext4"), "/release/oxy".into()));

        assert_eq!(locate(&job(), w, s, "/src/work/a"), (s.join("mount-1.ext4"), "/a".into()));

        assert_eq!(locate(&job(), w, s, "/src/docs/a.1"), (w.to_path_buf(), "/docs/a.1".into()));

        assert_eq!(locate(&job(), w, s, "/src/targets/x"), (w.to_path_buf(), "/targets/x".into()), "a prefix is not a parent");

    }

    fn files_are_read_from_directories_without_following_links() {

        let dir = tempfile::tempdir().unwrap();

        let tree = dir.path().join("tree");

        std::fs::create_dir_all(tree.join("release")).unwrap();

        std::fs::write(tree.join("release/tool"), b"built").unwrap();

        std::fs::write(dir.path().join("secret"), b"runner key").unwrap();

        std::os::unix::fs::symlink(dir.path().join("secret"), tree.join("release/link")).unwrap();

        std::os::unix::fs::symlink(dir.path(), tree.join("up")).unwrap();

        assert_eq!(read_dir_file(&tree, "/release/tool").unwrap(), b"built");

        assert!(read_dir_file(&tree, "/release/link").is_err(), "a link to a host file");

        assert!(read_dir_file(&tree, "/up/secret").is_err(), "a linked directory");

        assert!(read_dir_file(&tree, "/release/../release/tool").is_err());

        assert!(read_dir_file(&tree, "/release").is_err(), "a directory");

        let s = Path::new("/scratch");

        assert_eq!(locate_dir(&job(), &tree, s, "/src/target/a"), (s.join("mount-0"), "/a".into()));

    }

    fn reports_are_read_as_trees_without_following_links() {

        let dir = tempfile::tempdir().unwrap();

        let tree = dir.path().join("tree");

        let report = tree.join("out/report");

        std::fs::create_dir_all(report.join("data")).unwrap();

        std::fs::write(report.join("report.toml"), b"kind = \"custom\"\n").unwrap();

        std::fs::write(report.join("index.html"), b"<p>hi</p>").unwrap();

        std::fs::write(report.join("data/frames.csv"), b"1,2\n").unwrap();

        std::fs::write(dir.path().join("secret"), b"runner key").unwrap();

        std::os::unix::fs::symlink(dir.path().join("secret"), report.join("data/leak")).unwrap();

        std::os::unix::fs::symlink(dir.path(), report.join("up")).unwrap();

        let files = read_report_dir(&tree, "/out/report").unwrap();

        let paths: Vec<&str> = files.iter().map(|(p, _)| p.as_str()).collect();

        assert_eq!(paths, ["data/frames.csv", "index.html", "report.toml"], "sorted, links left out");

        assert_eq!(files[0].1, b"1,2\n");

        std::os::unix::fs::symlink(&report, tree.join("linked")).unwrap();

        assert!(read_report_dir(&tree, "/linked").is_err(), "a linked report directory");

        assert!(read_report_dir(&tree, "/out/../out/report").is_err());

        assert!(read_report_dir(&tree, "/out/report/index.html").is_err(), "a file");

        std::fs::remove_file(report.join("report.toml")).unwrap();

        assert!(format!("{:#}", read_tree(&report).unwrap_err()).contains("report.toml"));

        std::fs::write(report.join("report.toml"), b"").unwrap();

        std::fs::write(report.join("bad name"), b"").unwrap();

        assert!(read_tree(&report).is_err(), "a name that cannot travel as is");

    }

    fn files_are_read_back_from_an_ext4_image() {

        let dir = tempfile::tempdir().unwrap();

        let tree = dir.path().join("tree");

        std::fs::create_dir_all(tree.join("release")).unwrap();

        let body: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect();

        std::fs::write(tree.join("release/tool"), &body).unwrap();

        std::os::unix::fs::symlink("tool", tree.join("release/link")).unwrap();

        let image = dir.path().join("disk.ext4");

        crate::workspace::image(&tree, &image).unwrap();

        assert_eq!(read(&image, "/release/tool", dir.path()).unwrap(), body);

        assert!(read(&image, "/release/missing", dir.path()).is_err());

        assert!(read(&image, "/release/link", dir.path()).unwrap_err().to_string().contains("not a regular file"));

        std::fs::create_dir_all(tree.join("report/data")).unwrap();

        std::fs::write(tree.join("report/report.toml"), b"kind = \"custom\"\n").unwrap();

        std::fs::write(tree.join("report/data/a.csv"), &body).unwrap();

        std::os::unix::fs::symlink("/etc/passwd", tree.join("report/data/leak")).unwrap();

        crate::workspace::image(&tree, &image).unwrap();

        let files = read_report(&image, "/report", dir.path()).unwrap();

        assert_eq!(files.iter().map(|(p, _)| p.as_str()).collect::<Vec<_>>(), ["data/a.csv", "report.toml"]);

        assert_eq!(files[0].1, body);

        assert!(read_report(&image, "/release/tool", dir.path()).is_err(), "a file is not a report");

    }