Report outputs: a directory with report.toml, uploaded file by file with a manifest 6d53fb7d00ec

    // Outputs as [name, path] sorted by name, a report's with "report"

    // after them; absent when there are none.

        let output = |o: &crate::pipeline::Output| {

            let mut v = vec![text(&o.name), text(&o.path)];

            if !o.kind.is_artifact() {

                v.push(text(o.kind.word()));

            }

            Value::Array(v)

        };

        fields.push((7, Value::Array(j.outputs.iter().map(output).collect())));

    arch_ok, Identity, Job, Mount, MountSource, Network, Output, OutputKind, Pipeline, Problem, Problems, Runtime, Sandbox, Secret,

    PIPELINE_PATH,

    /// An artifact is one file; a report is a directory with a

    /// `report.toml`, kept file by file with a manifest (oxygenhub#127).

    #[serde(default, skip_serializing_if = "OutputKind::is_artifact")]

    pub kind: OutputKind,

/// What an [`Output`] is.

#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]

#[serde(rename_all = "lowercase")]

pub enum OutputKind {

    #[default]

    Artifact,

    Report,

}

impl OutputKind {

    pub fn is_artifact(&self) -> bool {

        *self == OutputKind::Artifact

    }

    pub fn word(&self) -> &'static str {

        match self {

            OutputKind::Artifact => "artifact",

            OutputKind::Report => "report",

        }

    }

}

        ctx.err(node.span(), "`output` takes a block of artifacts and reports");

        let kind = match a.name().value() {

            "artifact" => OutputKind::Artifact,

            "report" => OutputKind::Report,

            other => {

                ctx.err(

                    a.span(),

                    format!("unknown output `{other}`; outputs are `artifact \"name\" {{ path \"/src/file\" }}` and `report \"name\" {{ path \"/src/dir\" }}`"),

                );

                continue;

            }

        };

        let word = kind.word();

        if !ctx.name_ok(a.span(), word, &name) {

                    Some(p) => ctx

                        .err(c.span(), format!("{word} path {p:?} must be an absolute path under /src, without . or .., quotes or spaces")),

                "format" if kind == OutputKind::Artifact => {

                    Some(_) => ctx.err(c.span(), format!("{word} `when` is \"passed\" (the default) or \"always\"")),

                other => ctx.err(c.span(), format!("unknown {word} setting `{other}`")),

            ctx.err(a.span(), format!("{word} {name:?} needs a path"));

            ctx.err(a.span(), format!("output {name:?} is given twice (artifacts and reports share names)"));

            ctx.err(a.span(), format!("a job has at most {MAX_OUTPUTS} artifacts and reports"));

            job.outputs.push(Output { name, path, always, kind });

    // oxygenhub#127: a report is a directory output; it shares names with

    // artifacts, and its kind is part of the job's hash.

    let as_kind = |kind: &str| {

        Pipeline::parse(&src(&format!("    output {{\n        {kind} \"perf\" {{\n            path \"/src/target/report\"\n            when \"always\"\n        }}\n    }}\n"))).unwrap()

    };

    let (report, artifact) = (as_kind("report"), as_kind("artifact"));

    let o = &report.jobs["build"].outputs[0];

    assert!(o.kind == OutputKind::Report && o.always && o.path == "/src/target/report");

    assert!(serde_json::to_string(o).unwrap().contains("\"kind\":\"report\""));

    assert!(!serde_json::to_string(&artifact.jobs["build"].outputs[0]).unwrap().contains("kind"));

    assert_ne!(report.jobs["build"].hash(sb), artifact.jobs["build"].hash(sb));

    assert!(bad("    output {\n        artifact \"x\" {\n            path \"/src/a\"\n        }\n        report \"x\" {\n            path \"/src/b\"\n        }\n    }\n")

        .contains("given twice"));

    assert!(bad("    output {\n        report \"x\" {\n        }\n    }\n").contains("needs a path"));

/// Most files a report keeps (oxygenhub#127).

pub const MAX_REPORT_FILES: usize = 2000;

/// Most bytes a report's files hold together.

pub const MAX_REPORT_BYTES: u64 = 512 << 20;

/// Report directory `inner` of ext4 image `image`: dumped whole with

/// debugfs into `scratch`, then read as [`read_tree`] does.

pub fn read_report(image: &Path, inner: &str, scratch: &Path) -> Result<Vec<(String, Vec<u8>)>> {

    let stat = debugfs(image, &format!("stat \"{inner}\""))?;

    if !stat.contains("Type: directory") {

        bail!("{inner} is not a directory in the job's disk: a report is a directory holding report.toml");

    }

    let dest = scratch.join("report.out");

    let _ = std::fs::remove_dir_all(&dest);

    std::fs::create_dir_all(&dest)?;

    debugfs(image, &format!("rdump \"{inner}\" \"{}\"", dest.display()))?;

    let base = inner.trim_end_matches('/').rsplit('/').next().unwrap_or_default();

    let files = read_tree(&dest.join(base));

    let _ = std::fs::remove_dir_all(&dest);

    files

}

/// Report directory `inner` under `base` for the syscall tier: every

/// component of the path must be a directory, never a link, so a job

/// cannot point the runner outside its own tree.

pub fn read_report_dir(base: &Path, inner: &str) -> Result<Vec<(String, Vec<u8>)>> {

    let mut dir = base.to_path_buf();

    for part in inner.split('/').filter(|p| !p.is_empty()) {

        if part == "." || part == ".." {

            bail!("{inner} is not a clean path");

        }

        dir.push(part);

        let meta = std::fs::symlink_metadata(&dir).with_context(|| format!("{inner}: no such directory in the job's tree"))?;

        if !meta.is_dir() {

            bail!("{inner} is not a directory in the job's tree (a link is not followed): a report is a directory holding report.toml");

        }

    }

    read_tree(&dir)

}

/// A report file name: letters, digits, `.`, `_`, `-` and `+`, so a path

/// travels in a header and as a URL as it is.

fn report_name_ok(n: &str) -> bool {

    !n.is_empty() && n != "." && n != ".." && n.len() <= 255 && n.bytes().all(|b| b.is_ascii_alphanumeric() || b"._-+".contains(&b))

}

/// The regular files under host directory `root`, by `/`-separated path

/// relative to it, sorted. Links and anything that is not a file or a

/// directory are left out, never followed. Within the report limits, and

/// `report.toml` must be at the top.

pub fn read_tree(root: &Path) -> Result<Vec<(String, Vec<u8>)>> {

    let mut out: Vec<(String, Vec<u8>)> = Vec::new();

    let mut total = 0u64;

    let mut stack = vec![(root.to_path_buf(), String::new())];

    while let Some((dir, rel)) = stack.pop() {

        for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? {

            let entry = entry?;

            let name = entry.file_name().into_string().map_err(|_| anyhow::anyhow!("a report file name is not UTF-8"))?;

            let path = if rel.is_empty() { name.clone() } else { format!("{rel}/{name}") };

            if !report_name_ok(&name) {

                bail!("report file {path:?}: names may hold letters, digits, '.', '_', '-' and '+'");

            }

            let meta = std::fs::symlink_metadata(entry.path())?;

            if meta.is_dir() {

                stack.push((entry.path(), path));

            } else if meta.is_file() {

                if meta.len() > MAX_ARTIFACT_BYTES {

                    bail!("report file {path} is {} MiB; a file is at most {} MiB", meta.len() >> 20, MAX_ARTIFACT_BYTES >> 20);

                }

                total += meta.len();

                if total > MAX_REPORT_BYTES {

                    bail!("the report holds more than {} MiB", MAX_REPORT_BYTES >> 20);

                }

                if out.len() == MAX_REPORT_FILES {

                    bail!("the report holds more than {MAX_REPORT_FILES} files");

                }

                out.push((path, std::fs::read(entry.path())?));

            }

        }

    }

    if !out.iter().any(|(p, _)| p == "report.toml") {

        bail!("the report directory has no report.toml at its top");

    }

    out.sort();

    Ok(out)

}

    fn reports_are_read_as_trees_without_following_links() {

        let dir = tempfile::tempdir().unwrap();

        let tree = dir.path().join("tree");

        let report = tree.join("out/report");

        std::fs::create_dir_all(report.join("data")).unwrap();

        std::fs::write(report.join("report.toml"), b"kind = \"custom\"\n").unwrap();

        std::fs::write(report.join("index.html"), b"<p>hi</p>").unwrap();

        std::fs::write(report.join("data/frames.csv"), b"1,2\n").unwrap();

        std::fs::write(dir.path().join("secret"), b"runner key").unwrap();

        std::os::unix::fs::symlink(dir.path().join("secret"), report.join("data/leak")).unwrap();

        std::os::unix::fs::symlink(dir.path(), report.join("up")).unwrap();

        let files = read_report_dir(&tree, "/out/report").unwrap();

        let paths: Vec<&str> = files.iter().map(|(p, _)| p.as_str()).collect();

        assert_eq!(paths, ["data/frames.csv", "index.html", "report.toml"], "sorted, links left out");

        assert_eq!(files[0].1, b"1,2\n");

        std::os::unix::fs::symlink(&report, tree.join("linked")).unwrap();

        assert!(read_report_dir(&tree, "/linked").is_err(), "a linked report directory");

        assert!(read_report_dir(&tree, "/out/../out/report").is_err());

        assert!(read_report_dir(&tree, "/out/report/index.html").is_err(), "a file");

        std::fs::remove_file(report.join("report.toml")).unwrap();

        assert!(format!("{:#}", read_tree(&report).unwrap_err()).contains("report.toml"));

        std::fs::write(report.join("report.toml"), b"").unwrap();

        std::fs::write(report.join("bad name"), b"").unwrap();

        assert!(read_tree(&report).is_err(), "a name that cannot travel as is");

    }

    #[test]

        std::fs::create_dir_all(tree.join("report/data")).unwrap();

        std::fs::write(tree.join("report/report.toml"), b"kind = \"custom\"\n").unwrap();

        std::fs::write(tree.join("report/data/a.csv"), &body).unwrap();

        std::os::unix::fs::symlink("/etc/passwd", tree.join("report/data/leak")).unwrap();

        crate::workspace::image(&tree, &image).unwrap();

        let files = read_report(&image, "/report", dir.path()).unwrap();

        assert_eq!(files.iter().map(|(p, _)| p.as_str()).collect::<Vec<_>>(), ["data/a.csv", "report.toml"]);

        assert_eq!(files[0].1, body);

        assert!(read_report(&image, "/release/tool", dir.path()).is_err(), "a file is not a report");

    /// Upload file `path` of report `name` of job `job` (oxygenhub#127);

    /// the hub keeps it as a blob and answers with its ObjectRef. The

    /// report's manifest follows through [`Hub::artifact`].

    pub fn report_file(&self, job: i64, name: &str, path: &str, bytes: &[u8]) -> Result<String> {

        let url = format!("{}/api/v1/flow/jobs/{job}/reports/{name}/files", self.base);

        let mut delays = RETRY_DELAYS.iter();

        loop {

            let req = self

                .agent

                .post(&url)

                .timeout(Duration::from_secs(600))

                .set("Authorization", &format!("Bearer {}", self.token))

                .set("Content-Type", "application/octet-stream")

                .set("X-Flow-Report-Path", path);

            match req.send_bytes(bytes) {

                Ok(r) => {

                    let v: serde_json::Value = r.into_json()?;

                    return v["blob"].as_str().map(String::from).context("the hub named no blob");

                }

                Err(ureq::Error::Status(code, r)) => {

                    let text = r.into_string().unwrap_or_default();

                    match delays.next() {

                        Some(d) if code >= 500 => std::thread::sleep(*d),

                        _ => bail!("{url}: HTTP {code}: {text}"),

                    }

                }

                Err(e) => bail!("{url}: {e}"),

            }

        }

    }

            if o.kind == flow_core::OutputKind::Report {

                let read = if syscall_tier {

                    let (dir, inner) = crate::outputs::locate_dir(&claim.job, &scratch.join("tree"), &scratch, &o.path);

                    crate::outputs::read_report_dir(&dir, &inner)

                } else {

                    let (image, inner) = crate::outputs::locate(&claim.job, &scratch.join("workspace.ext4"), &scratch, &o.path);

                    crate::outputs::read_report(&image, &inner, &scratch)

                };

                let kept = read.and_then(|files| upload_report(hub, claim.job_id, &o.name, &files));

                match kept {

                    Ok((blob, files, bytes)) => {

                        let unattested = if passed { "" } else { " (the job failed: not in its receipt)" };

                        let hex = blob.to_hex();

                        note(&mut sink, format!("report {}: {files} files, {bytes} bytes, manifest {}{unattested}", o.name, &hex[..12]));

                        if passed {

                            outputs.push(blob.0);

                        }

                    }

                    Err(e) if passed => {

                        output_error = Some(format!("report {}: {e:#}", o.name));

                        break;

                    }

                    Err(e) => note(&mut sink, format!("report {}: not kept: {e:#}", o.name)),

                }

                continue;

            }

/// Upload a report's files, then its manifest under the report's name, and

/// check the hub kept each as the blob it should (oxygenhub#127). Answers

/// the manifest's blob, the file count and their total size.

fn upload_report(hub: &Hub, job: i64, name: &str, files: &[(String, Vec<u8>)]) -> Result<(oxy_core::hash::ObjectRef, usize, u64)> {

    let mut listed = Vec::with_capacity(files.len());

    let mut total = 0u64;

    for (path, bytes) in files {

        let blob = Repo::blob_ref(bytes).to_hex();

        let named = hub.report_file(job, name, path, bytes).with_context(|| format!("uploading {path}"))?;

        if named != blob {

            bail!("the hub kept {path} as {named}, not {blob}");

        }

        total += bytes.len() as u64;

        listed.push(serde_json::json!({ "path": path, "blob": blob, "size": bytes.len() }));

    }

    let manifest = serde_json::to_vec(&serde_json::json!({ "version": 1, "files": listed }))?;

    let blob = Repo::blob_ref(&manifest);

    let named = hub.artifact(job, name, &manifest)?;

    if named != blob.to_hex() {

        bail!("the hub kept the manifest as {named}, not {}", blob.to_hex());

    }

    Ok((blob, files.len(), total))

}

5. A `report` output (OxygenHub `docs/reports.md`) names a directory

   holding `report.toml` at its top. The runner reads its regular files

   (links and anything else are left out, never followed; file names hold

   letters, digits, `.`, `_`, `-` and `+`; at most 2000 files and 512 MiB),

   uploads each to `POST /api/v1/flow/jobs/{job}/reports/{name}/files` with

   its path in `X-Flow-Report-Path`, then uploads a manifest,

   `{"version":1,"files":[{"path","blob","size"}]}`, as the output under

   the report's name, checking every blob the hub answers. The receipt

   names the manifest. Reports and artifacts share names; a report's kind

   is part of the job's hash (`[name, path, "report"]`).

```kdl

output {

    report "frame-timing" {

        path "/src/out/report"

        when "always"

    }

}

```