lib.rs 94.5%

451 of 477 lines run, 94 of 132 functions · run #406 on main · run missed
//! Cylinder, step C1 (docs/cylinder.md §4.1): an **artifact** is an
//! immutable, content-addressed object naming a tree of files, each kept as
//! an ordinary Oxygen blob of FastCDC BLAKE3 chunks. An image rebuilt with
//! one changed file moves only that file's changed chunks, and a binary in
//! a release archive and in a runner image is the same chunks.
//!
//! An artifact is its canonical CBOR (spec §5.1), named by the BLAKE3 of
//! that encoding. Until the artifact is an object type of its own (the rest
//! of C1), the encoding and its tree are kept as blobs in the repository's
//! store, and `.oxy/artifacts/NAME` holds the encoding's blob so the
//! artifact can be found and garbage collection keeps it ([`roots`]).

use oxy_core::cbor::{self, Value};
use oxy_core::hash::ObjectRef;
use oxy_core::repo::Repo;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};

/// The encoding's version this crate writes and reads.
pub const VERSION: u64 = 1;

#[derive(Debug, thiserror::Error)]
pub enum Error {
    #[error(transparent)]
    Repo(#[from] oxy_core::repo::RepoError),
    #[error("io: {0}")]
    Io(#[from] std::io::Error),
    #[error("not a canonical artifact encoding: {0}")]
    Cbor(#[from] oxy_core::cbor::CborError),
    #[error("{0}")]
    Bad(String),
}

pub type Result<T> = std::result::Result<T, Error>;

fn bad<T>(msg: impl Into<String>) -> Result<T> {
    Err(Error::Bad(msg.into()))
}

/// What an artifact is for.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Kind {
    Container,
    RunnerImage,
    Package,
    Archive,
    Report,
}

impl Kind {
    pub const ALL: [Kind; 5] = [
        Kind::Container,
        Kind::RunnerImage,
        Kind::Package,
        Kind::Archive,
        Kind::Report,
    ];

    pub fn as_str(self) -> &'static str {
        match self {
            Kind::Container => "container",
            Kind::RunnerImage => "runner-image",
            Kind::Package => "package",
            Kind::Archive => "archive",
            Kind::Report => "report",
        }
    }

    pub fn parse(s: &str) -> Result<Kind> {
        Kind::ALL.into_iter().find(|k| k.as_str() == s).map_or_else(
            || {
                bad(format!(
                    "{s:?} is not a kind of artifact: {}",
                    Kind::ALL.map(|k| k.as_str()).join(", ")
                ))
            },
            Ok,
        )
    }
}

/// A file's mode in a tree: a regular file, an executable one, or a
/// symbolic link (whose blob is its target).
pub const MODE_FILE: u32 = 0o100644;
pub const MODE_EXEC: u32 = 0o100755;
pub const MODE_LINK: u32 = 0o120000;

/// A file of an artifact's tree.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Entry {
    /// `/`-separated, relative, never `.` or `..` parts.
    pub path: String,
    pub mode: u32,
    pub blob: ObjectRef,
    pub size: u64,
}

/// An artifact's files, in path order: paths, modes, blobs and sizes.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct DirTree {
    pub entries: Vec<Entry>,
}

fn path_ok(p: &str) -> bool {
    !p.is_empty()
        && !p.starts_with('/')
        && p.split('/').all(|s| !s.is_empty() && s != "." && s != "..")
}

fn obj(r: &ObjectRef) -> Value {
    Value::Bytes(r.0.to_vec())
}

fn get_obj(v: &Value, what: &str) -> Result<ObjectRef> {
    match v.as_bytes() {
        Some(b) if b.len() == 32 => {
            let mut a = [0u8; 32];
            a.copy_from_slice(b);
            Ok(ObjectRef(a))
        }
        _ => bad(format!("{what} is not an object reference")),
    }
}

fn get_text(v: &Value, what: &str) -> Result<String> {
    v.as_text()
        .map(str::to_string)
        .map_or_else(|| bad(format!("{what} is not text")), Ok)
}

fn get_uint(v: &Value, what: &str) -> Result<u64> {
    v.as_uint()
        .map_or_else(|| bad(format!("{what} is not a number")), Ok)
}

impl DirTree {
    pub fn encode(&self) -> Vec<u8> {
        let entries = self
            .entries
            .iter()
            .map(|e| {
                Value::Array(vec![
                    Value::Text(e.path.clone()),
                    Value::Uint(e.mode as u64),
                    obj(&e.blob),
                    Value::Uint(e.size),
                ])
            })
            .collect();
        cbor::encode(&Value::map([
            (0, Value::Uint(VERSION)),
            (1, Value::Array(entries)),
        ]))
    }

    /// Read and check a tree: a known version, clean paths, each once, in
    /// order, with a known mode.
    pub fn decode(bytes: &[u8]) -> Result<DirTree> {
        let v = cbor::decode(bytes)?;
        let m = v.as_map().map_or_else(|| bad("a tree is a map"), Ok)?;
        if m.get(&0).and_then(Value::as_uint) != Some(VERSION) {
            return bad(format!(
                "tree version {:?} is not known; this reads {VERSION}",
                m.get(&0)
            ));
        }
        let mut entries = Vec::new();
        for item in m.get(&1).and_then(Value::as_array).unwrap_or_default() {
            let [p, mode, blob, size] = item.as_array().unwrap_or_default() else {
                return bad("a tree entry is [path, mode, blob, size]");
            };
            let path = get_text(p, "a path")?;
            let mode = get_uint(mode, "a mode")? as u32;
            if !path_ok(&path) {
                return bad(format!("{path:?} is not a clean relative path"));
            }
            if ![MODE_FILE, MODE_EXEC, MODE_LINK].contains(&mode) {
                return bad(format!(
                    "{path}: mode {mode:o} is not a file, an executable or a link"
                ));
            }
            if entries.last().is_some_and(|l: &Entry| l.path >= path) {
                return bad(format!(
                    "{path}: entries are not in path order, or one is there twice"
                ));
            }
            entries.push(Entry {
                path,
                mode,
                blob: get_obj(blob, "a blob")?,
                size: get_uint(size, "a size")?,
            });
        }
        Ok(DirTree { entries })
    }
}

/// An artifact (docs/cylinder.md §4.1).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Artifact {
    pub kind: Kind,
    /// `x86_64`, `riscv64`, `aarch64`.
    pub arch: Option<String>,
    /// `linux`, `darwin`, `none`.
    pub os: Option<String>,
    /// The [`DirTree`]'s blob.
    pub tree: ObjectRef,
    /// The default command, for a container or an image.
    pub entrypoint: Option<Vec<String>>,
    pub env: BTreeMap<String, String>,
    pub labels: BTreeMap<String, String>,
    /// The files' bytes together.
    pub total_size: u64,
    /// The receipt of the Flow job that built it, when one did.
    pub receipt: Option<ObjectRef>,
}

fn pairs(m: &BTreeMap<String, String>) -> Value {
    if m.is_empty() {
        return Value::Null;
    }
    Value::Array(
        m.iter()
            .map(|(k, v)| Value::Array(vec![Value::Text(k.clone()), Value::Text(v.clone())]))
            .collect(),
    )
}

fn get_pairs(v: Option<&Value>, what: &str) -> Result<BTreeMap<String, String>> {
    let mut out = BTreeMap::new();
    let Some(v) = v.filter(|v| !v.is_null()) else {
        return Ok(out);
    };
    for p in v
        .as_array()
        .map_or_else(|| bad(format!("{what} is a list of pairs")), Ok)?
    {
        let [k, val] = p.as_array().unwrap_or_default() else {
            return bad(format!("{what}: a pair is [key, value]"));
        };
        let k = get_text(k, what)?;
        if out.insert(k.clone(), get_text(val, what)?).is_some() {
            return bad(format!("{what}: {k} is there twice"));
        }
    }
    Ok(out)
}

fn opt_text(v: &Option<String>) -> Value {
    v.as_ref().map_or(Value::Null, |s| Value::Text(s.clone()))
}

impl Artifact {
    /// Its canonical CBOR, keys as in docs/cylinder.md §4.1.
    pub fn encode(&self) -> Vec<u8> {
        let entry = self.entrypoint.as_ref().map_or(Value::Null, |a| {
            Value::Array(a.iter().cloned().map(Value::Text).collect())
        });
        cbor::encode(&Value::map([
            (0, Value::Uint(VERSION)),
            (1, Value::Text(self.kind.as_str().into())),
            (2, opt_text(&self.arch)),
            (3, opt_text(&self.os)),
            (4, obj(&self.tree)),
            (5, entry),
            (6, pairs(&self.env)),
            (7, pairs(&self.labels)),
            (8, Value::Uint(self.total_size)),
            (9, self.receipt.as_ref().map_or(Value::Null, obj)),
        ]))
    }

    pub fn decode(bytes: &[u8]) -> Result<Artifact> {
        let v = cbor::decode(bytes)?;
        let m = v.as_map().map_or_else(|| bad("an artifact is a map"), Ok)?;
        if m.get(&0).and_then(Value::as_uint) != Some(VERSION) {
            return bad(format!(
                "artifact version {:?} is not known; this reads {VERSION}",
                m.get(&0)
            ));
        }
        if let Some(k) = m.keys().find(|k| **k > 9) {
            return bad(format!("key {k} is not part of an artifact"));
        }
        let text = |k: u64, what: &str| -> Result<Option<String>> {
            match m.get(&k) {
                None | Some(Value::Null) => Ok(None),
                Some(v) => get_text(v, what).map(Some),
            }
        };
        let entrypoint = match m.get(&5) {
            None | Some(Value::Null) => None,
            Some(v) => Some(
                v.as_array()
                    .unwrap_or_default()
                    .iter()
                    .map(|a| get_text(a, "the entrypoint"))
                    .collect::<Result<_>>()?,
            ),
        };
        let receipt = match m.get(&9) {
            None | Some(Value::Null) => None,
            Some(v) => Some(get_obj(v, "the receipt")?),
        };
        Ok(Artifact {
            kind: Kind::parse(&text(1, "the kind")?.unwrap_or_default())?,
            arch: text(2, "the architecture")?,
            os: text(3, "the system")?,
            tree: get_obj(m.get(&4).unwrap_or(&Value::Null), "the tree")?,
            entrypoint,
            env: get_pairs(m.get(&6), "env")?,
            labels: get_pairs(m.get(&7), "labels")?,
            total_size: get_uint(m.get(&8).unwrap_or(&Value::Null), "the size")?,
            receipt,
        })
    }

    /// Its name: the BLAKE3 of its canonical encoding, hex.
    pub fn name(&self) -> String {
        blake3::hash(&self.encode()).to_hex().to_string()
    }
}

/// What to pack a directory as.
#[derive(Clone, Debug)]
pub struct Spec {
    pub kind: Kind,
    pub arch: Option<String>,
    pub os: Option<String>,
    pub entrypoint: Option<Vec<String>>,
    pub env: BTreeMap<String, String>,
    pub labels: BTreeMap<String, String>,
}

impl Spec {
    pub fn new(kind: Kind) -> Spec {
        Spec {
            kind,
            arch: None,
            os: None,
            entrypoint: None,
            env: BTreeMap::new(),
            labels: BTreeMap::new(),
        }
    }
}

/// An artifact kept in a repository, with its name.
#[derive(Clone, Debug)]
pub struct Kept {
    pub name: String,
    pub artifact: Artifact,
    /// The blob its encoding is kept in.
    pub blob: ObjectRef,
}

fn kept_dir(repo: &Repo) -> PathBuf {
    repo.oxy_dir().join("artifacts")
}

/// Every file under `dir`, by relative path: regular files and links, in
/// path order. `.oxy` and `.git` directories are not part of an artifact.
fn walk(dir: &Path) -> Result<Vec<(String, PathBuf, std::fs::Metadata)>> {
    let mut out = Vec::new();
    let mut stack = vec![(dir.to_path_buf(), String::new())];
    while let Some((d, rel)) = stack.pop() {
        for e in std::fs::read_dir(&d)? {
            let e = e?;
            let name = e
                .file_name()
                .into_string()
                .map_err(|n| Error::Bad(format!("{n:?} is not UTF-8")))?;
            let path = if rel.is_empty() {
                name.clone()
            } else {
                format!("{rel}/{name}")
            };
            let md = std::fs::symlink_metadata(e.path())?;
            if md.is_dir() {
                if name != ".oxy" && name != ".git" {
                    stack.push((e.path(), path));
                }
            } else if md.is_file() || md.file_type().is_symlink() {
                out.push((path, e.path(), md));
            }
        }
    }
    out.sort_by(|a, b| a.0.cmp(&b.0));
    Ok(out)
}

#[cfg(unix)]
fn mode_of(md: &std::fs::Metadata) -> u32 {
    use std::os::unix::fs::PermissionsExt;
    if md.file_type().is_symlink() {
        MODE_LINK
    } else if md.permissions().mode() & 0o111 != 0 {
        MODE_EXEC
    } else {
        MODE_FILE
    }
}

#[cfg(not(unix))]
fn mode_of(md: &std::fs::Metadata) -> u32 {
    if md.file_type().is_symlink() {
        MODE_LINK
    } else {
        MODE_FILE
    }
}

/// Pack the files under `dir` into an artifact of `spec`'s kind in
/// `repo`'s store: each file a blob, so files and chunks the store holds
/// already cost nothing; then its tree and its encoding, kept under its
/// name.
pub fn pack(repo: &Repo, dir: &Path, spec: &Spec) -> Result<Kept> {
    if !dir.is_dir() {
        return bad(format!("{} is not a directory", dir.display()));
    }
    let mut tree = DirTree::default();
    for (path, full, md) in walk(dir)? {
        let mode = mode_of(&md);
        let bytes = if mode == MODE_LINK {
            std::fs::read_link(&full)?
                .to_string_lossy()
                .into_owned()
                .into_bytes()
        } else {
            std::fs::read(&full)?
        };
        let blob = repo.store_blob(&bytes)?;
        tree.entries.push(Entry {
            path,
            mode,
            blob,
            size: bytes.len() as u64,
        });
    }
    let artifact = Artifact {
        kind: spec.kind,
        arch: spec.arch.clone(),
        os: spec.os.clone(),
        tree: repo.store_blob(&tree.encode())?,
        entrypoint: spec.entrypoint.clone(),
        env: spec.env.clone(),
        labels: spec.labels.clone(),
        total_size: tree.entries.iter().map(|e| e.size).sum(),
        receipt: None,
    };
    let bytes = artifact.encode();
    let blob = repo.store_blob(&bytes)?;
    let name = blake3::hash(&bytes).to_hex().to_string();
    std::fs::create_dir_all(kept_dir(repo))?;
    std::fs::write(kept_dir(repo).join(&name), blob.to_hex())?;
    Ok(Kept {
        name,
        artifact,
        blob,
    })
}

/// The names of the artifacts kept in `repo`, sorted.
pub fn list(repo: &Repo) -> Result<Vec<String>> {
    let mut out = Vec::new();
    match std::fs::read_dir(kept_dir(repo)) {
        Ok(rd) => {
            for e in rd {
                let n = e?.file_name().to_string_lossy().into_owned();
                if n.len() == 64 && n.bytes().all(|b| b.is_ascii_hexdigit()) {
                    out.push(n);
                }
            }
        }
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
        Err(e) => return Err(e.into()),
    }
    out.sort();
    Ok(out)
}

/// The kept artifact `name` names, or a unique prefix of it (at least six
/// characters), checked against its name.
pub fn open(repo: &Repo, name: &str) -> Result<Kept> {
    let name = name.to_ascii_lowercase();
    if name.len() < 6 {
        return bad("name an artifact by at least six characters of its name");
    }
    let found: Vec<String> = list(repo)?
        .into_iter()
        .filter(|n| n.starts_with(&name))
        .collect();
    let full = match found.as_slice() {
        [one] => one.clone(),
        [] => {
            return bad(format!(
                "no artifact {name} here; `oxy artifact list` shows them"
            ))
        }
        _ => {
            return bad(format!(
                "{name} names {} artifacts: give more of the name",
                found.len()
            ))
        }
    };
    let hex = std::fs::read_to_string(kept_dir(repo).join(&full))?;
    let blob = ObjectRef::from_hex(hex.trim()).map_or_else(
        || bad(format!("artifact {full}: its record is damaged")),
        Ok,
    )?;
    let bytes = repo.read_blob(&blob)?;
    if blake3::hash(&bytes).to_hex().as_str() != full {
        return bad(format!(
            "artifact {full}: its encoding no longer hashes to its name"
        ));
    }
    Ok(Kept {
        name: full,
        artifact: Artifact::decode(&bytes)?,
        blob,
    })
}

/// An artifact's tree.
pub fn tree(repo: &Repo, a: &Artifact) -> Result<DirTree> {
    DirTree::decode(&repo.read_blob(&a.tree)?)
}

/// Check that every file of `k` is in the store whole: its bytes read, at
/// the size its tree says. Answers how many files were read.
pub fn verify(repo: &Repo, k: &Kept) -> Result<usize> {
    let t = tree(repo, &k.artifact)?;
    let mut total = 0;
    for e in &t.entries {
        let n = repo.read_blob(&e.blob)?.len() as u64;
        if n != e.size {
            return bad(format!(
                "{}: {n} bytes, but the tree says {}",
                e.path, e.size
            ));
        }
        total += n;
    }
    if total != k.artifact.total_size {
        return bad(format!(
            "the files are {total} bytes together, but the artifact says {}",
            k.artifact.total_size
        ));
    }
    Ok(t.entries.len())
}

/// Write `k`'s files under `dest`, which must not exist or be empty.
pub fn unpack(repo: &Repo, k: &Kept, dest: &Path) -> Result<usize> {
    if dest.exists() && std::fs::read_dir(dest)?.next().is_some() {
        return bad(format!("{} is not empty", dest.display()));
    }
    let t = tree(repo, &k.artifact)?;
    for e in &t.entries {
        let out = dest.join(&e.path);
        if let Some(p) = out.parent() {
            std::fs::create_dir_all(p)?;
        }
        let bytes = repo.read_blob(&e.blob)?;
        if e.mode == MODE_LINK {
            #[cfg(unix)]
            std::os::unix::fs::symlink(String::from_utf8_lossy(&bytes).as_ref(), &out)?;
            #[cfg(not(unix))]
            std::fs::write(&out, &bytes)?;
            continue;
        }
        std::fs::write(&out, &bytes)?;
        #[cfg(unix)]
        if e.mode == MODE_EXEC {
            use std::os::unix::fs::PermissionsExt;
            std::fs::set_permissions(&out, std::fs::Permissions::from_mode(0o755))?;
        }
    }
    Ok(t.entries.len())
}

/// The blobs garbage collection must keep for the kept artifacts: each
/// one's encoding, its tree and its files (`Repo::gc_keeping`).
pub fn roots(repo: &Repo) -> Result<Vec<ObjectRef>> {
    let mut out = Vec::new();
    for name in list(repo)? {
        let k = open(repo, &name)?;
        out.push(k.blob);
        out.push(k.artifact.tree);
        out.extend(tree(repo, &k.artifact)?.entries.iter().map(|e| e.blob));
    }
    Ok(out)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn repo(dir: &Path) -> Repo {
        Repo::init(&dir.join("repo")).unwrap()
    }

    #[test]
    fn a_directory_packs_into_an_artifact_and_back() {
        let t = tempfile::tempdir().unwrap();
        let r = repo(t.path());
        let src = t.path().join("out");
        std::fs::create_dir_all(src.join("bin")).unwrap();
        std::fs::write(src.join("README"), b"hello\n").unwrap();
        std::fs::write(src.join("bin/tool"), vec![7u8; 300_000]).unwrap();
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt;
            std::fs::set_permissions(src.join("bin/tool"), std::fs::Permissions::from_mode(0o755))
                .unwrap();
            std::os::unix::fs::symlink("bin/tool", src.join("tool")).unwrap();
        }
        let mut spec = Spec::new(Kind::Archive);
        spec.arch = Some("x86_64".into());
        spec.labels.insert("version".into(), "0.1.0".into());
        let k = pack(&r, &src, &spec).unwrap();
        assert_eq!(k.name, k.artifact.name());
        assert_eq!(
            k.artifact.total_size,
            300_006 + if cfg!(unix) { 8 } else { 0 }
        );
        assert_eq!(list(&r).unwrap(), vec![k.name.clone()]);

        let again = open(&r, &k.name[..10]).unwrap();
        assert_eq!(again.artifact, k.artifact);
        let paths: Vec<(String, u32)> = tree(&r, &again.artifact)
            .unwrap()
            .entries
            .into_iter()
            .map(|e| (e.path, e.mode))
            .collect();
        assert_eq!(paths[0], ("README".into(), MODE_FILE));
        assert_eq!(
            paths[1],
            (
                "bin/tool".into(),
                if cfg!(unix) { MODE_EXEC } else { MODE_FILE }
            )
        );
        assert_eq!(verify(&r, &again).unwrap(), paths.len());

        let dest = t.path().join("back");
        unpack(&r, &again, &dest).unwrap();
        assert_eq!(
            std::fs::read(dest.join("bin/tool")).unwrap(),
            vec![7u8; 300_000]
        );
        #[cfg(unix)]
        assert_eq!(
            std::fs::read_link(dest.join("tool")).unwrap(),
            Path::new("bin/tool")
        );
        assert!(unpack(&r, &again, &dest)
            .unwrap_err()
            .to_string()
            .contains("not empty"));

        // The same files pack to the same name; garbage collection keeps it.
        assert_eq!(pack(&r, &src, &spec).unwrap().name, k.name);
        let keep = roots(&r).unwrap();
        assert!(keep.contains(&k.blob) && keep.contains(&k.artifact.tree));
        r.gc_keeping(true, &keep, &mut |_| {}).unwrap();
        assert_eq!(
            verify(&r, &open(&r, &k.name).unwrap()).unwrap(),
            paths.len()
        );
    }

    #[test]
    fn encodings_are_canonical_and_checked() {
        let a = Artifact {
            kind: Kind::RunnerImage,
            arch: Some("riscv64".into()),
            os: Some("linux".into()),
            tree: ObjectRef([1; 32]),
            entrypoint: Some(vec!["/sbin/init".into()]),
            env: [("PATH".to_string(), "/bin".to_string())].into(),
            labels: BTreeMap::new(),
            total_size: 42,
            receipt: Some(ObjectRef([2; 32])),
        };
        let bytes = a.encode();
        assert_eq!(Artifact::decode(&bytes).unwrap(), a);
        assert_eq!(a.name(), blake3::hash(&bytes).to_hex().to_string());
        assert!(Kind::parse("tarball")
            .unwrap_err()
            .to_string()
            .contains("runner-image"));

        let wrong = |v: Value| Artifact::decode(&cbor::encode(&v)).unwrap_err().to_string();
        assert!(wrong(Value::map([(0, Value::Uint(2))])).contains("version"));
        assert!(wrong(Value::map([(0, Value::Uint(1)), (10, Value::Null)])).contains("key 10"));

        let t = DirTree {
            entries: vec![Entry {
                path: "a".into(),
                mode: MODE_FILE,
                blob: ObjectRef([3; 32]),
                size: 1,
            }],
        };
        assert_eq!(DirTree::decode(&t.encode()).unwrap(), t);
        let entry = |p: &str, mode: u32| {
            Value::Array(vec![
                Value::Text(p.into()),
                Value::Uint(mode as u64),
                Value::Bytes(vec![0; 32]),
                Value::Uint(0),
            ])
        };
        let tree = |items: Vec<Value>| {
            DirTree::decode(&cbor::encode(&Value::map([
                (0, Value::Uint(1)),
                (1, Value::Array(items)),
            ])))
            .unwrap_err()
            .to_string()
        };
        assert!(tree(vec![entry("../x", MODE_FILE)]).contains("clean"));
        assert!(tree(vec![entry("b", MODE_FILE), entry("a", MODE_FILE)]).contains("order"));
        assert!(tree(vec![entry("a", 0o040000)]).contains("mode"));
    }
}
`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet
Coverage / crates/cylinder/src/lib.rs 94.5%

Keyboard
