The artifact as an object type of its own: oxy-core's artifact module, named by its object hash, checked by fsck (refs #108) 6c3dbe2018e0

//! Artifacts (Cylinder, docs/cylinder.md §4.1, oxygen#108): an immutable

//! object naming a tree of files, each an ordinary blob of FastCDC BLAKE3

//! chunks, so an image rebuilt with one changed file moves only that file's

//! changed chunks, and a binary in an archive and in an image is the same

//! chunks. An artifact is an object of its own (`artifact`) in canonical

//! CBOR; its tree is a blob. The `cylinder` crate packs, unpacks and keeps

//! them.

use crate::cbor::{self, Value};

use crate::hash::{hash_object, ObjectRef, ObjectType};

use crate::objects::{schema, ObjectError};

use std::collections::BTreeMap;

type R<T> = std::result::Result<T, ObjectError>;

/// The encoding's version this crate writes and reads.

pub const VERSION: u64 = 1;

/// What an artifact is for.

#[derive(Clone, Copy, Debug, PartialEq, Eq)]

pub enum Kind {

    Container,

    RunnerImage,

    Package,

    Archive,

    Report,

}

impl Kind {

    pub const ALL: [Kind; 5] = [

        Kind::Container,

        Kind::RunnerImage,

        Kind::Package,

        Kind::Archive,

        Kind::Report,

    ];

    pub fn as_str(self) -> &'static str {

        match self {

            Kind::Container => "container",

            Kind::RunnerImage => "runner-image",

            Kind::Package => "package",

            Kind::Archive => "archive",

            Kind::Report => "report",

        }

    }

    pub fn parse(s: &str) -> R<Kind> {

        Kind::ALL.into_iter().find(|k| k.as_str() == s).map_or_else(

            || {

                schema(format!(

                    "{s:?} is not a kind of artifact: {}",

                    Kind::ALL.map(|k| k.as_str()).join(", ")

                ))

            },

            Ok,

        )

    }

}

/// A file's mode in a tree: a regular file, an executable one, or a

/// symbolic link (whose blob is its target).

pub const MODE_FILE: u32 = 0o100644;

pub const MODE_EXEC: u32 = 0o100755;

pub const MODE_LINK: u32 = 0o120000;

/// A file of an artifact's tree.

#[derive(Clone, Debug, PartialEq, Eq)]

pub struct Entry {

    /// `/`-separated, relative, never `.` or `..` parts.

    pub path: String,

    pub mode: u32,

    pub blob: ObjectRef,

    pub size: u64,

}

/// An artifact's files, in path order: paths, modes, blobs and sizes.

#[derive(Clone, Debug, Default, PartialEq, Eq)]

pub struct DirTree {

    pub entries: Vec<Entry>,

}

fn path_ok(p: &str) -> bool {

    !p.is_empty()

        && !p.starts_with('/')

        && p.split('/').all(|s| !s.is_empty() && s != "." && s != "..")

}

fn obj(r: &ObjectRef) -> Value {

    Value::Bytes(r.0.to_vec())

}

fn get_obj(v: &Value, what: &str) -> R<ObjectRef> {

    match v.as_bytes() {

        Some(b) if b.len() == 32 => {

            let mut a = [0u8; 32];

            a.copy_from_slice(b);

            Ok(ObjectRef(a))

        }

        _ => schema(format!("{what} is not an object reference")),

    }

}

fn get_text(v: &Value, what: &str) -> R<String> {

    v.as_text()

        .map(str::to_string)

        .map_or_else(|| schema(format!("{what} is not text")), Ok)

}

fn get_uint(v: &Value, what: &str) -> R<u64> {

    v.as_uint()

        .map_or_else(|| schema(format!("{what} is not a number")), Ok)

}

impl DirTree {

    pub fn encode(&self) -> Vec<u8> {

        let entries = self

            .entries

            .iter()

            .map(|e| {

                Value::Array(vec![

                    Value::Text(e.path.clone()),

                    Value::Uint(e.mode as u64),

                    obj(&e.blob),

                    Value::Uint(e.size),

                ])

            })

            .collect();

        cbor::encode(&Value::map([

            (0, Value::Uint(VERSION)),

            (1, Value::Array(entries)),

        ]))

    }

    /// Read and check a tree: a known version, clean paths, each once, in

    /// order, with a known mode.

    pub fn decode(bytes: &[u8]) -> R<DirTree> {

        let v = cbor::decode(bytes)?;

        let m = v.as_map().map_or_else(|| schema("a tree is a map"), Ok)?;

        if m.get(&0).and_then(Value::as_uint) != Some(VERSION) {

            return schema(format!(

                "tree version {:?} is not known; this reads {VERSION}",

                m.get(&0)

            ));

        }

        let mut entries = Vec::new();

        for item in m.get(&1).and_then(Value::as_array).unwrap_or_default() {

            let [p, mode, blob, size] = item.as_array().unwrap_or_default() else {

                return schema("a tree entry is [path, mode, blob, size]");

            };

            let path = get_text(p, "a path")?;

            let mode = get_uint(mode, "a mode")? as u32;

            if !path_ok(&path) {

                return schema(format!("{path:?} is not a clean relative path"));

            }

            if ![MODE_FILE, MODE_EXEC, MODE_LINK].contains(&mode) {

                return schema(format!(

                    "{path}: mode {mode:o} is not a file, an executable or a link"

                ));

            }

            if entries.last().is_some_and(|l: &Entry| l.path >= path) {

                return schema(format!(

                    "{path}: entries are not in path order, or one is there twice"

                ));

            }

            entries.push(Entry {

                path,

                mode,

                blob: get_obj(blob, "a blob")?,

                size: get_uint(size, "a size")?,

            });

        }

        Ok(DirTree { entries })

    }

}

/// An artifact (docs/cylinder.md §4.1).

#[derive(Clone, Debug, PartialEq, Eq)]

pub struct Artifact {

    pub kind: Kind,

    /// `x86_64`, `riscv64`, `aarch64`.

    pub arch: Option<String>,

    /// `linux`, `darwin`, `none`.

    pub os: Option<String>,

    /// The [`DirTree`]'s blob.

    pub tree: ObjectRef,

    /// The default command, for a container or an image.

    pub entrypoint: Option<Vec<String>>,

    pub env: BTreeMap<String, String>,

    pub labels: BTreeMap<String, String>,

    /// The files' bytes together.

    pub total_size: u64,

    /// The receipt of the Flow job that built it, when one did.

    pub receipt: Option<ObjectRef>,

}

fn pairs(m: &BTreeMap<String, String>) -> Value {

    if m.is_empty() {

        return Value::Null;

    }

    Value::Array(

        m.iter()

            .map(|(k, v)| Value::Array(vec![Value::Text(k.clone()), Value::Text(v.clone())]))

            .collect(),

    )

}

fn get_pairs(v: Option<&Value>, what: &str) -> R<BTreeMap<String, String>> {

    let mut out = BTreeMap::new();

    let Some(v) = v.filter(|v| !v.is_null()) else {

        return Ok(out);

    };

    for p in v

        .as_array()

        .map_or_else(|| schema(format!("{what} is a list of pairs")), Ok)?

    {

        let [k, val] = p.as_array().unwrap_or_default() else {

            return schema(format!("{what}: a pair is [key, value]"));

        };

        let k = get_text(k, what)?;

        if out.insert(k.clone(), get_text(val, what)?).is_some() {

            return schema(format!("{what}: {k} is there twice"));

        }

    }

    Ok(out)

}

fn opt_text(v: &Option<String>) -> Value {

    v.as_ref().map_or(Value::Null, |s| Value::Text(s.clone()))

}

impl Artifact {

    /// Its canonical CBOR, keys as in docs/cylinder.md §4.1.

    pub fn encode(&self) -> Vec<u8> {

        let entry = self.entrypoint.as_ref().map_or(Value::Null, |a| {

            Value::Array(a.iter().cloned().map(Value::Text).collect())

        });

        cbor::encode(&Value::map([

            (0, Value::Uint(VERSION)),

            (1, Value::Text(self.kind.as_str().into())),

            (2, opt_text(&self.arch)),

            (3, opt_text(&self.os)),

            (4, obj(&self.tree)),

            (5, entry),

            (6, pairs(&self.env)),

            (7, pairs(&self.labels)),

            (8, Value::Uint(self.total_size)),

            (9, self.receipt.as_ref().map_or(Value::Null, obj)),

        ]))

    }

    pub fn decode(bytes: &[u8]) -> R<Artifact> {

        let v = cbor::decode(bytes)?;

        let m = v

            .as_map()

            .map_or_else(|| schema("an artifact is a map"), Ok)?;

        if m.get(&0).and_then(Value::as_uint) != Some(VERSION) {

            return schema(format!(

                "artifact version {:?} is not known; this reads {VERSION}",

                m.get(&0)

            ));

        }

        if let Some(k) = m.keys().find(|k| **k > 9) {

            return schema(format!("key {k} is not part of an artifact"));

        }

        let text = |k: u64, what: &str| -> R<Option<String>> {

            match m.get(&k) {

                None | Some(Value::Null) => Ok(None),

                Some(v) => get_text(v, what).map(Some),

            }

        };

        let entrypoint = match m.get(&5) {

            None | Some(Value::Null) => None,

            Some(v) => Some(

                v.as_array()

                    .unwrap_or_default()

                    .iter()

                    .map(|a| get_text(a, "the entrypoint"))

                    .collect::<R<_>>()?,

            ),

        };

        let receipt = match m.get(&9) {

            None | Some(Value::Null) => None,

            Some(v) => Some(get_obj(v, "the receipt")?),

        };

        Ok(Artifact {

            kind: Kind::parse(&text(1, "the kind")?.unwrap_or_default())?,

            arch: text(2, "the architecture")?,

            os: text(3, "the system")?,

            tree: get_obj(m.get(&4).unwrap_or(&Value::Null), "the tree")?,

            entrypoint,

            env: get_pairs(m.get(&6), "env")?,

            labels: get_pairs(m.get(&7), "labels")?,

            total_size: get_uint(m.get(&8).unwrap_or(&Value::Null), "the size")?,

            receipt,

        })

    }

    /// Its name: its object's hash, as every Oxygen object is named.

    pub fn name(&self) -> ObjectRef {

        hash_object(ObjectType::Artifact, &self.encode())

    }

}

- **Added: the `artifact` object type (Cylinder C1, #108).** An artifact is now an object of its own in the store (oxy-core's `artifact` module), named by its object hash as every object is, and `oxy fsck` checks it; its tree stays a blob. Stores written before read as they did: the type is a new one at the end of the list.

//! The model is oxy-core's ([`oxy_core::artifact`]): an `artifact` object

//! in canonical CBOR, named by its object hash, whose tree is a blob. This

//! crate packs a directory into one, opens, verifies and unpacks it, and

//! keeps it: `.oxy/artifacts/NAME` records each artifact this repository

//! keeps, which garbage collection keeps with its tree and files

//! ([`roots`]).

use oxy_core::hash::{ObjectRef, ObjectType};

pub use oxy_core::artifact::{

    Artifact, DirTree, Entry, Kind, MODE_EXEC, MODE_FILE, MODE_LINK, VERSION,

};

    #[error("not an artifact: {0}")]

    Object(#[from] oxy_core::objects::ObjectError),

    #[error("store: {0}")]

    Store(#[from] oxy_core::store::StoreError),

    /// Its object (its name, as a reference).

    pub object: ObjectRef,

    let object = repo.store().put(ObjectType::Artifact, &artifact.encode())?;

    let name = object.to_hex();

    std::fs::write(kept_dir(repo).join(&name), b"")?;

        object,

/// characters), read from the store, which checks it against its name.

    let object = ObjectRef::from_hex(&full).map_or_else(

    // The store checks the bytes against the object's name as it reads.

    let bytes = repo.store().get_typed(&object, ObjectType::Artifact)?;

        object,

    Ok(DirTree::decode(&repo.read_blob(&a.tree)?)?)

        out.push(k.object);

    use oxy_core::cbor::{self, Value};

        assert_eq!(k.name, k.artifact.name().to_hex());

        assert!(keep.contains(&k.object) && keep.contains(&k.artifact.tree));

        // An artifact is an object of its own, which fsck reads and accepts.

        let report = r.fsck().unwrap();

        let errors: Vec<_> = report

            .problems

            .iter()

            .filter(|p| p.severity == oxy_core::repo::Severity::Error)

            .collect();

        assert!(errors.is_empty(), "{errors:?}");

        assert_eq!(

            a.name(),

            oxy_core::hash::hash_object(ObjectType::Artifact, &bytes)

        );

    /// A tree of files as one immutable thing: an image, a package, an

    /// archive (Cylinder, docs/cylinder.md §4.1).

    Artifact,

    pub const ALL: [ObjectType; 13] = [

        ObjectType::Artifact,

            ObjectType::Artifact => "artifact",

pub mod artifact;

        ObjectType::Artifact => crate::artifact::Artifact::decode(body).map(|_| ()),

  unpack | verify`. The artifact is an object type of its own

  (`artifact`, in oxy-core's `artifact` module, checked by `oxy fsck`),

  named by its object hash like every object; its tree is a blob, and

  `.oxy/artifacts/NAME` records the artifacts a repository keeps. Next:

  runner images as artifacts, then moving them between repositories (C2).