//! Cylinder, step C1 (docs/cylinder.md §4.1): an **artifact** is an //! immutable, content-addressed object naming a tree of files, each kept as //! an ordinary Oxygen blob of FastCDC BLAKE3 chunks. An image rebuilt with //! one changed file moves only that file's changed chunks, and a binary in //! a release archive and in a runner image is the same chunks. //! //! An artifact is its canonical CBOR (spec §5.1), named by the BLAKE3 of //! that encoding. Until the artifact is an object type of its own (the rest //! of C1), the encoding and its tree are kept as blobs in the repository's //! store, and `.oxy/artifacts/NAME` holds the encoding's blob so the //! artifact can be found and garbage collection keeps it ([`roots`]). use oxy_core::cbor::{self, Value}; use oxy_core::hash::ObjectRef; use oxy_core::repo::Repo; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; /// The encoding's version this crate writes and reads. pub const VERSION: u64 = 1; #[derive(Debug, thiserror::Error)] pub enum Error { #[error(transparent)] Repo(#[from] oxy_core::repo::RepoError), #[error("io: {0}")] Io(#[from] std::io::Error), #[error("not a canonical artifact encoding: {0}")] Cbor(#[from] oxy_core::cbor::CborError), #[error("{0}")] Bad(String), } pub type Result = std::result::Result; fn bad(msg: impl Into) -> Result { Err(Error::Bad(msg.into())) } /// What an artifact is for. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Kind { Container, RunnerImage, Package, Archive, Report, } impl Kind { pub const ALL: [Kind; 5] = [ Kind::Container, Kind::RunnerImage, Kind::Package, Kind::Archive, Kind::Report, ]; pub fn as_str(self) -> &'static str { match self { Kind::Container => "container", Kind::RunnerImage => "runner-image", Kind::Package => "package", Kind::Archive => "archive", Kind::Report => "report", } } pub fn parse(s: &str) -> Result { Kind::ALL.into_iter().find(|k| k.as_str() == s).map_or_else( || { bad(format!( "{s:?} is not a kind of artifact: {}", Kind::ALL.map(|k| k.as_str()).join(", ") )) }, Ok, ) } } /// A file's mode in a tree: a regular file, an executable one, or a /// symbolic link (whose blob is its target). pub const MODE_FILE: u32 = 0o100644; pub const MODE_EXEC: u32 = 0o100755; pub const MODE_LINK: u32 = 0o120000; /// A file of an artifact's tree. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Entry { /// `/`-separated, relative, never `.` or `..` parts. pub path: String, pub mode: u32, pub blob: ObjectRef, pub size: u64, } /// An artifact's files, in path order: paths, modes, blobs and sizes. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct DirTree { pub entries: Vec, } fn path_ok(p: &str) -> bool { !p.is_empty() && !p.starts_with('/') && p.split('/').all(|s| !s.is_empty() && s != "." && s != "..") } fn obj(r: &ObjectRef) -> Value { Value::Bytes(r.0.to_vec()) } fn get_obj(v: &Value, what: &str) -> Result { match v.as_bytes() { Some(b) if b.len() == 32 => { let mut a = [0u8; 32]; a.copy_from_slice(b); Ok(ObjectRef(a)) } _ => bad(format!("{what} is not an object reference")), } } fn get_text(v: &Value, what: &str) -> Result { v.as_text() .map(str::to_string) .map_or_else(|| bad(format!("{what} is not text")), Ok) } fn get_uint(v: &Value, what: &str) -> Result { v.as_uint() .map_or_else(|| bad(format!("{what} is not a number")), Ok) } impl DirTree { pub fn encode(&self) -> Vec { let entries = self .entries .iter() .map(|e| { Value::Array(vec![ Value::Text(e.path.clone()), Value::Uint(e.mode as u64), obj(&e.blob), Value::Uint(e.size), ]) }) .collect(); cbor::encode(&Value::map([ (0, Value::Uint(VERSION)), (1, Value::Array(entries)), ])) } /// Read and check a tree: a known version, clean paths, each once, in /// order, with a known mode. pub fn decode(bytes: &[u8]) -> Result { let v = cbor::decode(bytes)?; let m = v.as_map().map_or_else(|| bad("a tree is a map"), Ok)?; if m.get(&0).and_then(Value::as_uint) != Some(VERSION) { return bad(format!( "tree version {:?} is not known; this reads {VERSION}", m.get(&0) )); } let mut entries = Vec::new(); for item in m.get(&1).and_then(Value::as_array).unwrap_or_default() { let [p, mode, blob, size] = item.as_array().unwrap_or_default() else { return bad("a tree entry is [path, mode, blob, size]"); }; let path = get_text(p, "a path")?; let mode = get_uint(mode, "a mode")? as u32; if !path_ok(&path) { return bad(format!("{path:?} is not a clean relative path")); } if ![MODE_FILE, MODE_EXEC, MODE_LINK].contains(&mode) { return bad(format!( "{path}: mode {mode:o} is not a file, an executable or a link" )); } if entries.last().is_some_and(|l: &Entry| l.path >= path) { return bad(format!( "{path}: entries are not in path order, or one is there twice" )); } entries.push(Entry { path, mode, blob: get_obj(blob, "a blob")?, size: get_uint(size, "a size")?, }); } Ok(DirTree { entries }) } } /// An artifact (docs/cylinder.md §4.1). #[derive(Clone, Debug, PartialEq, Eq)] pub struct Artifact { pub kind: Kind, /// `x86_64`, `riscv64`, `aarch64`. pub arch: Option, /// `linux`, `darwin`, `none`. pub os: Option, /// The [`DirTree`]'s blob. pub tree: ObjectRef, /// The default command, for a container or an image. pub entrypoint: Option>, pub env: BTreeMap, pub labels: BTreeMap, /// The files' bytes together. pub total_size: u64, /// The receipt of the Flow job that built it, when one did. pub receipt: Option, } fn pairs(m: &BTreeMap) -> Value { if m.is_empty() { return Value::Null; } Value::Array( m.iter() .map(|(k, v)| Value::Array(vec![Value::Text(k.clone()), Value::Text(v.clone())])) .collect(), ) } fn get_pairs(v: Option<&Value>, what: &str) -> Result> { let mut out = BTreeMap::new(); let Some(v) = v.filter(|v| !v.is_null()) else { return Ok(out); }; for p in v .as_array() .map_or_else(|| bad(format!("{what} is a list of pairs")), Ok)? { let [k, val] = p.as_array().unwrap_or_default() else { return bad(format!("{what}: a pair is [key, value]")); }; let k = get_text(k, what)?; if out.insert(k.clone(), get_text(val, what)?).is_some() { return bad(format!("{what}: {k} is there twice")); } } Ok(out) } fn opt_text(v: &Option) -> Value { v.as_ref().map_or(Value::Null, |s| Value::Text(s.clone())) } impl Artifact { /// Its canonical CBOR, keys as in docs/cylinder.md §4.1. pub fn encode(&self) -> Vec { let entry = self.entrypoint.as_ref().map_or(Value::Null, |a| { Value::Array(a.iter().cloned().map(Value::Text).collect()) }); cbor::encode(&Value::map([ (0, Value::Uint(VERSION)), (1, Value::Text(self.kind.as_str().into())), (2, opt_text(&self.arch)), (3, opt_text(&self.os)), (4, obj(&self.tree)), (5, entry), (6, pairs(&self.env)), (7, pairs(&self.labels)), (8, Value::Uint(self.total_size)), (9, self.receipt.as_ref().map_or(Value::Null, obj)), ])) } pub fn decode(bytes: &[u8]) -> Result { let v = cbor::decode(bytes)?; let m = v.as_map().map_or_else(|| bad("an artifact is a map"), Ok)?; if m.get(&0).and_then(Value::as_uint) != Some(VERSION) { return bad(format!( "artifact version {:?} is not known; this reads {VERSION}", m.get(&0) )); } if let Some(k) = m.keys().find(|k| **k > 9) { return bad(format!("key {k} is not part of an artifact")); } let text = |k: u64, what: &str| -> Result> { match m.get(&k) { None | Some(Value::Null) => Ok(None), Some(v) => get_text(v, what).map(Some), } }; let entrypoint = match m.get(&5) { None | Some(Value::Null) => None, Some(v) => Some( v.as_array() .unwrap_or_default() .iter() .map(|a| get_text(a, "the entrypoint")) .collect::>()?, ), }; let receipt = match m.get(&9) { None | Some(Value::Null) => None, Some(v) => Some(get_obj(v, "the receipt")?), }; Ok(Artifact { kind: Kind::parse(&text(1, "the kind")?.unwrap_or_default())?, arch: text(2, "the architecture")?, os: text(3, "the system")?, tree: get_obj(m.get(&4).unwrap_or(&Value::Null), "the tree")?, entrypoint, env: get_pairs(m.get(&6), "env")?, labels: get_pairs(m.get(&7), "labels")?, total_size: get_uint(m.get(&8).unwrap_or(&Value::Null), "the size")?, receipt, }) } /// Its name: the BLAKE3 of its canonical encoding, hex. pub fn name(&self) -> String { blake3::hash(&self.encode()).to_hex().to_string() } } /// What to pack a directory as. #[derive(Clone, Debug)] pub struct Spec { pub kind: Kind, pub arch: Option, pub os: Option, pub entrypoint: Option>, pub env: BTreeMap, pub labels: BTreeMap, } impl Spec { pub fn new(kind: Kind) -> Spec { Spec { kind, arch: None, os: None, entrypoint: None, env: BTreeMap::new(), labels: BTreeMap::new(), } } } /// An artifact kept in a repository, with its name. #[derive(Clone, Debug)] pub struct Kept { pub name: String, pub artifact: Artifact, /// The blob its encoding is kept in. pub blob: ObjectRef, } fn kept_dir(repo: &Repo) -> PathBuf { repo.oxy_dir().join("artifacts") } /// Every file under `dir`, by relative path: regular files and links, in /// path order. `.oxy` and `.git` directories are not part of an artifact. fn walk(dir: &Path) -> Result> { let mut out = Vec::new(); let mut stack = vec![(dir.to_path_buf(), String::new())]; while let Some((d, rel)) = stack.pop() { for e in std::fs::read_dir(&d)? { let e = e?; let name = e .file_name() .into_string() .map_err(|n| Error::Bad(format!("{n:?} is not UTF-8")))?; let path = if rel.is_empty() { name.clone() } else { format!("{rel}/{name}") }; let md = std::fs::symlink_metadata(e.path())?; if md.is_dir() { if name != ".oxy" && name != ".git" { stack.push((e.path(), path)); } } else if md.is_file() || md.file_type().is_symlink() { out.push((path, e.path(), md)); } } } out.sort_by(|a, b| a.0.cmp(&b.0)); Ok(out) } #[cfg(unix)] fn mode_of(md: &std::fs::Metadata) -> u32 { use std::os::unix::fs::PermissionsExt; if md.file_type().is_symlink() { MODE_LINK } else if md.permissions().mode() & 0o111 != 0 { MODE_EXEC } else { MODE_FILE } } #[cfg(not(unix))] fn mode_of(md: &std::fs::Metadata) -> u32 { if md.file_type().is_symlink() { MODE_LINK } else { MODE_FILE } } /// Pack the files under `dir` into an artifact of `spec`'s kind in /// `repo`'s store: each file a blob, so files and chunks the store holds /// already cost nothing; then its tree and its encoding, kept under its /// name. pub fn pack(repo: &Repo, dir: &Path, spec: &Spec) -> Result { if !dir.is_dir() { return bad(format!("{} is not a directory", dir.display())); } let mut tree = DirTree::default(); for (path, full, md) in walk(dir)? { let mode = mode_of(&md); let bytes = if mode == MODE_LINK { std::fs::read_link(&full)? .to_string_lossy() .into_owned() .into_bytes() } else { std::fs::read(&full)? }; let blob = repo.store_blob(&bytes)?; tree.entries.push(Entry { path, mode, blob, size: bytes.len() as u64, }); } let artifact = Artifact { kind: spec.kind, arch: spec.arch.clone(), os: spec.os.clone(), tree: repo.store_blob(&tree.encode())?, entrypoint: spec.entrypoint.clone(), env: spec.env.clone(), labels: spec.labels.clone(), total_size: tree.entries.iter().map(|e| e.size).sum(), receipt: None, }; let bytes = artifact.encode(); let blob = repo.store_blob(&bytes)?; let name = blake3::hash(&bytes).to_hex().to_string(); std::fs::create_dir_all(kept_dir(repo))?; std::fs::write(kept_dir(repo).join(&name), blob.to_hex())?; Ok(Kept { name, artifact, blob, }) } /// The names of the artifacts kept in `repo`, sorted. pub fn list(repo: &Repo) -> Result> { let mut out = Vec::new(); match std::fs::read_dir(kept_dir(repo)) { Ok(rd) => { for e in rd { let n = e?.file_name().to_string_lossy().into_owned(); if n.len() == 64 && n.bytes().all(|b| b.is_ascii_hexdigit()) { out.push(n); } } } Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => return Err(e.into()), } out.sort(); Ok(out) } /// The kept artifact `name` names, or a unique prefix of it (at least six /// characters), checked against its name. pub fn open(repo: &Repo, name: &str) -> Result { let name = name.to_ascii_lowercase(); if name.len() < 6 { return bad("name an artifact by at least six characters of its name"); } let found: Vec = list(repo)? .into_iter() .filter(|n| n.starts_with(&name)) .collect(); let full = match found.as_slice() { [one] => one.clone(), [] => { return bad(format!( "no artifact {name} here; `oxy artifact list` shows them" )) } _ => { return bad(format!( "{name} names {} artifacts: give more of the name", found.len() )) } }; let hex = std::fs::read_to_string(kept_dir(repo).join(&full))?; let blob = ObjectRef::from_hex(hex.trim()).map_or_else( || bad(format!("artifact {full}: its record is damaged")), Ok, )?; let bytes = repo.read_blob(&blob)?; if blake3::hash(&bytes).to_hex().as_str() != full { return bad(format!( "artifact {full}: its encoding no longer hashes to its name" )); } Ok(Kept { name: full, artifact: Artifact::decode(&bytes)?, blob, }) } /// An artifact's tree. pub fn tree(repo: &Repo, a: &Artifact) -> Result { DirTree::decode(&repo.read_blob(&a.tree)?) } /// Check that every file of `k` is in the store whole: its bytes read, at /// the size its tree says. Answers how many files were read. pub fn verify(repo: &Repo, k: &Kept) -> Result { let t = tree(repo, &k.artifact)?; let mut total = 0; for e in &t.entries { let n = repo.read_blob(&e.blob)?.len() as u64; if n != e.size { return bad(format!( "{}: {n} bytes, but the tree says {}", e.path, e.size )); } total += n; } if total != k.artifact.total_size { return bad(format!( "the files are {total} bytes together, but the artifact says {}", k.artifact.total_size )); } Ok(t.entries.len()) } /// Write `k`'s files under `dest`, which must not exist or be empty. pub fn unpack(repo: &Repo, k: &Kept, dest: &Path) -> Result { if dest.exists() && std::fs::read_dir(dest)?.next().is_some() { return bad(format!("{} is not empty", dest.display())); } let t = tree(repo, &k.artifact)?; for e in &t.entries { let out = dest.join(&e.path); if let Some(p) = out.parent() { std::fs::create_dir_all(p)?; } let bytes = repo.read_blob(&e.blob)?; if e.mode == MODE_LINK { #[cfg(unix)] std::os::unix::fs::symlink(String::from_utf8_lossy(&bytes).as_ref(), &out)?; #[cfg(not(unix))] std::fs::write(&out, &bytes)?; continue; } std::fs::write(&out, &bytes)?; #[cfg(unix)] if e.mode == MODE_EXEC { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(&out, std::fs::Permissions::from_mode(0o755))?; } } Ok(t.entries.len()) } /// The blobs garbage collection must keep for the kept artifacts: each /// one's encoding, its tree and its files (`Repo::gc_keeping`). pub fn roots(repo: &Repo) -> Result> { let mut out = Vec::new(); for name in list(repo)? { let k = open(repo, &name)?; out.push(k.blob); out.push(k.artifact.tree); out.extend(tree(repo, &k.artifact)?.entries.iter().map(|e| e.blob)); } Ok(out) } #[cfg(test)] mod tests { use super::*; fn repo(dir: &Path) -> Repo { Repo::init(&dir.join("repo")).unwrap() } #[test] fn a_directory_packs_into_an_artifact_and_back() { let t = tempfile::tempdir().unwrap(); let r = repo(t.path()); let src = t.path().join("out"); std::fs::create_dir_all(src.join("bin")).unwrap(); std::fs::write(src.join("README"), b"hello\n").unwrap(); std::fs::write(src.join("bin/tool"), vec![7u8; 300_000]).unwrap(); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(src.join("bin/tool"), std::fs::Permissions::from_mode(0o755)) .unwrap(); std::os::unix::fs::symlink("bin/tool", src.join("tool")).unwrap(); } let mut spec = Spec::new(Kind::Archive); spec.arch = Some("x86_64".into()); spec.labels.insert("version".into(), "0.1.0".into()); let k = pack(&r, &src, &spec).unwrap(); assert_eq!(k.name, k.artifact.name()); assert_eq!( k.artifact.total_size, 300_006 + if cfg!(unix) { 8 } else { 0 } ); assert_eq!(list(&r).unwrap(), vec![k.name.clone()]); let again = open(&r, &k.name[..10]).unwrap(); assert_eq!(again.artifact, k.artifact); let paths: Vec<(String, u32)> = tree(&r, &again.artifact) .unwrap() .entries .into_iter() .map(|e| (e.path, e.mode)) .collect(); assert_eq!(paths[0], ("README".into(), MODE_FILE)); assert_eq!( paths[1], ( "bin/tool".into(), if cfg!(unix) { MODE_EXEC } else { MODE_FILE } ) ); assert_eq!(verify(&r, &again).unwrap(), paths.len()); let dest = t.path().join("back"); unpack(&r, &again, &dest).unwrap(); assert_eq!( std::fs::read(dest.join("bin/tool")).unwrap(), vec![7u8; 300_000] ); #[cfg(unix)] assert_eq!( std::fs::read_link(dest.join("tool")).unwrap(), Path::new("bin/tool") ); assert!(unpack(&r, &again, &dest) .unwrap_err() .to_string() .contains("not empty")); // The same files pack to the same name; garbage collection keeps it. assert_eq!(pack(&r, &src, &spec).unwrap().name, k.name); let keep = roots(&r).unwrap(); assert!(keep.contains(&k.blob) && keep.contains(&k.artifact.tree)); r.gc_keeping(true, &keep, &mut |_| {}).unwrap(); assert_eq!( verify(&r, &open(&r, &k.name).unwrap()).unwrap(), paths.len() ); } #[test] fn encodings_are_canonical_and_checked() { let a = Artifact { kind: Kind::RunnerImage, arch: Some("riscv64".into()), os: Some("linux".into()), tree: ObjectRef([1; 32]), entrypoint: Some(vec!["/sbin/init".into()]), env: [("PATH".to_string(), "/bin".to_string())].into(), labels: BTreeMap::new(), total_size: 42, receipt: Some(ObjectRef([2; 32])), }; let bytes = a.encode(); assert_eq!(Artifact::decode(&bytes).unwrap(), a); assert_eq!(a.name(), blake3::hash(&bytes).to_hex().to_string()); assert!(Kind::parse("tarball") .unwrap_err() .to_string() .contains("runner-image")); let wrong = |v: Value| Artifact::decode(&cbor::encode(&v)).unwrap_err().to_string(); assert!(wrong(Value::map([(0, Value::Uint(2))])).contains("version")); assert!(wrong(Value::map([(0, Value::Uint(1)), (10, Value::Null)])).contains("key 10")); let t = DirTree { entries: vec![Entry { path: "a".into(), mode: MODE_FILE, blob: ObjectRef([3; 32]), size: 1, }], }; assert_eq!(DirTree::decode(&t.encode()).unwrap(), t); let entry = |p: &str, mode: u32| { Value::Array(vec![ Value::Text(p.into()), Value::Uint(mode as u64), Value::Bytes(vec![0; 32]), Value::Uint(0), ]) }; let tree = |items: Vec| { DirTree::decode(&cbor::encode(&Value::map([ (0, Value::Uint(1)), (1, Value::Array(items)), ]))) .unwrap_err() .to_string() }; assert!(tree(vec![entry("../x", MODE_FILE)]).contains("clean")); assert!(tree(vec![entry("b", MODE_FILE), entry("a", MODE_FILE)]).contains("order")); assert!(tree(vec![entry("a", 0o040000)]).contains("mode")); } }