//! Cylinder, step C1 (docs/cylinder.md ยง4.1): an **artifact** is an //! immutable, content-addressed object naming a tree of files, each kept as //! an ordinary Oxygen blob of FastCDC BLAKE3 chunks. An image rebuilt with //! one changed file moves only that file's changed chunks, and a binary in //! a release archive and in a runner image is the same chunks. //! //! The model is oxy-core's ([`oxy_core::artifact`]): an `artifact` object //! in canonical CBOR, named by its object hash, whose tree is a blob. This //! crate packs a directory into one, opens, verifies and unpacks it, and //! keeps it: `.oxy/artifacts/NAME` records each artifact this repository //! keeps, which garbage collection keeps with its tree and files //! ([`roots`]). use oxy_core::hash::{ObjectRef, ObjectType}; use oxy_core::repo::Repo; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; pub use oxy_core::artifact::{ Artifact, DirTree, Entry, Kind, MODE_EXEC, MODE_FILE, MODE_LINK, VERSION, }; #[derive(Debug, thiserror::Error)] pub enum Error { #[error(transparent)] Repo(#[from] oxy_core::repo::RepoError), #[error("io: {0}")] Io(#[from] std::io::Error), #[error("not an artifact: {0}")] Object(#[from] oxy_core::objects::ObjectError), #[error("store: {0}")] Store(#[from] oxy_core::store::StoreError), #[error("{0}")] Bad(String), } pub type Result = std::result::Result; fn bad(msg: impl Into) -> Result { Err(Error::Bad(msg.into())) } /// What to pack a directory as. #[derive(Clone, Debug)] pub struct Spec { pub kind: Kind, pub arch: Option, pub os: Option, pub entrypoint: Option>, pub env: BTreeMap, pub labels: BTreeMap, } impl Spec { pub fn new(kind: Kind) -> Spec { Spec { kind, arch: None, os: None, entrypoint: None, env: BTreeMap::new(), labels: BTreeMap::new(), } } } /// An artifact kept in a repository, with its name. #[derive(Clone, Debug)] pub struct Kept { pub name: String, pub artifact: Artifact, /// Its object (its name, as a reference). pub object: ObjectRef, } fn kept_dir(repo: &Repo) -> PathBuf { repo.oxy_dir().join("artifacts") } /// Every file under `dir`, by relative path: regular files and links, in /// path order. `.oxy` and `.git` directories are not part of an artifact. fn walk(dir: &Path) -> Result> { let mut out = Vec::new(); let mut stack = vec![(dir.to_path_buf(), String::new())]; while let Some((d, rel)) = stack.pop() { for e in std::fs::read_dir(&d)? { let e = e?; let name = e .file_name() .into_string() .map_err(|n| Error::Bad(format!("{n:?} is not UTF-8")))?; let path = if rel.is_empty() { name.clone() } else { format!("{rel}/{name}") }; let md = std::fs::symlink_metadata(e.path())?; if md.is_dir() { if name != ".oxy" && name != ".git" { stack.push((e.path(), path)); } } else if md.is_file() || md.file_type().is_symlink() { out.push((path, e.path(), md)); } } } out.sort_by(|a, b| a.0.cmp(&b.0)); Ok(out) } #[cfg(unix)] fn mode_of(md: &std::fs::Metadata) -> u32 { use std::os::unix::fs::PermissionsExt; if md.file_type().is_symlink() { MODE_LINK } else if md.permissions().mode() & 0o111 != 0 { MODE_EXEC } else { MODE_FILE } } #[cfg(not(unix))] fn mode_of(md: &std::fs::Metadata) -> u32 { if md.file_type().is_symlink() { MODE_LINK } else { MODE_FILE } } /// Pack the files under `dir` into an artifact of `spec`'s kind in /// `repo`'s store: each file a blob, so files and chunks the store holds /// already cost nothing; then its tree and its encoding, kept under its /// name. pub fn pack(repo: &Repo, dir: &Path, spec: &Spec) -> Result { if !dir.is_dir() { return bad(format!("{} is not a directory", dir.display())); } let mut tree = DirTree::default(); for (path, full, md) in walk(dir)? { let mode = mode_of(&md); let bytes = if mode == MODE_LINK { std::fs::read_link(&full)? .to_string_lossy() .into_owned() .into_bytes() } else { std::fs::read(&full)? }; let blob = repo.store_blob(&bytes)?; tree.entries.push(Entry { path, mode, blob, size: bytes.len() as u64, }); } let artifact = Artifact { kind: spec.kind, arch: spec.arch.clone(), os: spec.os.clone(), tree: repo.store_blob(&tree.encode())?, entrypoint: spec.entrypoint.clone(), env: spec.env.clone(), labels: spec.labels.clone(), total_size: tree.entries.iter().map(|e| e.size).sum(), receipt: None, }; let object = repo.store().put(ObjectType::Artifact, &artifact.encode())?; let name = object.to_hex(); std::fs::create_dir_all(kept_dir(repo))?; std::fs::write(kept_dir(repo).join(&name), b"")?; Ok(Kept { name, artifact, object, }) } /// The names of the artifacts kept in `repo`, sorted. pub fn list(repo: &Repo) -> Result> { let mut out = Vec::new(); match std::fs::read_dir(kept_dir(repo)) { Ok(rd) => { for e in rd { let n = e?.file_name().to_string_lossy().into_owned(); if n.len() == 64 && n.bytes().all(|b| b.is_ascii_hexdigit()) { out.push(n); } } } Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => return Err(e.into()), } out.sort(); Ok(out) } /// The kept artifact `name` names, or a unique prefix of it (at least six /// characters), read from the store, which checks it against its name. pub fn open(repo: &Repo, name: &str) -> Result { let name = name.to_ascii_lowercase(); if name.len() < 6 { return bad("name an artifact by at least six characters of its name"); } let found: Vec = list(repo)? .into_iter() .filter(|n| n.starts_with(&name)) .collect(); let full = match found.as_slice() { [one] => one.clone(), [] => { return bad(format!( "no artifact {name} here; `oxy artifact list` shows them" )) } _ => { return bad(format!( "{name} names {} artifacts: give more of the name", found.len() )) } }; let object = ObjectRef::from_hex(&full).map_or_else( || bad(format!("artifact {full}: its record is damaged")), Ok, )?; // The store checks the bytes against the object's name as it reads. let bytes = repo.store().get_typed(&object, ObjectType::Artifact)?; Ok(Kept { name: full, artifact: Artifact::decode(&bytes)?, object, }) } /// An artifact's tree. pub fn tree(repo: &Repo, a: &Artifact) -> Result { Ok(DirTree::decode(&repo.read_blob(&a.tree)?)?) } /// Check that every file of `k` is in the store whole: its bytes read, at /// the size its tree says. Answers how many files were read. pub fn verify(repo: &Repo, k: &Kept) -> Result { let t = tree(repo, &k.artifact)?; let mut total = 0; for e in &t.entries { let n = repo.read_blob(&e.blob)?.len() as u64; if n != e.size { return bad(format!( "{}: {n} bytes, but the tree says {}", e.path, e.size )); } total += n; } if total != k.artifact.total_size { return bad(format!( "the files are {total} bytes together, but the artifact says {}", k.artifact.total_size )); } Ok(t.entries.len()) } /// Write `k`'s files under `dest`, which must not exist or be empty. pub fn unpack(repo: &Repo, k: &Kept, dest: &Path) -> Result { if dest.exists() && std::fs::read_dir(dest)?.next().is_some() { return bad(format!("{} is not empty", dest.display())); } let t = tree(repo, &k.artifact)?; for e in &t.entries { let out = dest.join(&e.path); if let Some(p) = out.parent() { std::fs::create_dir_all(p)?; } let bytes = repo.read_blob(&e.blob)?; if e.mode == MODE_LINK { #[cfg(unix)] std::os::unix::fs::symlink(String::from_utf8_lossy(&bytes).as_ref(), &out)?; #[cfg(not(unix))] std::fs::write(&out, &bytes)?; continue; } std::fs::write(&out, &bytes)?; #[cfg(unix)] if e.mode == MODE_EXEC { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(&out, std::fs::Permissions::from_mode(0o755))?; } } Ok(t.entries.len()) } /// The blobs garbage collection must keep for the kept artifacts: each /// one's encoding, its tree and its files (`Repo::gc_keeping`). pub fn roots(repo: &Repo) -> Result> { let mut out = Vec::new(); for name in list(repo)? { let k = open(repo, &name)?; out.push(k.object); out.push(k.artifact.tree); out.extend(tree(repo, &k.artifact)?.entries.iter().map(|e| e.blob)); } Ok(out) } #[cfg(test)] mod tests { use super::*; use oxy_core::cbor::{self, Value}; fn repo(dir: &Path) -> Repo { Repo::init(&dir.join("repo")).unwrap() } #[test] fn a_directory_packs_into_an_artifact_and_back() { let t = tempfile::tempdir().unwrap(); let r = repo(t.path()); let src = t.path().join("out"); std::fs::create_dir_all(src.join("bin")).unwrap(); std::fs::write(src.join("README"), b"hello\n").unwrap(); std::fs::write(src.join("bin/tool"), vec![7u8; 300_000]).unwrap(); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(src.join("bin/tool"), std::fs::Permissions::from_mode(0o755)) .unwrap(); std::os::unix::fs::symlink("bin/tool", src.join("tool")).unwrap(); } let mut spec = Spec::new(Kind::Archive); spec.arch = Some("x86_64".into()); spec.labels.insert("version".into(), "0.1.0".into()); let k = pack(&r, &src, &spec).unwrap(); assert_eq!(k.name, k.artifact.name().to_hex()); assert_eq!( k.artifact.total_size, 300_006 + if cfg!(unix) { 8 } else { 0 } ); assert_eq!(list(&r).unwrap(), vec![k.name.clone()]); let again = open(&r, &k.name[..10]).unwrap(); assert_eq!(again.artifact, k.artifact); let paths: Vec<(String, u32)> = tree(&r, &again.artifact) .unwrap() .entries .into_iter() .map(|e| (e.path, e.mode)) .collect(); assert_eq!(paths[0], ("README".into(), MODE_FILE)); assert_eq!( paths[1], ( "bin/tool".into(), if cfg!(unix) { MODE_EXEC } else { MODE_FILE } ) ); assert_eq!(verify(&r, &again).unwrap(), paths.len()); let dest = t.path().join("back"); unpack(&r, &again, &dest).unwrap(); assert_eq!( std::fs::read(dest.join("bin/tool")).unwrap(), vec![7u8; 300_000] ); #[cfg(unix)] assert_eq!( std::fs::read_link(dest.join("tool")).unwrap(), Path::new("bin/tool") ); assert!(unpack(&r, &again, &dest) .unwrap_err() .to_string() .contains("not empty")); // The same files pack to the same name; garbage collection keeps it. assert_eq!(pack(&r, &src, &spec).unwrap().name, k.name); let keep = roots(&r).unwrap(); assert!(keep.contains(&k.object) && keep.contains(&k.artifact.tree)); r.gc_keeping(true, &keep, &mut |_| {}).unwrap(); // An artifact is an object of its own, which fsck reads and accepts. let report = r.fsck().unwrap(); let errors: Vec<_> = report .problems .iter() .filter(|p| p.severity == oxy_core::repo::Severity::Error) .collect(); assert!(errors.is_empty(), "{errors:?}"); assert_eq!( verify(&r, &open(&r, &k.name).unwrap()).unwrap(), paths.len() ); } #[test] fn encodings_are_canonical_and_checked() { let a = Artifact { kind: Kind::RunnerImage, arch: Some("riscv64".into()), os: Some("linux".into()), tree: ObjectRef([1; 32]), entrypoint: Some(vec!["/sbin/init".into()]), env: [("PATH".to_string(), "/bin".to_string())].into(), labels: BTreeMap::new(), total_size: 42, receipt: Some(ObjectRef([2; 32])), }; let bytes = a.encode(); assert_eq!(Artifact::decode(&bytes).unwrap(), a); assert_eq!( a.name(), oxy_core::hash::hash_object(ObjectType::Artifact, &bytes) ); assert!(Kind::parse("tarball") .unwrap_err() .to_string() .contains("runner-image")); let wrong = |v: Value| Artifact::decode(&cbor::encode(&v)).unwrap_err().to_string(); assert!(wrong(Value::map([(0, Value::Uint(2))])).contains("version")); assert!(wrong(Value::map([(0, Value::Uint(1)), (10, Value::Null)])).contains("key 10")); let t = DirTree { entries: vec![Entry { path: "a".into(), mode: MODE_FILE, blob: ObjectRef([3; 32]), size: 1, }], }; assert_eq!(DirTree::decode(&t.encode()).unwrap(), t); let entry = |p: &str, mode: u32| { Value::Array(vec![ Value::Text(p.into()), Value::Uint(mode as u64), Value::Bytes(vec![0; 32]), Value::Uint(0), ]) }; let tree = |items: Vec| { DirTree::decode(&cbor::encode(&Value::map([ (0, Value::Uint(1)), (1, Value::Array(items)), ]))) .unwrap_err() .to_string() }; assert!(tree(vec![entry("../x", MODE_FILE)]).contains("clean")); assert!(tree(vec![entry("b", MODE_FILE), entry("a", MODE_FILE)]).contains("order")); assert!(tree(vec![entry("a", 0o040000)]).contains("mode")); } }