lib.rs 94.9%

281 of 296 lines run, 37 of 49 functions · run #410 on main · run missed

//! Cylinder, step C1 (docs/cylinder.md §4.1): an **artifact** is an
//! immutable, content-addressed object naming a tree of files, each kept as
//! an ordinary Oxygen blob of FastCDC BLAKE3 chunks. An image rebuilt with
//! one changed file moves only that file's changed chunks, and a binary in
//! a release archive and in a runner image is the same chunks.
//!
//! The model is oxy-core's ([`oxy_core::artifact`]): an `artifact` object
//! in canonical CBOR, named by its object hash, whose tree is a blob. This
//! crate packs a directory into one, opens, verifies and unpacks it, and
//! keeps it: `.oxy/artifacts/NAME` records each artifact this repository
//! keeps, which garbage collection keeps with its tree and files
//! ([`roots`]).

use oxy_core::hash::{ObjectRef, ObjectType};
use oxy_core::repo::Repo;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};

pub use oxy_core::artifact::{
    Artifact, DirTree, Entry, Kind, MODE_EXEC, MODE_FILE, MODE_LINK, VERSION,
};

#[derive(Debug, thiserror::Error)]
pub enum Error {
    #[error(transparent)]
    Repo(#[from] oxy_core::repo::RepoError),
    #[error("io: {0}")]
    Io(#[from] std::io::Error),
    #[error("not an artifact: {0}")]
    Object(#[from] oxy_core::objects::ObjectError),
    #[error("store: {0}")]
    Store(#[from] oxy_core::store::StoreError),
    #[error("{0}")]
    Bad(String),
}

pub type Result<T> = std::result::Result<T, Error>;

fn bad<T>(msg: impl Into<String>) -> Result<T> {
    Err(Error::Bad(msg.into()))
}

/// What to pack a directory as.
#[derive(Clone, Debug)]
pub struct Spec {
    pub kind: Kind,
    pub arch: Option<String>,
    pub os: Option<String>,
    pub entrypoint: Option<Vec<String>>,
    pub env: BTreeMap<String, String>,
    pub labels: BTreeMap<String, String>,
}

impl Spec {
    pub fn new(kind: Kind) -> Spec {
        Spec {
            kind,
            arch: None,
            os: None,
            entrypoint: None,
            env: BTreeMap::new(),
            labels: BTreeMap::new(),
        }
    }
}

/// An artifact kept in a repository, with its name.
#[derive(Clone, Debug)]
pub struct Kept {
    pub name: String,
    pub artifact: Artifact,
    /// Its object (its name, as a reference).
    pub object: ObjectRef,
}

fn kept_dir(repo: &Repo) -> PathBuf {
    repo.oxy_dir().join("artifacts")
}

/// Every file under `dir`, by relative path: regular files and links, in
/// path order. `.oxy` and `.git` directories are not part of an artifact.
fn walk(dir: &Path) -> Result<Vec<(String, PathBuf, std::fs::Metadata)>> {
    let mut out = Vec::new();
    let mut stack = vec![(dir.to_path_buf(), String::new())];
    while let Some((d, rel)) = stack.pop() {
        for e in std::fs::read_dir(&d)? {
            let e = e?;
            let name = e
                .file_name()
                .into_string()
                .map_err(|n| Error::Bad(format!("{n:?} is not UTF-8")))?;
            let path = if rel.is_empty() {
                name.clone()
            } else {
                format!("{rel}/{name}")
            };
            let md = std::fs::symlink_metadata(e.path())?;
            if md.is_dir() {
                if name != ".oxy" && name != ".git" {
                    stack.push((e.path(), path));
                }
            } else if md.is_file() || md.file_type().is_symlink() {
                out.push((path, e.path(), md));
            }
        }
    }
    out.sort_by(|a, b| a.0.cmp(&b.0));
    Ok(out)
}

#[cfg(unix)]
fn mode_of(md: &std::fs::Metadata) -> u32 {
    use std::os::unix::fs::PermissionsExt;
    if md.file_type().is_symlink() {
        MODE_LINK
    } else if md.permissions().mode() & 0o111 != 0 {
        MODE_EXEC
    } else {
        MODE_FILE
    }
}

#[cfg(not(unix))]
fn mode_of(md: &std::fs::Metadata) -> u32 {
    if md.file_type().is_symlink() {
        MODE_LINK
    } else {
        MODE_FILE
    }
}

/// Pack the files under `dir` into an artifact of `spec`'s kind in
/// `repo`'s store: each file a blob, so files and chunks the store holds
/// already cost nothing; then its tree and its encoding, kept under its
/// name.
pub fn pack(repo: &Repo, dir: &Path, spec: &Spec) -> Result<Kept> {
    if !dir.is_dir() {
        return bad(format!("{} is not a directory", dir.display()));
    }
    let mut tree = DirTree::default();
    for (path, full, md) in walk(dir)? {
        let mode = mode_of(&md);
        let bytes = if mode == MODE_LINK {
            std::fs::read_link(&full)?
                .to_string_lossy()
                .into_owned()
                .into_bytes()
        } else {
            std::fs::read(&full)?
        };
        let blob = repo.store_blob(&bytes)?;
        tree.entries.push(Entry {
            path,
            mode,
            blob,
            size: bytes.len() as u64,
        });
    }
    let artifact = Artifact {
        kind: spec.kind,
        arch: spec.arch.clone(),
        os: spec.os.clone(),
        tree: repo.store_blob(&tree.encode())?,
        entrypoint: spec.entrypoint.clone(),
        env: spec.env.clone(),
        labels: spec.labels.clone(),
        total_size: tree.entries.iter().map(|e| e.size).sum(),
        receipt: None,
    };
    let object = repo.store().put(ObjectType::Artifact, &artifact.encode())?;
    let name = object.to_hex();
    std::fs::create_dir_all(kept_dir(repo))?;
    std::fs::write(kept_dir(repo).join(&name), b"")?;
    Ok(Kept {
        name,
        artifact,
        object,
    })
}

/// The names of the artifacts kept in `repo`, sorted.
pub fn list(repo: &Repo) -> Result<Vec<String>> {
    let mut out = Vec::new();
    match std::fs::read_dir(kept_dir(repo)) {
        Ok(rd) => {
            for e in rd {
                let n = e?.file_name().to_string_lossy().into_owned();
                if n.len() == 64 && n.bytes().all(|b| b.is_ascii_hexdigit()) {
                    out.push(n);
                }
            }
        }
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
        Err(e) => return Err(e.into()),
    }
    out.sort();
    Ok(out)
}

/// The kept artifact `name` names, or a unique prefix of it (at least six
/// characters), read from the store, which checks it against its name.
pub fn open(repo: &Repo, name: &str) -> Result<Kept> {
    let name = name.to_ascii_lowercase();
    if name.len() < 6 {
        return bad("name an artifact by at least six characters of its name");
    }
    let found: Vec<String> = list(repo)?
        .into_iter()
        .filter(|n| n.starts_with(&name))
        .collect();
    let full = match found.as_slice() {
        [one] => one.clone(),
        [] => {
            return bad(format!(
                "no artifact {name} here; `oxy artifact list` shows them"
            ))
        }
        _ => {
            return bad(format!(
                "{name} names {} artifacts: give more of the name",
                found.len()
            ))
        }
    };
    let object = ObjectRef::from_hex(&full).map_or_else(
        || bad(format!("artifact {full}: its record is damaged")),
        Ok,
    )?;
    // The store checks the bytes against the object's name as it reads.
    let bytes = repo.store().get_typed(&object, ObjectType::Artifact)?;
    Ok(Kept {
        name: full,
        artifact: Artifact::decode(&bytes)?,
        object,
    })
}

/// An artifact's tree.
pub fn tree(repo: &Repo, a: &Artifact) -> Result<DirTree> {
    Ok(DirTree::decode(&repo.read_blob(&a.tree)?)?)
}

/// Check that every file of `k` is in the store whole: its bytes read, at
/// the size its tree says. Answers how many files were read.
pub fn verify(repo: &Repo, k: &Kept) -> Result<usize> {
    let t = tree(repo, &k.artifact)?;
    let mut total = 0;
    for e in &t.entries {
        let n = repo.read_blob(&e.blob)?.len() as u64;
        if n != e.size {
            return bad(format!(
                "{}: {n} bytes, but the tree says {}",
                e.path, e.size
            ));
        }
        total += n;
    }
    if total != k.artifact.total_size {
        return bad(format!(
            "the files are {total} bytes together, but the artifact says {}",
            k.artifact.total_size
        ));
    }
    Ok(t.entries.len())
}

/// Write `k`'s files under `dest`, which must not exist or be empty.
pub fn unpack(repo: &Repo, k: &Kept, dest: &Path) -> Result<usize> {
    if dest.exists() && std::fs::read_dir(dest)?.next().is_some() {
        return bad(format!("{} is not empty", dest.display()));
    }
    let t = tree(repo, &k.artifact)?;
    for e in &t.entries {
        let out = dest.join(&e.path);
        if let Some(p) = out.parent() {
            std::fs::create_dir_all(p)?;
        }
        let bytes = repo.read_blob(&e.blob)?;
        if e.mode == MODE_LINK {
            #[cfg(unix)]
            std::os::unix::fs::symlink(String::from_utf8_lossy(&bytes).as_ref(), &out)?;
            #[cfg(not(unix))]
            std::fs::write(&out, &bytes)?;
            continue;
        }
        std::fs::write(&out, &bytes)?;
        #[cfg(unix)]
        if e.mode == MODE_EXEC {
            use std::os::unix::fs::PermissionsExt;
            std::fs::set_permissions(&out, std::fs::Permissions::from_mode(0o755))?;
        }
    }
    Ok(t.entries.len())
}

/// The blobs garbage collection must keep for the kept artifacts: each
/// one's encoding, its tree and its files (`Repo::gc_keeping`).
pub fn roots(repo: &Repo) -> Result<Vec<ObjectRef>> {
    let mut out = Vec::new();
    for name in list(repo)? {
        let k = open(repo, &name)?;
        out.push(k.object);
        out.push(k.artifact.tree);
        out.extend(tree(repo, &k.artifact)?.entries.iter().map(|e| e.blob));
    }
    Ok(out)
}

#[cfg(test)]
mod tests {
    use super::*;
    use oxy_core::cbor::{self, Value};

    fn repo(dir: &Path) -> Repo {
        Repo::init(&dir.join("repo")).unwrap()
    }

    #[test]
    fn a_directory_packs_into_an_artifact_and_back() {
        let t = tempfile::tempdir().unwrap();
        let r = repo(t.path());
        let src = t.path().join("out");
        std::fs::create_dir_all(src.join("bin")).unwrap();
        std::fs::write(src.join("README"), b"hello\n").unwrap();
        std::fs::write(src.join("bin/tool"), vec![7u8; 300_000]).unwrap();
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt;
            std::fs::set_permissions(src.join("bin/tool"), std::fs::Permissions::from_mode(0o755))
                .unwrap();
            std::os::unix::fs::symlink("bin/tool", src.join("tool")).unwrap();
        }
        let mut spec = Spec::new(Kind::Archive);
        spec.arch = Some("x86_64".into());
        spec.labels.insert("version".into(), "0.1.0".into());
        let k = pack(&r, &src, &spec).unwrap();
        assert_eq!(k.name, k.artifact.name().to_hex());
        assert_eq!(
            k.artifact.total_size,
            300_006 + if cfg!(unix) { 8 } else { 0 }
        );
        assert_eq!(list(&r).unwrap(), vec![k.name.clone()]);

        let again = open(&r, &k.name[..10]).unwrap();
        assert_eq!(again.artifact, k.artifact);
        let paths: Vec<(String, u32)> = tree(&r, &again.artifact)
            .unwrap()
            .entries
            .into_iter()
            .map(|e| (e.path, e.mode))
            .collect();
        assert_eq!(paths[0], ("README".into(), MODE_FILE));
        assert_eq!(
            paths[1],
            (
                "bin/tool".into(),
                if cfg!(unix) { MODE_EXEC } else { MODE_FILE }
            )
        );
        assert_eq!(verify(&r, &again).unwrap(), paths.len());

        let dest = t.path().join("back");
        unpack(&r, &again, &dest).unwrap();
        assert_eq!(
            std::fs::read(dest.join("bin/tool")).unwrap(),
            vec![7u8; 300_000]
        );
        #[cfg(unix)]
        assert_eq!(
            std::fs::read_link(dest.join("tool")).unwrap(),
            Path::new("bin/tool")
        );
        assert!(unpack(&r, &again, &dest)
            .unwrap_err()
            .to_string()
            .contains("not empty"));

        // The same files pack to the same name; garbage collection keeps it.
        assert_eq!(pack(&r, &src, &spec).unwrap().name, k.name);
        let keep = roots(&r).unwrap();
        assert!(keep.contains(&k.object) && keep.contains(&k.artifact.tree));
        r.gc_keeping(true, &keep, &mut |_| {}).unwrap();
        // An artifact is an object of its own, which fsck reads and accepts.
        let report = r.fsck().unwrap();
        let errors: Vec<_> = report
            .problems
            .iter()
            .filter(|p| p.severity == oxy_core::repo::Severity::Error)
            .collect();
        assert!(errors.is_empty(), "{errors:?}");
        assert_eq!(
            verify(&r, &open(&r, &k.name).unwrap()).unwrap(),
            paths.len()
        );
    }

    #[test]
    fn encodings_are_canonical_and_checked() {
        let a = Artifact {
            kind: Kind::RunnerImage,
            arch: Some("riscv64".into()),
            os: Some("linux".into()),
            tree: ObjectRef([1; 32]),
            entrypoint: Some(vec!["/sbin/init".into()]),
            env: [("PATH".to_string(), "/bin".to_string())].into(),
            labels: BTreeMap::new(),
            total_size: 42,
            receipt: Some(ObjectRef([2; 32])),
        };
        let bytes = a.encode();
        assert_eq!(Artifact::decode(&bytes).unwrap(), a);
        assert_eq!(
            a.name(),
            oxy_core::hash::hash_object(ObjectType::Artifact, &bytes)
        );
        assert!(Kind::parse("tarball")
            .unwrap_err()
            .to_string()
            .contains("runner-image"));

        let wrong = |v: Value| Artifact::decode(&cbor::encode(&v)).unwrap_err().to_string();
        assert!(wrong(Value::map([(0, Value::Uint(2))])).contains("version"));
        assert!(wrong(Value::map([(0, Value::Uint(1)), (10, Value::Null)])).contains("key 10"));

        let t = DirTree {
            entries: vec![Entry {
                path: "a".into(),
                mode: MODE_FILE,
                blob: ObjectRef([3; 32]),
                size: 1,
            }],
        };
        assert_eq!(DirTree::decode(&t.encode()).unwrap(), t);
        let entry = |p: &str, mode: u32| {
            Value::Array(vec![
                Value::Text(p.into()),
                Value::Uint(mode as u64),
                Value::Bytes(vec![0; 32]),
                Value::Uint(0),
            ])
        };
        let tree = |items: Vec<Value>| {
            DirTree::decode(&cbor::encode(&Value::map([
                (0, Value::Uint(1)),
                (1, Value::Array(items)),
            ])))
            .unwrap_err()
            .to_string()
        };
        assert!(tree(vec![entry("../x", MODE_FILE)]).contains("clean"));
        assert!(tree(vec![entry("b", MODE_FILE), entry("a", MODE_FILE)]).contains("order"));
        assert!(tree(vec![entry("a", 0o040000)]).contains("mode"));
    }
}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1		//! Cylinder, step C1 (docs/cylinder.md §4.1): an artifact is an
2		//! immutable, content-addressed object naming a tree of files, each kept as
3		//! an ordinary Oxygen blob of FastCDC BLAKE3 chunks. An image rebuilt with
4		//! one changed file moves only that file's changed chunks, and a binary in
5		//! a release archive and in a runner image is the same chunks.
6		//!
7		//! The model is oxy-core's ([`oxy_core::artifact`]): an `artifact` object
8		//! in canonical CBOR, named by its object hash, whose tree is a blob. This
9		//! crate packs a directory into one, opens, verifies and unpacks it, and
10		//! keeps it: `.oxy/artifacts/NAME` records each artifact this repository
11		//! keeps, which garbage collection keeps with its tree and files
12		//! ([`roots`]).
13
14		use oxy_core::hash::{ObjectRef, ObjectType};
15		use oxy_core::repo::Repo;
16		use std::collections::BTreeMap;
17		use std::path::{Path, PathBuf};
18
19		pub use oxy_core::artifact::{
20		Artifact, DirTree, Entry, Kind, MODE_EXEC, MODE_FILE, MODE_LINK, VERSION,
21		};
22
23		#[derive(Debug, thiserror::Error)]
24		pub enum Error {
25		#[error(transparent)]
26		Repo(#[from] oxy_core::repo::RepoError),
27		#[error("io: {0}")]
28		Io(#[from] std::io::Error),
29		#[error("not an artifact: {0}")]
30		Object(#[from] oxy_core::objects::ObjectError),
31		#[error("store: {0}")]
32		Store(#[from] oxy_core::store::StoreError),
33		#[error("{0}")]
34		Bad(String),
35		}
36
37		pub type Result<T> = std::result::Result<T, Error>;
38
39	1×	fn bad<T>(msg: impl Into<String>) -> Result<T> {
40	1×	Err(Error::Bad(msg.into()))
41	1×	}
42
43		/// What to pack a directory as.
44		#[derive(Clone, Debug)]
45		pub struct Spec {
46		pub kind: Kind,
47		pub arch: Option<String>,
48		pub os: Option<String>,
49		pub entrypoint: Option<Vec<String>>,
50		pub env: BTreeMap<String, String>,
51		pub labels: BTreeMap<String, String>,
52		}
53
54		impl Spec {
55	2×	pub fn new(kind: Kind) -> Spec {
56	2×	Spec {
57	2×	kind,
58	2×	arch: None,
59	2×	os: None,
60	2×	entrypoint: None,
61	2×	env: BTreeMap::new(),
62	2×	labels: BTreeMap::new(),
63	2×	}
64	2×	}
65		}
66
67		/// An artifact kept in a repository, with its name.
68		#[derive(Clone, Debug)]
69		pub struct Kept {
70		pub name: String,
71		pub artifact: Artifact,
72		/// Its object (its name, as a reference).
73		pub object: ObjectRef,
74		}
75
76	17×	fn kept_dir(repo: &Repo) -> PathBuf {
77	17×	repo.oxy_dir().join("artifacts")
78	17×	}
79
80		/// Every file under `dir`, by relative path: regular files and links, in
81		/// path order. `.oxy` and `.git` directories are not part of an artifact.
82	3×	fn walk(dir: &Path) -> Result<Vec<(String, PathBuf, std::fs::Metadata)>> {
83	3×	let mut out = Vec::new();
84	3×	let mut stack = vec![(dir.to_path_buf(), String::new())];
85	9×	while let Some((d, rel)) = stack.pop() {
86	11×	for e in std::fs::read_dir(&d)? {
87	11×	let e = e?;
88	11×	let name = e
89	11×	.file_name()
90	11×	.into_string()
91	11×	.map_err(\|n\| Error::Bad(format!("{n:?} is not UTF-8")))?;
92	11×	let path = if rel.is_empty() {
93	8×	name.clone()
94		} else {
95	3×	format!("{rel}/{name}")
96		};
97	11×	let md = std::fs::symlink_metadata(e.path())?;
98	11×	if md.is_dir() {
99	3×	if name != ".oxy" && name != ".git" {
100	3×	stack.push((e.path(), path));
101	3×	}
102	8×	} else if md.is_file() \|\| md.file_type().is_symlink() {
103	8×	out.push((path, e.path(), md));
104	8×	}
105		}
106		}
107	7×	out.sort_by(\|a, b\| a.0.cmp(&b.0));
108	3×	Ok(out)
109	3×	}
110
111		#[cfg(unix)]
112	8×	fn mode_of(md: &std::fs::Metadata) -> u32 {
113		use std::os::unix::fs::PermissionsExt;
114	8×	if md.file_type().is_symlink() {
115	2×	MODE_LINK
116	6×	} else if md.permissions().mode() & 0o111 != 0 {
117	2×	MODE_EXEC
118		} else {
119	4×	MODE_FILE
120		}
121	8×	}
122
123		#[cfg(not(unix))]
124		fn mode_of(md: &std::fs::Metadata) -> u32 {
125		if md.file_type().is_symlink() {
126		MODE_LINK
127		} else {
128		MODE_FILE
129		}
130		}
131
132		/// Pack the files under `dir` into an artifact of `spec`'s kind in
133		/// `repo`'s store: each file a blob, so files and chunks the store holds
134		/// already cost nothing; then its tree and its encoding, kept under its
135		/// name.
136	3×	pub fn pack(repo: &Repo, dir: &Path, spec: &Spec) -> Result<Kept> {
137	3×	if !dir.is_dir() {
138	0×	return bad(format!("{} is not a directory", dir.display()));
139	3×	}
140	3×	let mut tree = DirTree::default();
141	8×	for (path, full, md) in walk(dir)? {
142	8×	let mode = mode_of(&md);
143	8×	let bytes = if mode == MODE_LINK {
144	2×	std::fs::read_link(&full)?
145	2×	.to_string_lossy()
146	2×	.into_owned()
147	2×	.into_bytes()
148		} else {
149	6×	std::fs::read(&full)?
150		};
151	8×	let blob = repo.store_blob(&bytes)?;
152	8×	tree.entries.push(Entry {
153	8×	path,
154	8×	mode,
155	8×	blob,
156	8×	size: bytes.len() as u64,
157	8×	});
158		}
159	3×	let artifact = Artifact {
160	3×	kind: spec.kind,
161	3×	arch: spec.arch.clone(),
162	3×	os: spec.os.clone(),
163	3×	tree: repo.store_blob(&tree.encode())?,
164	3×	entrypoint: spec.entrypoint.clone(),
165	3×	env: spec.env.clone(),
166	3×	labels: spec.labels.clone(),
167	3×	total_size: tree.entries.iter().map(\|e\| e.size).sum(),
168	3×	receipt: None,
169		};
170	3×	let object = repo.store().put(ObjectType::Artifact, &artifact.encode())?;
171	3×	let name = object.to_hex();
172	3×	std::fs::create_dir_all(kept_dir(repo))?;
173	3×	std::fs::write(kept_dir(repo).join(&name), b"")?;
174	3×	Ok(Kept {
175	3×	name,
176	3×	artifact,
177	3×	object,
178	3×	})
179	3×	}
180
181		/// The names of the artifacts kept in `repo`, sorted.
182	11×	pub fn list(repo: &Repo) -> Result<Vec<String>> {
183	11×	let mut out = Vec::new();
184	11×	match std::fs::read_dir(kept_dir(repo)) {
185	11×	Ok(rd) => {
186	11×	for e in rd {
187	11×	let n = e?.file_name().to_string_lossy().into_owned();
188	704×	if n.len() == 64 && n.bytes().all(\|b\| b.is_ascii_hexdigit()) {
189	11×	out.push(n);
190	11×	}
191		}
192		}
193	0×	Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
194	0×	Err(e) => return Err(e.into()),
195		}
196	11×	out.sort();
197	11×	Ok(out)
198	11×	}
199
200		/// The kept artifact `name` names, or a unique prefix of it (at least six
201		/// characters), read from the store, which checks it against its name.
202	7×	pub fn open(repo: &Repo, name: &str) -> Result<Kept> {
203	7×	let name = name.to_ascii_lowercase();
204	7×	if name.len() < 6 {
205	0×	return bad("name an artifact by at least six characters of its name");
206	7×	}
207	7×	let found: Vec<String> = list(repo)?
208	7×	.into_iter()
209	7×	.filter(\|n\| n.starts_with(&name))
210	7×	.collect();
211	7×	let full = match found.as_slice() {
212	7×	[one] => one.clone(),
213	0×	[] => {
214	0×	return bad(format!(
215		"no artifact {name} here; `oxy artifact list` shows them"
216		))
217		}
218		_ => {
219	0×	return bad(format!(
220		"{name} names {} artifacts: give more of the name",
221	0×	found.len()
222		))
223		}
224		};
225	7×	let object = ObjectRef::from_hex(&full).map_or_else(
226	0×	\|\| bad(format!("artifact {full}: its record is damaged")),
227		Ok,
228	0×	)?;
229		// The store checks the bytes against the object's name as it reads.
230	7×	let bytes = repo.store().get_typed(&object, ObjectType::Artifact)?;
231		Ok(Kept {
232	7×	name: full,
233	7×	artifact: Artifact::decode(&bytes)?,
234	7×	object,
235		})
236	7×	}
237
238		/// An artifact's tree.
239	10×	pub fn tree(repo: &Repo, a: &Artifact) -> Result<DirTree> {
240	10×	Ok(DirTree::decode(&repo.read_blob(&a.tree)?)?)
241	10×	}
242
243		/// Check that every file of `k` is in the store whole: its bytes read, at
244		/// the size its tree says. Answers how many files were read.
245	3×	pub fn verify(repo: &Repo, k: &Kept) -> Result<usize> {
246	3×	let t = tree(repo, &k.artifact)?;
247	3×	let mut total = 0;
248	8×	for e in &t.entries {
249	8×	let n = repo.read_blob(&e.blob)?.len() as u64;
250	8×	if n != e.size {
251	0×	return bad(format!(
252		"{}: {n} bytes, but the tree says {}",
253		e.path, e.size
254		));
255	8×	}
256	8×	total += n;
257		}
258	3×	if total != k.artifact.total_size {
259	0×	return bad(format!(
260		"the files are {total} bytes together, but the artifact says {}",
261		k.artifact.total_size
262		));
263	3×	}
264	3×	Ok(t.entries.len())
265	3×	}
266
267		/// Write `k`'s files under `dest`, which must not exist or be empty.
268	3×	pub fn unpack(repo: &Repo, k: &Kept, dest: &Path) -> Result<usize> {
269	3×	if dest.exists() && std::fs::read_dir(dest)?.next().is_some() {
270	1×	return bad(format!("{} is not empty", dest.display()));
271	2×	}
272	2×	let t = tree(repo, &k.artifact)?;
273	5×	for e in &t.entries {
274	5×	let out = dest.join(&e.path);
275	5×	if let Some(p) = out.parent() {
276	5×	std::fs::create_dir_all(p)?;
277	0×	}
278	5×	let bytes = repo.read_blob(&e.blob)?;
279	5×	if e.mode == MODE_LINK {
280		#[cfg(unix)]
281	1×	std::os::unix::fs::symlink(String::from_utf8_lossy(&bytes).as_ref(), &out)?;
282		#[cfg(not(unix))]
283		std::fs::write(&out, &bytes)?;
284	1×	continue;
285	4×	}
286	4×	std::fs::write(&out, &bytes)?;
287		#[cfg(unix)]
288	4×	if e.mode == MODE_EXEC {
289		use std::os::unix::fs::PermissionsExt;
290	1×	std::fs::set_permissions(&out, std::fs::Permissions::from_mode(0o755))?;
291	3×	}
292		}
293	2×	Ok(t.entries.len())
294	3×	}
295
296		/// The blobs garbage collection must keep for the kept artifacts: each
297		/// one's encoding, its tree and its files (`Repo::gc_keeping`).
298	2×	pub fn roots(repo: &Repo) -> Result<Vec<ObjectRef>> {
299	2×	let mut out = Vec::new();
300	2×	for name in list(repo)? {
301	2×	let k = open(repo, &name)?;
302	2×	out.push(k.object);
303	2×	out.push(k.artifact.tree);
304	2×	out.extend(tree(repo, &k.artifact)?.entries.iter().map(\|e\| e.blob));
305		}
306	2×	Ok(out)
307	2×	}
308
309		#[cfg(test)]
310		mod tests {
311		use super::*;
312		use oxy_core::cbor::{self, Value};
313
314	1×	fn repo(dir: &Path) -> Repo {
315	1×	Repo::init(&dir.join("repo")).unwrap()
316	1×	}
317
318		#[test]
319	1×	fn a_directory_packs_into_an_artifact_and_back() {
320	1×	let t = tempfile::tempdir().unwrap();
321	1×	let r = repo(t.path());
322	1×	let src = t.path().join("out");
323	1×	std::fs::create_dir_all(src.join("bin")).unwrap();
324	1×	std::fs::write(src.join("README"), b"hello\n").unwrap();
325	1×	std::fs::write(src.join("bin/tool"), vec![7u8; 300_000]).unwrap();
326		#[cfg(unix)]
327		{
328		use std::os::unix::fs::PermissionsExt;
329	1×	std::fs::set_permissions(src.join("bin/tool"), std::fs::Permissions::from_mode(0o755))
330	1×	.unwrap();
331	1×	std::os::unix::fs::symlink("bin/tool", src.join("tool")).unwrap();
332		}
333	1×	let mut spec = Spec::new(Kind::Archive);
334	1×	spec.arch = Some("x86_64".into());
335	1×	spec.labels.insert("version".into(), "0.1.0".into());
336	1×	let k = pack(&r, &src, &spec).unwrap();
337	1×	assert_eq!(k.name, k.artifact.name().to_hex());
338	1×	assert_eq!(
339		k.artifact.total_size,
340	1×	300_006 + if cfg!(unix) { 8 } else { 0 }
341		);
342	1×	assert_eq!(list(&r).unwrap(), vec![k.name.clone()]);
343
344	1×	let again = open(&r, &k.name[..10]).unwrap();
345	1×	assert_eq!(again.artifact, k.artifact);
346	1×	let paths: Vec<(String, u32)> = tree(&r, &again.artifact)
347	1×	.unwrap()
348	1×	.entries
349	1×	.into_iter()
350	3×	.map(\|e\| (e.path, e.mode))
351	1×	.collect();
352	1×	assert_eq!(paths[0], ("README".into(), MODE_FILE));
353	1×	assert_eq!(
354	1×	paths[1],
355		(
356	1×	"bin/tool".into(),
357	1×	if cfg!(unix) { MODE_EXEC } else { MODE_FILE }
358		)
359		);
360	1×	assert_eq!(verify(&r, &again).unwrap(), paths.len());
361
362	1×	let dest = t.path().join("back");
363	1×	unpack(&r, &again, &dest).unwrap();
364	1×	assert_eq!(
365	1×	std::fs::read(dest.join("bin/tool")).unwrap(),
366	1×	vec![7u8; 300_000]
367		);
368		#[cfg(unix)]
369	1×	assert_eq!(
370	1×	std::fs::read_link(dest.join("tool")).unwrap(),
371	1×	Path::new("bin/tool")
372		);
373	1×	assert!(unpack(&r, &again, &dest)
374	1×	.unwrap_err()
375	1×	.to_string()
376	1×	.contains("not empty"));
377
378		// The same files pack to the same name; garbage collection keeps it.
379	1×	assert_eq!(pack(&r, &src, &spec).unwrap().name, k.name);
380	1×	let keep = roots(&r).unwrap();
381	1×	assert!(keep.contains(&k.object) && keep.contains(&k.artifact.tree));
382	5×	r.gc_keeping(true, &keep, &mut \|_\| {}).unwrap();
383		// An artifact is an object of its own, which fsck reads and accepts.
384	1×	let report = r.fsck().unwrap();
385	1×	let errors: Vec<_> = report
386	1×	.problems
387	1×	.iter()
388	1×	.filter(\|p\| p.severity == oxy_core::repo::Severity::Error)
389	1×	.collect();
390	1×	assert!(errors.is_empty(), "{errors:?}");
391	1×	assert_eq!(
392	1×	verify(&r, &open(&r, &k.name).unwrap()).unwrap(),
393	1×	paths.len()
394		);
395	1×	}
396
397		#[test]
398	1×	fn encodings_are_canonical_and_checked() {
399	1×	let a = Artifact {
400	1×	kind: Kind::RunnerImage,
401	1×	arch: Some("riscv64".into()),
402	1×	os: Some("linux".into()),
403	1×	tree: ObjectRef([1; 32]),
404	1×	entrypoint: Some(vec!["/sbin/init".into()]),
405	1×	env: [("PATH".to_string(), "/bin".to_string())].into(),
406	1×	labels: BTreeMap::new(),
407	1×	total_size: 42,
408	1×	receipt: Some(ObjectRef([2; 32])),
409	1×	};
410	1×	let bytes = a.encode();
411	1×	assert_eq!(Artifact::decode(&bytes).unwrap(), a);
412	1×	assert_eq!(
413	1×	a.name(),
414	1×	oxy_core::hash::hash_object(ObjectType::Artifact, &bytes)
415		);
416	1×	assert!(Kind::parse("tarball")
417	1×	.unwrap_err()
418	1×	.to_string()
419	1×	.contains("runner-image"));
420
421	2×	let wrong = \|v: Value\| Artifact::decode(&cbor::encode(&v)).unwrap_err().to_string();
422	1×	assert!(wrong(Value::map([(0, Value::Uint(2))])).contains("version"));
423	1×	assert!(wrong(Value::map([(0, Value::Uint(1)), (10, Value::Null)])).contains("key 10"));
424
425	1×	let t = DirTree {
426	1×	entries: vec![Entry {
427	1×	path: "a".into(),
428	1×	mode: MODE_FILE,
429	1×	blob: ObjectRef([3; 32]),
430	1×	size: 1,
431	1×	}],
432	1×	};
433	1×	assert_eq!(DirTree::decode(&t.encode()).unwrap(), t);
434	4×	let entry = \|p: &str, mode: u32\| {
435	4×	Value::Array(vec![
436	4×	Value::Text(p.into()),
437	4×	Value::Uint(mode as u64),
438	4×	Value::Bytes(vec![0; 32]),
439	4×	Value::Uint(0),
440	4×	])
441	4×	};
442	3×	let tree = \|items: Vec<Value>\| {
443	3×	DirTree::decode(&cbor::encode(&Value::map([
444	3×	(0, Value::Uint(1)),
445	3×	(1, Value::Array(items)),
446	3×	])))
447	3×	.unwrap_err()
448	3×	.to_string()
449	3×	};
450	1×	assert!(tree(vec![entry("../x", MODE_FILE)]).contains("clean"));
451	1×	assert!(tree(vec![entry("b", MODE_FILE), entry("a", MODE_FILE)]).contains("order"));
452	1×	assert!(tree(vec![entry("a", 0o040000)]).contains("mode"));
453	1×	}
454		}

Coverage / crates/cylinder/src/lib.rs 94.9%

Keyboard