crates/oxy-core/src/artifact.rs at 6c3dbe2018e0 · andymccall/oxygen

# 6c3dbe2018e0 oxygen / crates / oxy-core / src / artifact.rs

9.6 KiB · Rust · last changed The artifact as an object type of its own: oxy-core's artifact module, named by its object hash, checked by fsck (refs #108) 7 days ago source blame history raw

//! Artifacts (Cylinder, docs/cylinder.md §4.1, oxygen#108): an immutable
//! object naming a tree of files, each an ordinary blob of FastCDC BLAKE3
//! chunks, so an image rebuilt with one changed file moves only that file's
//! changed chunks, and a binary in an archive and in an image is the same
//! chunks. An artifact is an object of its own (`artifact`) in canonical
//! CBOR; its tree is a blob. The `cylinder` crate packs, unpacks and keeps
//! them.

use crate::cbor::{self, Value};
use crate::hash::{hash_object, ObjectRef, ObjectType};
use crate::objects::{schema, ObjectError};
use std::collections::BTreeMap;

type R<T> = std::result::Result<T, ObjectError>;

/// The encoding's version this crate writes and reads.
pub const VERSION: u64 = 1;

/// What an artifact is for.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Kind {
    Container,
    RunnerImage,
    Package,
    Archive,
    Report,
}

impl Kind {
    pub const ALL: [Kind; 5] = [
        Kind::Container,
        Kind::RunnerImage,
        Kind::Package,
        Kind::Archive,
        Kind::Report,
    ];

    pub fn as_str(self) -> &'static str {
        match self {
            Kind::Container => "container",
            Kind::RunnerImage => "runner-image",
            Kind::Package => "package",
            Kind::Archive => "archive",
            Kind::Report => "report",
        }
    }

    pub fn parse(s: &str) -> R<Kind> {
        Kind::ALL.into_iter().find(|k| k.as_str() == s).map_or_else(
            || {
                schema(format!(
                    "{s:?} is not a kind of artifact: {}",
                    Kind::ALL.map(|k| k.as_str()).join(", ")
                ))
            },
            Ok,
        )
    }
}

/// A file's mode in a tree: a regular file, an executable one, or a
/// symbolic link (whose blob is its target).
pub const MODE_FILE: u32 = 0o100644;
pub const MODE_EXEC: u32 = 0o100755;
pub const MODE_LINK: u32 = 0o120000;

/// A file of an artifact's tree.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Entry {
    /// `/`-separated, relative, never `.` or `..` parts.
    pub path: String,
    pub mode: u32,
    pub blob: ObjectRef,
    pub size: u64,
}

/// An artifact's files, in path order: paths, modes, blobs and sizes.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct DirTree {
    pub entries: Vec<Entry>,
}

fn path_ok(p: &str) -> bool {
    !p.is_empty()
        && !p.starts_with('/')
        && p.split('/').all(|s| !s.is_empty() && s != "." && s != "..")
}

fn obj(r: &ObjectRef) -> Value {
    Value::Bytes(r.0.to_vec())
}

fn get_obj(v: &Value, what: &str) -> R<ObjectRef> {
    match v.as_bytes() {
        Some(b) if b.len() == 32 => {
            let mut a = [0u8; 32];
            a.copy_from_slice(b);
            Ok(ObjectRef(a))
        }
        _ => schema(format!("{what} is not an object reference")),
    }
}

fn get_text(v: &Value, what: &str) -> R<String> {
    v.as_text()
        .map(str::to_string)
        .map_or_else(|| schema(format!("{what} is not text")), Ok)
}

fn get_uint(v: &Value, what: &str) -> R<u64> {
    v.as_uint()
        .map_or_else(|| schema(format!("{what} is not a number")), Ok)
}

impl DirTree {
    pub fn encode(&self) -> Vec<u8> {
        let entries = self
            .entries
            .iter()
            .map(|e| {
                Value::Array(vec![
                    Value::Text(e.path.clone()),
                    Value::Uint(e.mode as u64),
                    obj(&e.blob),
                    Value::Uint(e.size),
                ])
            })
            .collect();
        cbor::encode(&Value::map([
            (0, Value::Uint(VERSION)),
            (1, Value::Array(entries)),
        ]))
    }

    /// Read and check a tree: a known version, clean paths, each once, in
    /// order, with a known mode.
    pub fn decode(bytes: &[u8]) -> R<DirTree> {
        let v = cbor::decode(bytes)?;
        let m = v.as_map().map_or_else(|| schema("a tree is a map"), Ok)?;
        if m.get(&0).and_then(Value::as_uint) != Some(VERSION) {
            return schema(format!(
                "tree version {:?} is not known; this reads {VERSION}",
                m.get(&0)
            ));
        }
        let mut entries = Vec::new();
        for item in m.get(&1).and_then(Value::as_array).unwrap_or_default() {
            let [p, mode, blob, size] = item.as_array().unwrap_or_default() else {
                return schema("a tree entry is [path, mode, blob, size]");
            };
            let path = get_text(p, "a path")?;
            let mode = get_uint(mode, "a mode")? as u32;
            if !path_ok(&path) {
                return schema(format!("{path:?} is not a clean relative path"));
            }
            if ![MODE_FILE, MODE_EXEC, MODE_LINK].contains(&mode) {
                return schema(format!(
                    "{path}: mode {mode:o} is not a file, an executable or a link"
                ));
            }
            if entries.last().is_some_and(|l: &Entry| l.path >= path) {
                return schema(format!(
                    "{path}: entries are not in path order, or one is there twice"
                ));
            }
            entries.push(Entry {
                path,
                mode,
                blob: get_obj(blob, "a blob")?,
                size: get_uint(size, "a size")?,
            });
        }
        Ok(DirTree { entries })
    }
}

/// An artifact (docs/cylinder.md §4.1).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Artifact {
    pub kind: Kind,
    /// `x86_64`, `riscv64`, `aarch64`.
    pub arch: Option<String>,
    /// `linux`, `darwin`, `none`.
    pub os: Option<String>,
    /// The [`DirTree`]'s blob.
    pub tree: ObjectRef,
    /// The default command, for a container or an image.
    pub entrypoint: Option<Vec<String>>,
    pub env: BTreeMap<String, String>,
    pub labels: BTreeMap<String, String>,
    /// The files' bytes together.
    pub total_size: u64,
    /// The receipt of the Flow job that built it, when one did.
    pub receipt: Option<ObjectRef>,
}

fn pairs(m: &BTreeMap<String, String>) -> Value {
    if m.is_empty() {
        return Value::Null;
    }
    Value::Array(
        m.iter()
            .map(|(k, v)| Value::Array(vec![Value::Text(k.clone()), Value::Text(v.clone())]))
            .collect(),
    )
}

fn get_pairs(v: Option<&Value>, what: &str) -> R<BTreeMap<String, String>> {
    let mut out = BTreeMap::new();
    let Some(v) = v.filter(|v| !v.is_null()) else {
        return Ok(out);
    };
    for p in v
        .as_array()
        .map_or_else(|| schema(format!("{what} is a list of pairs")), Ok)?
    {
        let [k, val] = p.as_array().unwrap_or_default() else {
            return schema(format!("{what}: a pair is [key, value]"));
        };
        let k = get_text(k, what)?;
        if out.insert(k.clone(), get_text(val, what)?).is_some() {
            return schema(format!("{what}: {k} is there twice"));
        }
    }
    Ok(out)
}

fn opt_text(v: &Option<String>) -> Value {
    v.as_ref().map_or(Value::Null, |s| Value::Text(s.clone()))
}

impl Artifact {
    /// Its canonical CBOR, keys as in docs/cylinder.md §4.1.
    pub fn encode(&self) -> Vec<u8> {
        let entry = self.entrypoint.as_ref().map_or(Value::Null, |a| {
            Value::Array(a.iter().cloned().map(Value::Text).collect())
        });
        cbor::encode(&Value::map([
            (0, Value::Uint(VERSION)),
            (1, Value::Text(self.kind.as_str().into())),
            (2, opt_text(&self.arch)),
            (3, opt_text(&self.os)),
            (4, obj(&self.tree)),
            (5, entry),
            (6, pairs(&self.env)),
            (7, pairs(&self.labels)),
            (8, Value::Uint(self.total_size)),
            (9, self.receipt.as_ref().map_or(Value::Null, obj)),
        ]))
    }

    pub fn decode(bytes: &[u8]) -> R<Artifact> {
        let v = cbor::decode(bytes)?;
        let m = v
            .as_map()
            .map_or_else(|| schema("an artifact is a map"), Ok)?;
        if m.get(&0).and_then(Value::as_uint) != Some(VERSION) {
            return schema(format!(
                "artifact version {:?} is not known; this reads {VERSION}",
                m.get(&0)
            ));
        }
        if let Some(k) = m.keys().find(|k| **k > 9) {
            return schema(format!("key {k} is not part of an artifact"));
        }
        let text = |k: u64, what: &str| -> R<Option<String>> {
            match m.get(&k) {
                None | Some(Value::Null) => Ok(None),
                Some(v) => get_text(v, what).map(Some),
            }
        };
        let entrypoint = match m.get(&5) {
            None | Some(Value::Null) => None,
            Some(v) => Some(
                v.as_array()
                    .unwrap_or_default()
                    .iter()
                    .map(|a| get_text(a, "the entrypoint"))
                    .collect::<R<_>>()?,
            ),
        };
        let receipt = match m.get(&9) {
            None | Some(Value::Null) => None,
            Some(v) => Some(get_obj(v, "the receipt")?),
        };
        Ok(Artifact {
            kind: Kind::parse(&text(1, "the kind")?.unwrap_or_default())?,
            arch: text(2, "the architecture")?,
            os: text(3, "the system")?,
            tree: get_obj(m.get(&4).unwrap_or(&Value::Null), "the tree")?,
            entrypoint,
            env: get_pairs(m.get(&6), "env")?,
            labels: get_pairs(m.get(&7), "labels")?,
            total_size: get_uint(m.get(&8).unwrap_or(&Value::Null), "the size")?,
            receipt,
        })
    }

    /// Its name: its object's hash, as every Oxygen object is named.
    pub fn name(&self) -> ObjectRef {
        hash_object(ObjectType::Artifact, &self.encode())
    }
}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1	//! Artifacts (Cylinder, docs/cylinder.md §4.1, oxygen#108): an immutable
2	//! object naming a tree of files, each an ordinary blob of FastCDC BLAKE3
3	//! chunks, so an image rebuilt with one changed file moves only that file's
4	//! changed chunks, and a binary in an archive and in an image is the same
5	//! chunks. An artifact is an object of its own (`artifact`) in canonical
6	//! CBOR; its tree is a blob. The `cylinder` crate packs, unpacks and keeps
7	//! them.
8
9	use crate::cbor::{self, Value};
10	use crate::hash::{hash_object, ObjectRef, ObjectType};
11	use crate::objects::{schema, ObjectError};
12	use std::collections::BTreeMap;
13
14	type R<T> = std::result::Result<T, ObjectError>;
15
16	/// The encoding's version this crate writes and reads.
17	pub const VERSION: u64 = 1;
18
19	/// What an artifact is for.
20	#[derive(Clone, Copy, Debug, PartialEq, Eq)]
21	pub enum Kind {
22	Container,
23	RunnerImage,
24	Package,
25	Archive,
26	Report,
27	}
28
29	impl Kind {
30	pub const ALL: [Kind; 5] = [
31	Kind::Container,
32	Kind::RunnerImage,
33	Kind::Package,
34	Kind::Archive,
35	Kind::Report,
36	];
37
38	pub fn as_str(self) -> &'static str {
39	match self {
40	Kind::Container => "container",
41	Kind::RunnerImage => "runner-image",
42	Kind::Package => "package",
43	Kind::Archive => "archive",
44	Kind::Report => "report",
45	}
46	}
47
48	pub fn parse(s: &str) -> R<Kind> {
49	Kind::ALL.into_iter().find(\|k\| k.as_str() == s).map_or_else(
50	\|\| {
51	schema(format!(
52	"{s:?} is not a kind of artifact: {}",
53	Kind::ALL.map(\|k\| k.as_str()).join(", ")
54	))
55	},
56	Ok,
57	)
58	}
59	}
60
61	/// A file's mode in a tree: a regular file, an executable one, or a
62	/// symbolic link (whose blob is its target).
63	pub const MODE_FILE: u32 = 0o100644;
64	pub const MODE_EXEC: u32 = 0o100755;
65	pub const MODE_LINK: u32 = 0o120000;
66
67	/// A file of an artifact's tree.
68	#[derive(Clone, Debug, PartialEq, Eq)]
69	pub struct Entry {
70	/// `/`-separated, relative, never `.` or `..` parts.
71	pub path: String,
72	pub mode: u32,
73	pub blob: ObjectRef,
74	pub size: u64,
75	}
76
77	/// An artifact's files, in path order: paths, modes, blobs and sizes.
78	#[derive(Clone, Debug, Default, PartialEq, Eq)]
79	pub struct DirTree {
80	pub entries: Vec<Entry>,
81	}
82
83	fn path_ok(p: &str) -> bool {
84	!p.is_empty()
85	&& !p.starts_with('/')
86	&& p.split('/').all(\|s\| !s.is_empty() && s != "." && s != "..")
87	}
88
89	fn obj(r: &ObjectRef) -> Value {
90	Value::Bytes(r.0.to_vec())
91	}
92
93	fn get_obj(v: &Value, what: &str) -> R<ObjectRef> {
94	match v.as_bytes() {
95	Some(b) if b.len() == 32 => {
96	let mut a = [0u8; 32];
97	a.copy_from_slice(b);
98	Ok(ObjectRef(a))
99	}
100	_ => schema(format!("{what} is not an object reference")),
101	}
102	}
103
104	fn get_text(v: &Value, what: &str) -> R<String> {
105	v.as_text()
106	.map(str::to_string)
107	.map_or_else(\|\| schema(format!("{what} is not text")), Ok)
108	}
109
110	fn get_uint(v: &Value, what: &str) -> R<u64> {
111	v.as_uint()
112	.map_or_else(\|\| schema(format!("{what} is not a number")), Ok)
113	}
114
115	impl DirTree {
116	pub fn encode(&self) -> Vec<u8> {
117	let entries = self
118	.entries
119	.iter()
120	.map(\|e\| {
121	Value::Array(vec![
122	Value::Text(e.path.clone()),
123	Value::Uint(e.mode as u64),
124	obj(&e.blob),
125	Value::Uint(e.size),
126	])
127	})
128	.collect();
129	cbor::encode(&Value::map([
130	(0, Value::Uint(VERSION)),
131	(1, Value::Array(entries)),
132	]))
133	}
134
135	/// Read and check a tree: a known version, clean paths, each once, in
136	/// order, with a known mode.
137	pub fn decode(bytes: &[u8]) -> R<DirTree> {
138	let v = cbor::decode(bytes)?;
139	let m = v.as_map().map_or_else(\|\| schema("a tree is a map"), Ok)?;
140	if m.get(&0).and_then(Value::as_uint) != Some(VERSION) {
141	return schema(format!(
142	"tree version {:?} is not known; this reads {VERSION}",
143	m.get(&0)
144	));
145	}
146	let mut entries = Vec::new();
147	for item in m.get(&1).and_then(Value::as_array).unwrap_or_default() {
148	let [p, mode, blob, size] = item.as_array().unwrap_or_default() else {
149	return schema("a tree entry is [path, mode, blob, size]");
150	};
151	let path = get_text(p, "a path")?;
152	let mode = get_uint(mode, "a mode")? as u32;
153	if !path_ok(&path) {
154	return schema(format!("{path:?} is not a clean relative path"));
155	}
156	if ![MODE_FILE, MODE_EXEC, MODE_LINK].contains(&mode) {
157	return schema(format!(
158	"{path}: mode {mode:o} is not a file, an executable or a link"
159	));
160	}
161	if entries.last().is_some_and(\|l: &Entry\| l.path >= path) {
162	return schema(format!(
163	"{path}: entries are not in path order, or one is there twice"
164	));
165	}
166	entries.push(Entry {
167	path,
168	mode,
169	blob: get_obj(blob, "a blob")?,
170	size: get_uint(size, "a size")?,
171	});
172	}
173	Ok(DirTree { entries })
174	}
175	}
176
177	/// An artifact (docs/cylinder.md §4.1).
178	#[derive(Clone, Debug, PartialEq, Eq)]
179	pub struct Artifact {
180	pub kind: Kind,
181	/// `x86_64`, `riscv64`, `aarch64`.
182	pub arch: Option<String>,
183	/// `linux`, `darwin`, `none`.
184	pub os: Option<String>,
185	/// The [`DirTree`]'s blob.
186	pub tree: ObjectRef,
187	/// The default command, for a container or an image.
188	pub entrypoint: Option<Vec<String>>,
189	pub env: BTreeMap<String, String>,
190	pub labels: BTreeMap<String, String>,
191	/// The files' bytes together.
192	pub total_size: u64,
193	/// The receipt of the Flow job that built it, when one did.
194	pub receipt: Option<ObjectRef>,
195	}
196
197	fn pairs(m: &BTreeMap<String, String>) -> Value {
198	if m.is_empty() {
199	return Value::Null;
200	}
201	Value::Array(
202	m.iter()
203	.map(\|(k, v)\| Value::Array(vec![Value::Text(k.clone()), Value::Text(v.clone())]))
204	.collect(),
205	)
206	}
207
208	fn get_pairs(v: Option<&Value>, what: &str) -> R<BTreeMap<String, String>> {
209	let mut out = BTreeMap::new();
210	let Some(v) = v.filter(\|v\| !v.is_null()) else {
211	return Ok(out);
212	};
213	for p in v
214	.as_array()
215	.map_or_else(\|\| schema(format!("{what} is a list of pairs")), Ok)?
216	{
217	let [k, val] = p.as_array().unwrap_or_default() else {
218	return schema(format!("{what}: a pair is [key, value]"));
219	};
220	let k = get_text(k, what)?;
221	if out.insert(k.clone(), get_text(val, what)?).is_some() {
222	return schema(format!("{what}: {k} is there twice"));
223	}
224	}
225	Ok(out)
226	}
227
228	fn opt_text(v: &Option<String>) -> Value {
229	v.as_ref().map_or(Value::Null, \|s\| Value::Text(s.clone()))
230	}
231
232	impl Artifact {
233	/// Its canonical CBOR, keys as in docs/cylinder.md §4.1.
234	pub fn encode(&self) -> Vec<u8> {
235	let entry = self.entrypoint.as_ref().map_or(Value::Null, \|a\| {
236	Value::Array(a.iter().cloned().map(Value::Text).collect())
237	});
238	cbor::encode(&Value::map([
239	(0, Value::Uint(VERSION)),
240	(1, Value::Text(self.kind.as_str().into())),
241	(2, opt_text(&self.arch)),
242	(3, opt_text(&self.os)),
243	(4, obj(&self.tree)),
244	(5, entry),
245	(6, pairs(&self.env)),
246	(7, pairs(&self.labels)),
247	(8, Value::Uint(self.total_size)),
248	(9, self.receipt.as_ref().map_or(Value::Null, obj)),
249	]))
250	}
251
252	pub fn decode(bytes: &[u8]) -> R<Artifact> {
253	let v = cbor::decode(bytes)?;
254	let m = v
255	.as_map()
256	.map_or_else(\|\| schema("an artifact is a map"), Ok)?;
257	if m.get(&0).and_then(Value::as_uint) != Some(VERSION) {
258	return schema(format!(
259	"artifact version {:?} is not known; this reads {VERSION}",
260	m.get(&0)
261	));
262	}
263	if let Some(k) = m.keys().find(\|k\| **k > 9) {
264	return schema(format!("key {k} is not part of an artifact"));
265	}
266	let text = \|k: u64, what: &str\| -> R<Option<String>> {
267	match m.get(&k) {
268	None \| Some(Value::Null) => Ok(None),
269	Some(v) => get_text(v, what).map(Some),
270	}
271	};
272	let entrypoint = match m.get(&5) {
273	None \| Some(Value::Null) => None,
274	Some(v) => Some(
275	v.as_array()
276	.unwrap_or_default()
277	.iter()
278	.map(\|a\| get_text(a, "the entrypoint"))
279	.collect::<R<_>>()?,
280	),
281	};
282	let receipt = match m.get(&9) {
283	None \| Some(Value::Null) => None,
284	Some(v) => Some(get_obj(v, "the receipt")?),
285	};
286	Ok(Artifact {
287	kind: Kind::parse(&text(1, "the kind")?.unwrap_or_default())?,
288	arch: text(2, "the architecture")?,
289	os: text(3, "the system")?,
290	tree: get_obj(m.get(&4).unwrap_or(&Value::Null), "the tree")?,
291	entrypoint,
292	env: get_pairs(m.get(&6), "env")?,
293	labels: get_pairs(m.get(&7), "labels")?,
294	total_size: get_uint(m.get(&8).unwrap_or(&Value::Null), "the size")?,
295	receipt,
296	})
297	}
298
299	/// Its name: its object's hash, as every Oxygen object is named.
300	pub fn name(&self) -> ObjectRef {
301	hash_object(ObjectType::Artifact, &self.encode())
302	}
303	}

Keyboard