OXYGENHUB
◆ v0.11.0 oxygen / crates / cylinder / src / lib.rs
14.7 KiB · Rust · last changed The artifact as an object type of its own: oxy-core's artifact module, named by its object hash, checked by fsck (refs #108) 7 days ago source blame history raw
1
//! Cylinder, step C1 (docs/cylinder.md §4.1): an **artifact** is an
2
//! immutable, content-addressed object naming a tree of files, each kept as
3
//! an ordinary Oxygen blob of FastCDC BLAKE3 chunks. An image rebuilt with
4
//! one changed file moves only that file's changed chunks, and a binary in
5
//! a release archive and in a runner image is the same chunks.
6
//!
7
//! The model is oxy-core's ([`oxy_core::artifact`]): an `artifact` object
8
//! in canonical CBOR, named by its object hash, whose tree is a blob. This
9
//! crate packs a directory into one, opens, verifies and unpacks it, and
10
//! keeps it: `.oxy/artifacts/NAME` records each artifact this repository
11
//! keeps, which garbage collection keeps with its tree and files
12
//! ([`roots`]).
13
14
use oxy_core::hash::{ObjectRef, ObjectType};
15
use oxy_core::repo::Repo;
16
use std::collections::BTreeMap;
17
use std::path::{Path, PathBuf};
18
19
pub use oxy_core::artifact::{
20
    Artifact, DirTree, Entry, Kind, MODE_EXEC, MODE_FILE, MODE_LINK, VERSION,
21
};
22
23
#[derive(Debug, thiserror::Error)]
24
pub enum Error {
25
    #[error(transparent)]
26
    Repo(#[from] oxy_core::repo::RepoError),
27
    #[error("io: {0}")]
28
    Io(#[from] std::io::Error),
29
    #[error("not an artifact: {0}")]
30
    Object(#[from] oxy_core::objects::ObjectError),
31
    #[error("store: {0}")]
32
    Store(#[from] oxy_core::store::StoreError),
33
    #[error("{0}")]
34
    Bad(String),
35
}
36
37
pub type Result<T> = std::result::Result<T, Error>;
38
39
fn bad<T>(msg: impl Into<String>) -> Result<T> {
40
    Err(Error::Bad(msg.into()))
41
}
42
43
/// What to pack a directory as.
44
#[derive(Clone, Debug)]
45
pub struct Spec {
46
    pub kind: Kind,
47
    pub arch: Option<String>,
48
    pub os: Option<String>,
49
    pub entrypoint: Option<Vec<String>>,
50
    pub env: BTreeMap<String, String>,
51
    pub labels: BTreeMap<String, String>,
52
}
53
54
impl Spec {
55
    pub fn new(kind: Kind) -> Spec {
56
        Spec {
57
            kind,
58
            arch: None,
59
            os: None,
60
            entrypoint: None,
61
            env: BTreeMap::new(),
62
            labels: BTreeMap::new(),
63
        }
64
    }
65
}
66
67
/// An artifact kept in a repository, with its name.
68
#[derive(Clone, Debug)]
69
pub struct Kept {
70
    pub name: String,
71
    pub artifact: Artifact,
72
    /// Its object (its name, as a reference).
73
    pub object: ObjectRef,
74
}
75
76
fn kept_dir(repo: &Repo) -> PathBuf {
77
    repo.oxy_dir().join("artifacts")
78
}
79
80
/// Every file under `dir`, by relative path: regular files and links, in
81
/// path order. `.oxy` and `.git` directories are not part of an artifact.
82
fn walk(dir: &Path) -> Result<Vec<(String, PathBuf, std::fs::Metadata)>> {
83
    let mut out = Vec::new();
84
    let mut stack = vec![(dir.to_path_buf(), String::new())];
85
    while let Some((d, rel)) = stack.pop() {
86
        for e in std::fs::read_dir(&d)? {
87
            let e = e?;
88
            let name = e
89
                .file_name()
90
                .into_string()
91
                .map_err(|n| Error::Bad(format!("{n:?} is not UTF-8")))?;
92
            let path = if rel.is_empty() {
93
                name.clone()
94
            } else {
95
                format!("{rel}/{name}")
96
            };
97
            let md = std::fs::symlink_metadata(e.path())?;
98
            if md.is_dir() {
99
                if name != ".oxy" && name != ".git" {
100
                    stack.push((e.path(), path));
101
                }
102
            } else if md.is_file() || md.file_type().is_symlink() {
103
                out.push((path, e.path(), md));
104
            }
105
        }
106
    }
107
    out.sort_by(|a, b| a.0.cmp(&b.0));
108
    Ok(out)
109
}
110
111
#[cfg(unix)]
112
fn mode_of(md: &std::fs::Metadata) -> u32 {
113
    use std::os::unix::fs::PermissionsExt;
114
    if md.file_type().is_symlink() {
115
        MODE_LINK
116
    } else if md.permissions().mode() & 0o111 != 0 {
117
        MODE_EXEC
118
    } else {
119
        MODE_FILE
120
    }
121
}
122
123
#[cfg(not(unix))]
124
fn mode_of(md: &std::fs::Metadata) -> u32 {
125
    if md.file_type().is_symlink() {
126
        MODE_LINK
127
    } else {
128
        MODE_FILE
129
    }
130
}
131
132
/// Pack the files under `dir` into an artifact of `spec`'s kind in
133
/// `repo`'s store: each file a blob, so files and chunks the store holds
134
/// already cost nothing; then its tree and its encoding, kept under its
135
/// name.
136
pub fn pack(repo: &Repo, dir: &Path, spec: &Spec) -> Result<Kept> {
137
    if !dir.is_dir() {
138
        return bad(format!("{} is not a directory", dir.display()));
139
    }
140
    let mut tree = DirTree::default();
141
    for (path, full, md) in walk(dir)? {
142
        let mode = mode_of(&md);
143
        let bytes = if mode == MODE_LINK {
144
            std::fs::read_link(&full)?
145
                .to_string_lossy()
146
                .into_owned()
147
                .into_bytes()
148
        } else {
149
            std::fs::read(&full)?
150
        };
151
        let blob = repo.store_blob(&bytes)?;
152
        tree.entries.push(Entry {
153
            path,
154
            mode,
155
            blob,
156
            size: bytes.len() as u64,
157
        });
158
    }
159
    let artifact = Artifact {
160
        kind: spec.kind,
161
        arch: spec.arch.clone(),
162
        os: spec.os.clone(),
163
        tree: repo.store_blob(&tree.encode())?,
164
        entrypoint: spec.entrypoint.clone(),
165
        env: spec.env.clone(),
166
        labels: spec.labels.clone(),
167
        total_size: tree.entries.iter().map(|e| e.size).sum(),
168
        receipt: None,
169
    };
170
    let object = repo.store().put(ObjectType::Artifact, &artifact.encode())?;
171
    let name = object.to_hex();
172
    std::fs::create_dir_all(kept_dir(repo))?;
173
    std::fs::write(kept_dir(repo).join(&name), b"")?;
174
    Ok(Kept {
175
        name,
176
        artifact,
177
        object,
178
    })
179
}
180
181
/// The names of the artifacts kept in `repo`, sorted.
182
pub fn list(repo: &Repo) -> Result<Vec<String>> {
183
    let mut out = Vec::new();
184
    match std::fs::read_dir(kept_dir(repo)) {
185
        Ok(rd) => {
186
            for e in rd {
187
                let n = e?.file_name().to_string_lossy().into_owned();
188
                if n.len() == 64 && n.bytes().all(|b| b.is_ascii_hexdigit()) {
189
                    out.push(n);
190
                }
191
            }
192
        }
193
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
194
        Err(e) => return Err(e.into()),
195
    }
196
    out.sort();
197
    Ok(out)
198
}
199
200
/// The kept artifact `name` names, or a unique prefix of it (at least six
201
/// characters), read from the store, which checks it against its name.
202
pub fn open(repo: &Repo, name: &str) -> Result<Kept> {
203
    let name = name.to_ascii_lowercase();
204
    if name.len() < 6 {
205
        return bad("name an artifact by at least six characters of its name");
206
    }
207
    let found: Vec<String> = list(repo)?
208
        .into_iter()
209
        .filter(|n| n.starts_with(&name))
210
        .collect();
211
    let full = match found.as_slice() {
212
        [one] => one.clone(),
213
        [] => {
214
            return bad(format!(
215
                "no artifact {name} here; `oxy artifact list` shows them"
216
            ))
217
        }
218
        _ => {
219
            return bad(format!(
220
                "{name} names {} artifacts: give more of the name",
221
                found.len()
222
            ))
223
        }
224
    };
225
    let object = ObjectRef::from_hex(&full).map_or_else(
226
        || bad(format!("artifact {full}: its record is damaged")),
227
        Ok,
228
    )?;
229
    // The store checks the bytes against the object's name as it reads.
230
    let bytes = repo.store().get_typed(&object, ObjectType::Artifact)?;
231
    Ok(Kept {
232
        name: full,
233
        artifact: Artifact::decode(&bytes)?,
234
        object,
235
    })
236
}
237
238
/// An artifact's tree.
239
pub fn tree(repo: &Repo, a: &Artifact) -> Result<DirTree> {
240
    Ok(DirTree::decode(&repo.read_blob(&a.tree)?)?)
241
}
242
243
/// Check that every file of `k` is in the store whole: its bytes read, at
244
/// the size its tree says. Answers how many files were read.
245
pub fn verify(repo: &Repo, k: &Kept) -> Result<usize> {
246
    let t = tree(repo, &k.artifact)?;
247
    let mut total = 0;
248
    for e in &t.entries {
249
        let n = repo.read_blob(&e.blob)?.len() as u64;
250
        if n != e.size {
251
            return bad(format!(
252
                "{}: {n} bytes, but the tree says {}",
253
                e.path, e.size
254
            ));
255
        }
256
        total += n;
257
    }
258
    if total != k.artifact.total_size {
259
        return bad(format!(
260
            "the files are {total} bytes together, but the artifact says {}",
261
            k.artifact.total_size
262
        ));
263
    }
264
    Ok(t.entries.len())
265
}
266
267
/// Write `k`'s files under `dest`, which must not exist or be empty.
268
pub fn unpack(repo: &Repo, k: &Kept, dest: &Path) -> Result<usize> {
269
    if dest.exists() && std::fs::read_dir(dest)?.next().is_some() {
270
        return bad(format!("{} is not empty", dest.display()));
271
    }
272
    let t = tree(repo, &k.artifact)?;
273
    for e in &t.entries {
274
        let out = dest.join(&e.path);
275
        if let Some(p) = out.parent() {
276
            std::fs::create_dir_all(p)?;
277
        }
278
        let bytes = repo.read_blob(&e.blob)?;
279
        if e.mode == MODE_LINK {
280
            #[cfg(unix)]
281
            std::os::unix::fs::symlink(String::from_utf8_lossy(&bytes).as_ref(), &out)?;
282
            #[cfg(not(unix))]
283
            std::fs::write(&out, &bytes)?;
284
            continue;
285
        }
286
        std::fs::write(&out, &bytes)?;
287
        #[cfg(unix)]
288
        if e.mode == MODE_EXEC {
289
            use std::os::unix::fs::PermissionsExt;
290
            std::fs::set_permissions(&out, std::fs::Permissions::from_mode(0o755))?;
291
        }
292
    }
293
    Ok(t.entries.len())
294
}
295
296
/// The blobs garbage collection must keep for the kept artifacts: each
297
/// one's encoding, its tree and its files (`Repo::gc_keeping`).
298
pub fn roots(repo: &Repo) -> Result<Vec<ObjectRef>> {
299
    let mut out = Vec::new();
300
    for name in list(repo)? {
301
        let k = open(repo, &name)?;
302
        out.push(k.object);
303
        out.push(k.artifact.tree);
304
        out.extend(tree(repo, &k.artifact)?.entries.iter().map(|e| e.blob));
305
    }
306
    Ok(out)
307
}
308
309
#[cfg(test)]
310
mod tests {
311
    use super::*;
312
    use oxy_core::cbor::{self, Value};
313
314
    fn repo(dir: &Path) -> Repo {
315
        Repo::init(&dir.join("repo")).unwrap()
316
    }
317
318
    #[test]
319
    fn a_directory_packs_into_an_artifact_and_back() {
320
        let t = tempfile::tempdir().unwrap();
321
        let r = repo(t.path());
322
        let src = t.path().join("out");
323
        std::fs::create_dir_all(src.join("bin")).unwrap();
324
        std::fs::write(src.join("README"), b"hello\n").unwrap();
325
        std::fs::write(src.join("bin/tool"), vec![7u8; 300_000]).unwrap();
326
        #[cfg(unix)]
327
        {
328
            use std::os::unix::fs::PermissionsExt;
329
            std::fs::set_permissions(src.join("bin/tool"), std::fs::Permissions::from_mode(0o755))
330
                .unwrap();
331
            std::os::unix::fs::symlink("bin/tool", src.join("tool")).unwrap();
332
        }
333
        let mut spec = Spec::new(Kind::Archive);
334
        spec.arch = Some("x86_64".into());
335
        spec.labels.insert("version".into(), "0.1.0".into());
336
        let k = pack(&r, &src, &spec).unwrap();
337
        assert_eq!(k.name, k.artifact.name().to_hex());
338
        assert_eq!(
339
            k.artifact.total_size,
340
            300_006 + if cfg!(unix) { 8 } else { 0 }
341
        );
342
        assert_eq!(list(&r).unwrap(), vec![k.name.clone()]);
343
344
        let again = open(&r, &k.name[..10]).unwrap();
345
        assert_eq!(again.artifact, k.artifact);
346
        let paths: Vec<(String, u32)> = tree(&r, &again.artifact)
347
            .unwrap()
348
            .entries
349
            .into_iter()
350
            .map(|e| (e.path, e.mode))
351
            .collect();
352
        assert_eq!(paths[0], ("README".into(), MODE_FILE));
353
        assert_eq!(
354
            paths[1],
355
            (
356
                "bin/tool".into(),
357
                if cfg!(unix) { MODE_EXEC } else { MODE_FILE }
358
            )
359
        );
360
        assert_eq!(verify(&r, &again).unwrap(), paths.len());
361
362
        let dest = t.path().join("back");
363
        unpack(&r, &again, &dest).unwrap();
364
        assert_eq!(
365
            std::fs::read(dest.join("bin/tool")).unwrap(),
366
            vec![7u8; 300_000]
367
        );
368
        #[cfg(unix)]
369
        assert_eq!(
370
            std::fs::read_link(dest.join("tool")).unwrap(),
371
            Path::new("bin/tool")
372
        );
373
        assert!(unpack(&r, &again, &dest)
374
            .unwrap_err()
375
            .to_string()
376
            .contains("not empty"));
377
378
        // The same files pack to the same name; garbage collection keeps it.
379
        assert_eq!(pack(&r, &src, &spec).unwrap().name, k.name);
380
        let keep = roots(&r).unwrap();
381
        assert!(keep.contains(&k.object) && keep.contains(&k.artifact.tree));
382
        r.gc_keeping(true, &keep, &mut |_| {}).unwrap();
383
        // An artifact is an object of its own, which fsck reads and accepts.
384
        let report = r.fsck().unwrap();
385
        let errors: Vec<_> = report
386
            .problems
387
            .iter()
388
            .filter(|p| p.severity == oxy_core::repo::Severity::Error)
389
            .collect();
390
        assert!(errors.is_empty(), "{errors:?}");
391
        assert_eq!(
392
            verify(&r, &open(&r, &k.name).unwrap()).unwrap(),
393
            paths.len()
394
        );
395
    }
396
397
    #[test]
398
    fn encodings_are_canonical_and_checked() {
399
        let a = Artifact {
400
            kind: Kind::RunnerImage,
401
            arch: Some("riscv64".into()),
402
            os: Some("linux".into()),
403
            tree: ObjectRef([1; 32]),
404
            entrypoint: Some(vec!["/sbin/init".into()]),
405
            env: [("PATH".to_string(), "/bin".to_string())].into(),
406
            labels: BTreeMap::new(),
407
            total_size: 42,
408
            receipt: Some(ObjectRef([2; 32])),
409
        };
410
        let bytes = a.encode();
411
        assert_eq!(Artifact::decode(&bytes).unwrap(), a);
412
        assert_eq!(
413
            a.name(),
414
            oxy_core::hash::hash_object(ObjectType::Artifact, &bytes)
415
        );
416
        assert!(Kind::parse("tarball")
417
            .unwrap_err()
418
            .to_string()
419
            .contains("runner-image"));
420
421
        let wrong = |v: Value| Artifact::decode(&cbor::encode(&v)).unwrap_err().to_string();
422
        assert!(wrong(Value::map([(0, Value::Uint(2))])).contains("version"));
423
        assert!(wrong(Value::map([(0, Value::Uint(1)), (10, Value::Null)])).contains("key 10"));
424
425
        let t = DirTree {
426
            entries: vec![Entry {
427
                path: "a".into(),
428
                mode: MODE_FILE,
429
                blob: ObjectRef([3; 32]),
430
                size: 1,
431
            }],
432
        };
433
        assert_eq!(DirTree::decode(&t.encode()).unwrap(), t);
434
        let entry = |p: &str, mode: u32| {
435
            Value::Array(vec![
436
                Value::Text(p.into()),
437
                Value::Uint(mode as u64),
438
                Value::Bytes(vec![0; 32]),
439
                Value::Uint(0),
440
            ])
441
        };
442
        let tree = |items: Vec<Value>| {
443
            DirTree::decode(&cbor::encode(&Value::map([
444
                (0, Value::Uint(1)),
445
                (1, Value::Array(items)),
446
            ])))
447
            .unwrap_err()
448
            .to_string()
449
        };
450
        assert!(tree(vec![entry("../x", MODE_FILE)]).contains("clean"));
451
        assert!(tree(vec![entry("b", MODE_FILE), entry("a", MODE_FILE)]).contains("order"));
452
        assert!(tree(vec![entry("a", 0o040000)]).contains("mode"));
453
    }
454
}