| 642da263 Andy McCall ↶ |
1 |
|
| 2 |
|
| 3 |
|
| 4 |
|
| 5 |
|
| 6 |
|
| 6c3dbe20 Andy McCall ↶ |
7 |
|
| 8 |
|
| 9 |
|
| 10 |
|
| 11 |
|
| 12 |
|
| 642da263 Andy McCall ↶ |
13 |
|
| 6c3dbe20 Andy McCall ↶ |
14 |
use oxy_core::hash::{ObjectRef, ObjectType}; |
| 642da263 Andy McCall ↶ |
15 |
use oxy_core::repo::Repo; |
| 16 |
use std::collections::BTreeMap; |
| 17 |
use std::path::{Path, PathBuf}; |
| 18 |
|
| 6c3dbe20 Andy McCall ↶ |
19 |
pub use oxy_core::artifact::{ |
| 20 |
Artifact, DirTree, Entry, Kind, MODE_EXEC, MODE_FILE, MODE_LINK, VERSION, |
| 21 |
}; |
| 642da263 Andy McCall ↶ |
22 |
|
| 23 |
#[derive(Debug, thiserror::Error)] |
| 24 |
pub enum Error { |
| 25 |
#[error(transparent)] |
| 26 |
Repo(#[from] oxy_core::repo::RepoError), |
| 27 |
#[error("io: {0}")] |
| 28 |
Io(#[from] std::io::Error), |
| 6c3dbe20 Andy McCall ↶ |
29 |
#[error("not an artifact: {0}")] |
| 30 |
Object(#[from] oxy_core::objects::ObjectError), |
| 31 |
#[error("store: {0}")] |
| 32 |
Store(#[from] oxy_core::store::StoreError), |
| 642da263 Andy McCall ↶ |
33 |
#[error("{0}")] |
| 34 |
Bad(String), |
| 35 |
} |
| 36 |
|
| 37 |
pub type Result<T> = std::result::Result<T, Error>; |
| 38 |
|
| 39 |
fn bad<T>(msg: impl Into<String>) -> Result<T> { |
| 40 |
Err(Error::Bad(msg.into())) |
| 41 |
} |
| 42 |
|
| 43 |
|
| 44 |
#[derive(Clone, Debug)] |
| 45 |
pub struct Spec { |
| 46 |
pub kind: Kind, |
| 47 |
pub arch: Option<String>, |
| 48 |
pub os: Option<String>, |
| 49 |
pub entrypoint: Option<Vec<String>>, |
| 50 |
pub env: BTreeMap<String, String>, |
| 51 |
pub labels: BTreeMap<String, String>, |
| 52 |
} |
| 53 |
|
| 54 |
impl Spec { |
| 55 |
pub fn new(kind: Kind) -> Spec { |
| 56 |
Spec { |
| 57 |
kind, |
| 58 |
arch: None, |
| 59 |
os: None, |
| 60 |
entrypoint: None, |
| 61 |
env: BTreeMap::new(), |
| 62 |
labels: BTreeMap::new(), |
| 63 |
} |
| 64 |
} |
| 65 |
} |
| 66 |
|
| 67 |
|
| 68 |
#[derive(Clone, Debug)] |
| 69 |
pub struct Kept { |
| 70 |
pub name: String, |
| 71 |
pub artifact: Artifact, |
| 6c3dbe20 Andy McCall ↶ |
72 |
|
| 73 |
pub object: ObjectRef, |
| 642da263 Andy McCall ↶ |
74 |
} |
| 75 |
|
| 76 |
fn kept_dir(repo: &Repo) -> PathBuf { |
| 77 |
repo.oxy_dir().join("artifacts") |
| 78 |
} |
| 79 |
|
| 80 |
|
| 81 |
|
| 82 |
fn walk(dir: &Path) -> Result<Vec<(String, PathBuf, std::fs::Metadata)>> { |
| 83 |
let mut out = Vec::new(); |
| 84 |
let mut stack = vec![(dir.to_path_buf(), String::new())]; |
| 85 |
while let Some((d, rel)) = stack.pop() { |
| 86 |
for e in std::fs::read_dir(&d)? { |
| 87 |
let e = e?; |
| 88 |
let name = e |
| 89 |
.file_name() |
| 90 |
.into_string() |
| 91 |
.map_err(|n| Error::Bad(format!("{n:?} is not UTF-8")))?; |
| 92 |
let path = if rel.is_empty() { |
| 93 |
name.clone() |
| 94 |
} else { |
| 95 |
format!("{rel}/{name}") |
| 96 |
}; |
| 97 |
let md = std::fs::symlink_metadata(e.path())?; |
| 98 |
if md.is_dir() { |
| 99 |
if name != ".oxy" && name != ".git" { |
| 100 |
stack.push((e.path(), path)); |
| 101 |
} |
| 102 |
} else if md.is_file() || md.file_type().is_symlink() { |
| 103 |
out.push((path, e.path(), md)); |
| 104 |
} |
| 105 |
} |
| 106 |
} |
| 107 |
out.sort_by(|a, b| a.0.cmp(&b.0)); |
| 108 |
Ok(out) |
| 109 |
} |
| 110 |
|
| 111 |
#[cfg(unix)] |
| 112 |
fn mode_of(md: &std::fs::Metadata) -> u32 { |
| 113 |
use std::os::unix::fs::PermissionsExt; |
| 114 |
if md.file_type().is_symlink() { |
| 115 |
MODE_LINK |
| 116 |
} else if md.permissions().mode() & 0o111 != 0 { |
| 117 |
MODE_EXEC |
| 118 |
} else { |
| 119 |
MODE_FILE |
| 120 |
} |
| 121 |
} |
| 122 |
|
| 123 |
#[cfg(not(unix))] |
| 124 |
fn mode_of(md: &std::fs::Metadata) -> u32 { |
| 125 |
if md.file_type().is_symlink() { |
| 126 |
MODE_LINK |
| 127 |
} else { |
| 128 |
MODE_FILE |
| 129 |
} |
| 130 |
} |
| 131 |
|
| 132 |
|
| 133 |
|
| 134 |
|
| 135 |
|
| 136 |
pub fn pack(repo: &Repo, dir: &Path, spec: &Spec) -> Result<Kept> { |
| 137 |
if !dir.is_dir() { |
| 138 |
return bad(format!("{} is not a directory", dir.display())); |
| 139 |
} |
| 140 |
let mut tree = DirTree::default(); |
| 141 |
for (path, full, md) in walk(dir)? { |
| 142 |
let mode = mode_of(&md); |
| 143 |
let bytes = if mode == MODE_LINK { |
| 144 |
std::fs::read_link(&full)? |
| 145 |
.to_string_lossy() |
| 146 |
.into_owned() |
| 147 |
.into_bytes() |
| 148 |
} else { |
| 149 |
std::fs::read(&full)? |
| 150 |
}; |
| 151 |
let blob = repo.store_blob(&bytes)?; |
| 152 |
tree.entries.push(Entry { |
| 153 |
path, |
| 154 |
mode, |
| 155 |
blob, |
| 156 |
size: bytes.len() as u64, |
| 157 |
}); |
| 158 |
} |
| 159 |
let artifact = Artifact { |
| 160 |
kind: spec.kind, |
| 161 |
arch: spec.arch.clone(), |
| 162 |
os: spec.os.clone(), |
| 163 |
tree: repo.store_blob(&tree.encode())?, |
| 164 |
entrypoint: spec.entrypoint.clone(), |
| 165 |
env: spec.env.clone(), |
| 166 |
labels: spec.labels.clone(), |
| 167 |
total_size: tree.entries.iter().map(|e| e.size).sum(), |
| 168 |
receipt: None, |
| 169 |
}; |
| 6c3dbe20 Andy McCall ↶ |
170 |
let object = repo.store().put(ObjectType::Artifact, &artifact.encode())?; |
| 171 |
let name = object.to_hex(); |
| 642da263 Andy McCall ↶ |
172 |
std::fs::create_dir_all(kept_dir(repo))?; |
| 6c3dbe20 Andy McCall ↶ |
173 |
std::fs::write(kept_dir(repo).join(&name), b"")?; |
| 642da263 Andy McCall ↶ |
174 |
Ok(Kept { |
| 175 |
name, |
| 176 |
artifact, |
| 6c3dbe20 Andy McCall ↶ |
177 |
object, |
| 642da263 Andy McCall ↶ |
178 |
}) |
| 179 |
} |
| 180 |
|
| 181 |
|
| 182 |
pub fn list(repo: &Repo) -> Result<Vec<String>> { |
| 183 |
let mut out = Vec::new(); |
| 184 |
match std::fs::read_dir(kept_dir(repo)) { |
| 185 |
Ok(rd) => { |
| 186 |
for e in rd { |
| 187 |
let n = e?.file_name().to_string_lossy().into_owned(); |
| 188 |
if n.len() == 64 && n.bytes().all(|b| b.is_ascii_hexdigit()) { |
| 189 |
out.push(n); |
| 190 |
} |
| 191 |
} |
| 192 |
} |
| 193 |
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} |
| 194 |
Err(e) => return Err(e.into()), |
| 195 |
} |
| 196 |
out.sort(); |
| 197 |
Ok(out) |
| 198 |
} |
| 199 |
|
| 200 |
|
| 6c3dbe20 Andy McCall ↶ |
201 |
|
| 642da263 Andy McCall ↶ |
202 |
pub fn open(repo: &Repo, name: &str) -> Result<Kept> { |
| 203 |
let name = name.to_ascii_lowercase(); |
| 204 |
if name.len() < 6 { |
| 205 |
return bad("name an artifact by at least six characters of its name"); |
| 206 |
} |
| 207 |
let found: Vec<String> = list(repo)? |
| 208 |
.into_iter() |
| 209 |
.filter(|n| n.starts_with(&name)) |
| 210 |
.collect(); |
| 211 |
let full = match found.as_slice() { |
| 212 |
[one] => one.clone(), |
| 213 |
[] => { |
| 214 |
return bad(format!( |
| 215 |
"no artifact {name} here; `oxy artifact list` shows them" |
| 216 |
)) |
| 217 |
} |
| 218 |
_ => { |
| 219 |
return bad(format!( |
| 220 |
"{name} names {} artifacts: give more of the name", |
| 221 |
found.len() |
| 222 |
)) |
| 223 |
} |
| 224 |
}; |
| 6c3dbe20 Andy McCall ↶ |
225 |
let object = ObjectRef::from_hex(&full).map_or_else( |
| 642da263 Andy McCall ↶ |
226 |
|| bad(format!("artifact {full}: its record is damaged")), |
| 227 |
Ok, |
| 228 |
)?; |
| 6c3dbe20 Andy McCall ↶ |
229 |
|
| 230 |
let bytes = repo.store().get_typed(&object, ObjectType::Artifact)?; |
| 642da263 Andy McCall ↶ |
231 |
Ok(Kept { |
| 232 |
name: full, |
| 233 |
artifact: Artifact::decode(&bytes)?, |
| 6c3dbe20 Andy McCall ↶ |
234 |
object, |
| 642da263 Andy McCall ↶ |
235 |
}) |
| 236 |
} |
| 237 |
|
| 238 |
|
| 239 |
pub fn tree(repo: &Repo, a: &Artifact) -> Result<DirTree> { |
| 6c3dbe20 Andy McCall ↶ |
240 |
Ok(DirTree::decode(&repo.read_blob(&a.tree)?)?) |
| 642da263 Andy McCall ↶ |
241 |
} |
| 242 |
|
| 243 |
|
| 244 |
|
| 245 |
pub fn verify(repo: &Repo, k: &Kept) -> Result<usize> { |
| 246 |
let t = tree(repo, &k.artifact)?; |
| 247 |
let mut total = 0; |
| 248 |
for e in &t.entries { |
| 249 |
let n = repo.read_blob(&e.blob)?.len() as u64; |
| 250 |
if n != e.size { |
| 251 |
return bad(format!( |
| 252 |
"{}: {n} bytes, but the tree says {}", |
| 253 |
e.path, e.size |
| 254 |
)); |
| 255 |
} |
| 256 |
total += n; |
| 257 |
} |
| 258 |
if total != k.artifact.total_size { |
| 259 |
return bad(format!( |
| 260 |
"the files are {total} bytes together, but the artifact says {}", |
| 261 |
k.artifact.total_size |
| 262 |
)); |
| 263 |
} |
| 264 |
Ok(t.entries.len()) |
| 265 |
} |
| 266 |
|
| 267 |
|
| 268 |
pub fn unpack(repo: &Repo, k: &Kept, dest: &Path) -> Result<usize> { |
| 269 |
if dest.exists() && std::fs::read_dir(dest)?.next().is_some() { |
| 270 |
return bad(format!("{} is not empty", dest.display())); |
| 271 |
} |
| 272 |
let t = tree(repo, &k.artifact)?; |
| 273 |
for e in &t.entries { |
| 274 |
let out = dest.join(&e.path); |
| 275 |
if let Some(p) = out.parent() { |
| 276 |
std::fs::create_dir_all(p)?; |
| 277 |
} |
| 278 |
let bytes = repo.read_blob(&e.blob)?; |
| 279 |
if e.mode == MODE_LINK { |
| 280 |
#[cfg(unix)] |
| 281 |
std::os::unix::fs::symlink(String::from_utf8_lossy(&bytes).as_ref(), &out)?; |
| 282 |
#[cfg(not(unix))] |
| 283 |
std::fs::write(&out, &bytes)?; |
| 284 |
continue; |
| 285 |
} |
| 286 |
std::fs::write(&out, &bytes)?; |
| 287 |
#[cfg(unix)] |
| 288 |
if e.mode == MODE_EXEC { |
| 289 |
use std::os::unix::fs::PermissionsExt; |
| 290 |
std::fs::set_permissions(&out, std::fs::Permissions::from_mode(0o755))?; |
| 291 |
} |
| 292 |
} |
| 293 |
Ok(t.entries.len()) |
| 294 |
} |
| 295 |
|
| 296 |
|
| 297 |
|
| 298 |
pub fn roots(repo: &Repo) -> Result<Vec<ObjectRef>> { |
| 299 |
let mut out = Vec::new(); |
| 300 |
for name in list(repo)? { |
| 301 |
let k = open(repo, &name)?; |
| 6c3dbe20 Andy McCall ↶ |
302 |
out.push(k.object); |
| 642da263 Andy McCall ↶ |
303 |
out.push(k.artifact.tree); |
| 304 |
out.extend(tree(repo, &k.artifact)?.entries.iter().map(|e| e.blob)); |
| 305 |
} |
| 306 |
Ok(out) |
| 307 |
} |
| 308 |
|
| 309 |
#[cfg(test)] |
| 310 |
mod tests { |
| 311 |
use super::*; |
| 6c3dbe20 Andy McCall ↶ |
312 |
use oxy_core::cbor::{self, Value}; |
| 642da263 Andy McCall ↶ |
313 |
|
| 314 |
fn repo(dir: &Path) -> Repo { |
| 315 |
Repo::init(&dir.join("repo")).unwrap() |
| 316 |
} |
| 317 |
|
| 318 |
#[test] |
| 319 |
fn a_directory_packs_into_an_artifact_and_back() { |
| 320 |
let t = tempfile::tempdir().unwrap(); |
| 321 |
let r = repo(t.path()); |
| 322 |
let src = t.path().join("out"); |
| 323 |
std::fs::create_dir_all(src.join("bin")).unwrap(); |
| 324 |
std::fs::write(src.join("README"), b"hello\n").unwrap(); |
| 325 |
std::fs::write(src.join("bin/tool"), vec![7u8; 300_000]).unwrap(); |
| 326 |
#[cfg(unix)] |
| 327 |
{ |
| 328 |
use std::os::unix::fs::PermissionsExt; |
| 329 |
std::fs::set_permissions(src.join("bin/tool"), std::fs::Permissions::from_mode(0o755)) |
| 330 |
.unwrap(); |
| 331 |
std::os::unix::fs::symlink("bin/tool", src.join("tool")).unwrap(); |
| 332 |
} |
| 333 |
let mut spec = Spec::new(Kind::Archive); |
| 334 |
spec.arch = Some("x86_64".into()); |
| 335 |
spec.labels.insert("version".into(), "0.1.0".into()); |
| 336 |
let k = pack(&r, &src, &spec).unwrap(); |
| 6c3dbe20 Andy McCall ↶ |
337 |
assert_eq!(k.name, k.artifact.name().to_hex()); |
| 642da263 Andy McCall ↶ |
338 |
assert_eq!( |
| 339 |
k.artifact.total_size, |
| 340 |
300_006 + if cfg!(unix) { 8 } else { 0 } |
| 341 |
); |
| 342 |
assert_eq!(list(&r).unwrap(), vec![k.name.clone()]); |
| 343 |
|
| 344 |
let again = open(&r, &k.name[..10]).unwrap(); |
| 345 |
assert_eq!(again.artifact, k.artifact); |
| 346 |
let paths: Vec<(String, u32)> = tree(&r, &again.artifact) |
| 347 |
.unwrap() |
| 348 |
.entries |
| 349 |
.into_iter() |
| 350 |
.map(|e| (e.path, e.mode)) |
| 351 |
.collect(); |
| 352 |
assert_eq!(paths[0], ("README".into(), MODE_FILE)); |
| 353 |
assert_eq!( |
| 354 |
paths[1], |
| 355 |
( |
| 356 |
"bin/tool".into(), |
| 357 |
if cfg!(unix) { MODE_EXEC } else { MODE_FILE } |
| 358 |
) |
| 359 |
); |
| 360 |
assert_eq!(verify(&r, &again).unwrap(), paths.len()); |
| 361 |
|
| 362 |
let dest = t.path().join("back"); |
| 363 |
unpack(&r, &again, &dest).unwrap(); |
| 364 |
assert_eq!( |
| 365 |
std::fs::read(dest.join("bin/tool")).unwrap(), |
| 366 |
vec![7u8; 300_000] |
| 367 |
); |
| 368 |
#[cfg(unix)] |
| 369 |
assert_eq!( |
| 370 |
std::fs::read_link(dest.join("tool")).unwrap(), |
| 371 |
Path::new("bin/tool") |
| 372 |
); |
| 373 |
assert!(unpack(&r, &again, &dest) |
| 374 |
.unwrap_err() |
| 375 |
.to_string() |
| 376 |
.contains("not empty")); |
| 377 |
|
| 378 |
|
| 379 |
assert_eq!(pack(&r, &src, &spec).unwrap().name, k.name); |
| 380 |
let keep = roots(&r).unwrap(); |
| 6c3dbe20 Andy McCall ↶ |
381 |
assert!(keep.contains(&k.object) && keep.contains(&k.artifact.tree)); |
| 642da263 Andy McCall ↶ |
382 |
r.gc_keeping(true, &keep, &mut |_| {}).unwrap(); |
| 6c3dbe20 Andy McCall ↶ |
383 |
|
| 384 |
let report = r.fsck().unwrap(); |
| 385 |
let errors: Vec<_> = report |
| 386 |
.problems |
| 387 |
.iter() |
| 388 |
.filter(|p| p.severity == oxy_core::repo::Severity::Error) |
| 389 |
.collect(); |
| 390 |
assert!(errors.is_empty(), "{errors:?}"); |
| 642da263 Andy McCall ↶ |
391 |
assert_eq!( |
| 392 |
verify(&r, &open(&r, &k.name).unwrap()).unwrap(), |
| 393 |
paths.len() |
| 394 |
); |
| 395 |
} |
| 396 |
|
| 397 |
#[test] |
| 398 |
fn encodings_are_canonical_and_checked() { |
| 399 |
let a = Artifact { |
| 400 |
kind: Kind::RunnerImage, |
| 401 |
arch: Some("riscv64".into()), |
| 402 |
os: Some("linux".into()), |
| 403 |
tree: ObjectRef([1; 32]), |
| 404 |
entrypoint: Some(vec!["/sbin/init".into()]), |
| 405 |
env: [("PATH".to_string(), "/bin".to_string())].into(), |
| 406 |
labels: BTreeMap::new(), |
| 407 |
total_size: 42, |
| 408 |
receipt: Some(ObjectRef([2; 32])), |
| 409 |
}; |
| 410 |
let bytes = a.encode(); |
| 411 |
assert_eq!(Artifact::decode(&bytes).unwrap(), a); |
| 6c3dbe20 Andy McCall ↶ |
412 |
assert_eq!( |
| 413 |
a.name(), |
| 414 |
oxy_core::hash::hash_object(ObjectType::Artifact, &bytes) |
| 415 |
); |
| 642da263 Andy McCall ↶ |
416 |
assert!(Kind::parse("tarball") |
| 417 |
.unwrap_err() |
| 418 |
.to_string() |
| 419 |
.contains("runner-image")); |
| 420 |
|
| 421 |
let wrong = |v: Value| Artifact::decode(&cbor::encode(&v)).unwrap_err().to_string(); |
| 422 |
assert!(wrong(Value::map([(0, Value::Uint(2))])).contains("version")); |
| 423 |
assert!(wrong(Value::map([(0, Value::Uint(1)), (10, Value::Null)])).contains("key 10")); |
| 424 |
|
| 425 |
let t = DirTree { |
| 426 |
entries: vec![Entry { |
| 427 |
path: "a".into(), |
| 428 |
mode: MODE_FILE, |
| 429 |
blob: ObjectRef([3; 32]), |
| 430 |
size: 1, |
| 431 |
}], |
| 432 |
}; |
| 433 |
assert_eq!(DirTree::decode(&t.encode()).unwrap(), t); |
| 434 |
let entry = |p: &str, mode: u32| { |
| 435 |
Value::Array(vec![ |
| 436 |
Value::Text(p.into()), |
| 437 |
Value::Uint(mode as u64), |
| 438 |
Value::Bytes(vec![0; 32]), |
| 439 |
Value::Uint(0), |
| 440 |
]) |
| 441 |
}; |
| 442 |
let tree = |items: Vec<Value>| { |
| 443 |
DirTree::decode(&cbor::encode(&Value::map([ |
| 444 |
(0, Value::Uint(1)), |
| 445 |
(1, Value::Array(items)), |
| 446 |
]))) |
| 447 |
.unwrap_err() |
| 448 |
.to_string() |
| 449 |
}; |
| 450 |
assert!(tree(vec![entry("../x", MODE_FILE)]).contains("clean")); |
| 451 |
assert!(tree(vec![entry("b", MODE_FILE), entry("a", MODE_FILE)]).contains("order")); |
| 452 |
assert!(tree(vec![entry("a", 0o040000)]).contains("mode")); |
| 453 |
} |
| 454 |
} |