//! A job's workspace: the files of the state being built, written to a //! scratch directory and packed into an ext4 image for the microVM //! (docs/m0.md ยง2). Returns each file's content hash for the scope hash. use anyhow::{bail, Context, Result}; use flow_core::Hash; use oxy_core::hash::ChangeHash; use oxy_core::objects::NodeKind; use oxy_core::repo::Repo; use oxy_core::state::FileContent; use std::path::Path; pub type Files = Vec<(String, Hash)>; /// Write the state `heads` of `repo` under `dir`. pub fn export_state(repo: &Repo, heads: &[ChangeHash], dir: &Path) -> Result { let snapshot = repo.load_state(heads).context("load the state")?.materialize(); let mut files = Vec::new(); for (path, f) in &snapshot.files { if path.split('/').any(|p| p.is_empty() || p == "." || p == ".." || p == ".oxy") { continue; } let target = dir.join(path); match (&f.kind, &f.content) { (NodeKind::Dir, _) | (_, FileContent::Dir) => std::fs::create_dir_all(&target)?, (NodeKind::Subrepo, _) => {} (kind, content) => { let bytes = match content { FileContent::Text { bytes, .. } => bytes.clone(), FileContent::Opaque { blobs } => match blobs.first() { Some(b) => repo.read_blob(b)?, None => Vec::new(), }, FileContent::Dir => unreachable!(), }; if let Some(parent) = target.parent() { std::fs::create_dir_all(parent)?; } if *kind == NodeKind::Symlink { let to = String::from_utf8_lossy(&bytes).into_owned(); std::os::unix::fs::symlink(&to, &target).with_context(|| format!("symlink {path}"))?; } else { std::fs::write(&target, &bytes).with_context(|| format!("write {path}"))?; if f.mode.exec { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o755))?; } } files.push((path.clone(), *blake3::hash(&bytes).as_bytes())); } } } Ok(files) } /// Copy a working directory (without `.oxy` and what `.oxyignore` and /// `.gitignore` exclude) under `dir`, for `flow run`. pub fn copy_tree(src: &Path, dir: &Path) -> Result { let mut files = Vec::new(); let walk = ignore::WalkBuilder::new(src) .hidden(false) .git_ignore(true) .require_git(false) .add_custom_ignore_filename(".oxyignore") .filter_entry(|e| e.file_name() != ".oxy" && e.file_name() != ".git") .build(); for entry in walk { let entry = entry?; let rel = entry.path().strip_prefix(src)?; if rel.as_os_str().is_empty() { continue; } let rel_s = rel.to_string_lossy().replace('\\', "/"); let target = dir.join(rel); let ft = entry.file_type().context("file type")?; if ft.is_dir() { std::fs::create_dir_all(&target)?; } else if ft.is_symlink() { let to = std::fs::read_link(entry.path())?; std::os::unix::fs::symlink(&to, &target)?; files.push((rel_s, *blake3::hash(to.to_string_lossy().as_bytes()).as_bytes())); } else if ft.is_file() { std::fs::copy(entry.path(), &target)?; files.push((rel_s, *blake3::hash(&std::fs::read(entry.path())?).as_bytes())); } } Ok(files) } /// Pack `dir` into an ext4 image whose root belongs to this user, so the /// job (which runs as the owner of `/src`) can write to it. pub fn image(dir: &Path, out: &Path) -> Result<()> { let used: u64 = ignore::WalkBuilder::new(dir) .hidden(false) .ignore(false) .git_ignore(false) .build() .flatten() .filter_map(|e| e.metadata().ok()) .map(|m| m.len().max(4096)) .sum(); let mib = used * 12 / 10 / (1 << 20) + 512; let (uid, gid) = unsafe { (libc::getuid(), libc::getgid()) }; let status = std::process::Command::new("mkfs.ext4") .args(["-q", "-F", "-L", "workspace", "-E"]) .arg(format!("root_owner={uid}:{gid}")) .arg("-d") .arg(dir) .arg(out) .arg(format!("{mib}M")) .status() .context("run mkfs.ext4 (e2fsprogs 1.43 or later)")?; if !status.success() { bail!("mkfs.ext4 failed for the workspace image"); } Ok(()) }