OXYGENHUB
# e3fd8257ce05 oxygenflow / crates / flow / src / workspace.rs
5.2 KiB · Rust · last changed Ephemeral scratch mounts in M0: ephemeral://tmpfs and ephemeral://disk (a sparse disk per job), checked targets and sizes, in the canonical job and the receipt; flow-init mounts them for the job's user; make deploy-runner 1 month ago source blame history raw
e0918b88 Andy McCall 1
//! A job's workspace: the files of the state being built, written to a
2
//! scratch directory and packed into an ext4 image for the microVM
3
//! (docs/m0.md §2). Returns each file's content hash for the scope hash.
4
5
use anyhow::{bail, Context, Result};
6
use flow_core::Hash;
7
use oxy_core::hash::ChangeHash;
8
use oxy_core::objects::NodeKind;
9
use oxy_core::repo::Repo;
10
use oxy_core::state::FileContent;
11
use std::path::Path;
12
13
pub type Files = Vec<(String, Hash)>;
14
15
/// Write the state `heads` of `repo` under `dir`.
16
pub fn export_state(repo: &Repo, heads: &[ChangeHash], dir: &Path) -> Result<Files> {
17
    let snapshot = repo.load_state(heads).context("load the state")?.materialize();
18
    let mut files = Vec::new();
19
    for (path, f) in &snapshot.files {
20
        if path.split('/').any(|p| p.is_empty() || p == "." || p == ".." || p == ".oxy") {
21
            continue;
22
        }
23
        let target = dir.join(path);
24
        match (&f.kind, &f.content) {
25
            (NodeKind::Dir, _) | (_, FileContent::Dir) => std::fs::create_dir_all(&target)?,
26
            (NodeKind::Subrepo, _) => {}
27
            (kind, content) => {
28
                let bytes = match content {
29
                    FileContent::Text { bytes, .. } => bytes.clone(),
30
                    FileContent::Opaque { blobs } => match blobs.first() {
31
                        Some(b) => repo.read_blob(b)?,
32
                        None => Vec::new(),
33
                    },
34
                    FileContent::Dir => unreachable!(),
35
                };
36
                if let Some(parent) = target.parent() {
37
                    std::fs::create_dir_all(parent)?;
38
                }
39
                if *kind == NodeKind::Symlink {
40
                    let to = String::from_utf8_lossy(&bytes).into_owned();
41
                    std::os::unix::fs::symlink(&to, &target).with_context(|| format!("symlink {path}"))?;
42
                } else {
43
                    std::fs::write(&target, &bytes).with_context(|| format!("write {path}"))?;
44
                    if f.mode.exec {
45
                        use std::os::unix::fs::PermissionsExt;
46
                        std::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o755))?;
47
                    }
48
                }
49
                files.push((path.clone(), *blake3::hash(&bytes).as_bytes()));
50
            }
51
        }
52
    }
53
    Ok(files)
54
}
55
56
/// Copy a working directory (without `.oxy` and what `.oxyignore` and
57
/// `.gitignore` exclude) under `dir`, for `flow run`.
58
pub fn copy_tree(src: &Path, dir: &Path) -> Result<Files> {
59
    let mut files = Vec::new();
60
    let walk = ignore::WalkBuilder::new(src)
61
        .hidden(false)
62
        .git_ignore(true)
63
        .require_git(false)
64
        .add_custom_ignore_filename(".oxyignore")
65
        .filter_entry(|e| e.file_name() != ".oxy" && e.file_name() != ".git")
66
        .build();
67
    for entry in walk {
68
        let entry = entry?;
69
        let rel = entry.path().strip_prefix(src)?;
70
        if rel.as_os_str().is_empty() {
71
            continue;
72
        }
73
        let rel_s = rel.to_string_lossy().replace('\\', "/");
74
        let target = dir.join(rel);
75
        let ft = entry.file_type().context("file type")?;
76
        if ft.is_dir() {
77
            std::fs::create_dir_all(&target)?;
78
        } else if ft.is_symlink() {
79
            let to = std::fs::read_link(entry.path())?;
80
            std::os::unix::fs::symlink(&to, &target)?;
81
            files.push((rel_s, *blake3::hash(to.to_string_lossy().as_bytes()).as_bytes()));
82
        } else if ft.is_file() {
83
            std::fs::copy(entry.path(), &target)?;
84
            files.push((rel_s, *blake3::hash(&std::fs::read(entry.path())?).as_bytes()));
85
        }
86
    }
87
    Ok(files)
88
}
89
90
/// Pack `dir` into an ext4 image whose root belongs to this user, so the
91
/// job (which runs as the owner of `/src`) can write to it.
92
pub fn image(dir: &Path, out: &Path) -> Result<()> {
93
    let used: u64 = ignore::WalkBuilder::new(dir)
94
        .hidden(false)
95
        .ignore(false)
96
        .git_ignore(false)
97
        .build()
98
        .flatten()
99
        .filter_map(|e| e.metadata().ok())
100
        .map(|m| m.len().max(4096))
101
        .sum();
102
    let mib = used * 12 / 10 / (1 << 20) + 512;
103
    let (uid, gid) = unsafe { (libc::getuid(), libc::getgid()) };
104
    let status = std::process::Command::new("mkfs.ext4")
105
        .args(["-q", "-F", "-L", "workspace", "-E"])
106
        .arg(format!("root_owner={uid}:{gid}"))
107
        .arg("-d")
108
        .arg(dir)
109
        .arg(out)
110
        .arg(format!("{mib}M"))
111
        .status()
112
        .context("run mkfs.ext4 (e2fsprogs 1.43 or later)")?;
113
    if !status.success() {
114
        bail!("mkfs.ext4 failed for the workspace image");
115
    }
116
    Ok(())
117
}
fc10d9e8 Andy McCall ↶ 118
119
/// An empty ext4 filesystem of `mib` MiB at `out`, owned by this user like
120
/// the workspace. mkfs.ext4 creates the file sparse, so only what the job
121
/// writes takes space on the runner.
122
pub fn empty_ext4(out: &Path, mib: u32, label: &str) -> Result<()> {
123
    let (uid, gid) = unsafe { (libc::getuid(), libc::getgid()) };
124
    let status = std::process::Command::new("mkfs.ext4")
125
        .args(["-q", "-F", "-L", label, "-E"])
126
        .arg(format!("root_owner={uid}:{gid}"))
127
        .arg(out)
128
        .arg(format!("{mib}M"))
129
        .status()
130
        .context("run mkfs.ext4 (e2fsprogs 1.43 or later)")?;
131
    if !status.success() {
132
        bail!("mkfs.ext4 failed for a scratch disk");
133
    }
134
    Ok(())
135
}