OXYGENHUB
# ed4aef8b70ca oxygenflow / crates / flow-core / src / pipeline.rs
50.2 KiB · Rust · last changed A build cache: disk mounts that carry a build forward 23 days ago source blame history raw
e0918b88 Andy McCall 1
//! `.flow/pipeline.kdl`: the M0 subset of spec §3 (docs/m0.md §1), parsed
2
//! from KDL 2.0 and validated before anything runs. Every problem is
3
//! reported with its line, and all of them at once.
4
5
use serde::{Deserialize, Serialize};
6
use std::collections::{BTreeMap, BTreeSet};
7
use std::fmt;
8
9
/// Where a pipeline lives in the state being built.
10
pub const PIPELINE_PATH: &str = ".flow/pipeline.kdl";
11
12
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
13
pub struct Pipeline {
14
    pub schema_version: u32,
15
    /// By name.
16
    pub sandboxes: BTreeMap<String, Sandbox>,
17
    /// By name.
18
    pub jobs: BTreeMap<String, Job>,
bf7bb1b8 Andy McCall ↶ 19
    /// When the pipeline's jobs run, unless a job says (oxygenflow#24);
20
    /// absent: on every push and from "Run pipeline".
21
    #[serde(default, skip_serializing_if = "Option::is_none")]
22
    pub on: Option<On>,
23
}
24
25
/// What starts a job (`on { push "main"; tag "v*"; manual; schedule "0 3 * * *" }`).
26
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
27
pub struct On {
28
    /// Bookmark globs a push runs it for; `Some(vec![])`: any bookmark.
29
    #[serde(default, skip_serializing_if = "Option::is_none")]
30
    pub push: Option<Vec<String>>,
31
    /// Tag globs a pushed tag runs it for; `Some(vec![])`: any tag.
32
    #[serde(default, skip_serializing_if = "Option::is_none")]
33
    pub tag: Option<Vec<String>>,
34
    /// From "Run pipeline" (and `oh run start`).
35
    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
36
    pub manual: bool,
37
    /// Five-field cron lines, on the hub's clock, for the default bookmark.
38
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
39
    pub schedule: Vec<String>,
e0918b88 Andy McCall 40
}
41
bf7bb1b8 Andy McCall ↶ 42
/// Why a run started, for [`On::matches`].
43
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
44
pub enum Event<'a> {
45
    /// A push moved this bookmark.
46
    Push(&'a str),
47
    /// This tag was pushed.
48
    Tag(&'a str),
49
    /// "Run pipeline", `oh run start`.
50
    Manual,
51
    /// A `schedule` line came due.
52
    Schedule,
53
}
54
55
impl On {
56
    /// What a job without any `on` runs for: every push, and by hand.
57
    pub fn default_on() -> On {
58
        On { push: Some(vec![]), tag: None, manual: true, schedule: vec![] }
59
    }
60
61
    pub fn matches(&self, event: Event<'_>) -> bool {
62
        let any = |globs: &Option<Vec<String>>, name: &str| match globs {
63
            None => false,
64
            Some(g) if g.is_empty() => true,
65
            Some(g) => g.iter().any(|p| globset::Glob::new(p).is_ok_and(|p| p.compile_matcher().is_match(name))),
66
        };
67
        match event {
68
            Event::Push(b) => any(&self.push, b),
69
            Event::Tag(t) => any(&self.tag, t),
70
            Event::Manual => self.manual,
71
            Event::Schedule => !self.schedule.is_empty(),
72
        }
73
    }
74
}
75
76
impl Pipeline {
77
    /// When `job` runs: its own `on`, else the pipeline's, else the default.
78
    pub fn on_of(&self, job: &Job) -> On {
79
        job.on.clone().or_else(|| self.on.clone()).unwrap_or_else(On::default_on)
80
    }
81
}
82
e0918b88 Andy McCall 83
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
84
#[serde(rename_all = "lowercase")]
85
pub enum Runtime {
86
    /// Tier 0 (spec §4.1); refused in M0.
87
    Wasm,
88
    /// Tier 1 (spec §4.2): namespaces, Landlock, seccomp, cgroups.
89
    Syscall,
90
    /// Tier 2 (spec §4.3), the default.
91
    Microvm,
92
}
93
94
impl Runtime {
95
    pub fn as_str(self) -> &'static str {
96
        match self {
97
            Runtime::Wasm => "wasm",
98
            Runtime::Syscall => "syscall",
99
            Runtime::Microvm => "microvm",
100
        }
101
    }
102
}
103
104
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
105
#[serde(rename_all = "lowercase")]
106
pub enum Network {
107
    Isolated,
108
    Loopback,
109
    Egress,
110
}
111
112
impl Network {
113
    pub fn as_str(self) -> &'static str {
114
        match self {
115
            Network::Isolated => "isolated",
116
            Network::Loopback => "loopback",
117
            Network::Egress => "egress",
118
        }
119
    }
120
}
121
122
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
123
pub struct Sandbox {
124
    pub name: String,
125
    pub runtime: Runtime,
126
    /// BLAKE3 of the rootfs image, hex.
127
    pub rootfs: String,
128
    pub memory_mib: u32,
129
    pub cpus: u32,
130
    pub network: Network,
a72abeee Andy McCall ↶ 131
    /// Environment every job in this sandbox starts from; a job's own `env`
132
    /// overrides it.
133
    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
134
    pub env: BTreeMap<String, String>,
e576e086 Andy McCall ↶ 135
    /// Labels a runner must have to run jobs here (`runs-on`), sorted.
136
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
137
    pub runs_on: Vec<String>,
02b127d2 Andy McCall ↶ 138
    /// How long each job here may run, in seconds (`timeout "2h"`), when
139
    /// the job does not say; the hub caps it at the site's maximum.
140
    #[serde(default, skip_serializing_if = "Option::is_none")]
141
    pub timeout_s: Option<u64>,
1abadd4a Andy McCall ↶ 142
    /// The architecture the jobs need (`arch "riscv64"`, spec §11.3);
143
    /// without it, the rootfs image's.
144
    #[serde(default, skip_serializing_if = "Option::is_none")]
145
    pub arch: Option<String>,
146
}
147
148
/// An architecture name as runners report it: `x86_64`, `aarch64`, `riscv64`.
149
pub fn arch_ok(a: &str) -> bool {
150
    (2..=16).contains(&a.len()) && a.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
e0918b88 Andy McCall 151
}
152
153
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
154
pub struct Job {
155
    pub name: String,
156
    pub sandbox: String,
157
    /// Glob patterns over repository paths (spec §6.2).
158
    pub scope: Vec<String>,
159
    /// Sorted.
160
    pub depends_on: Vec<String>,
161
    pub env: BTreeMap<String, String>,
162
    /// Executed as is: no shell, no expansion (spec §2.2 I2).
163
    pub exec: Vec<String>,
fc10d9e8 Andy McCall ↶ 164
    /// Ephemeral scratch mounts, sorted by target (spec §3.1 `mount`).
165
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
166
    pub mounts: Vec<Mount>,
a67b97e9 Andy McCall ↶ 167
    /// Files the job produces, sorted by name (spec §5.2 `output`).
168
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
169
    pub outputs: Vec<Output>,
e2c12561 Andy McCall ↶ 170
    /// Secrets the job mounts, sorted by name (spec §9 `secret`): the hub
171
    /// supplies each by name, the runner mounts it as a file.
172
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
173
    pub secrets: Vec<Secret>,
720eb819 Andy McCall ↶ 174
    /// Identities the job asks the hub for (oxygenhub#103): an OIDC token
175
    /// for an audience, mounted as a file like a secret.
176
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
177
    pub identities: Vec<Identity>,
e576e086 Andy McCall ↶ 178
    /// Labels a runner must have to take this job (`runs-on`), sorted;
179
    /// with its sandbox's, what the hub matches against a runner's.
180
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
181
    pub runs_on: Vec<String>,
02b127d2 Andy McCall ↶ 182
    /// How long the job may run, in seconds (`timeout "90m"`); its
183
    /// sandbox's when absent, else the site's default (oxygenflow#23).
184
    #[serde(default, skip_serializing_if = "Option::is_none")]
185
    pub timeout_s: Option<u64>,
bf7bb1b8 Andy McCall ↶ 186
    /// When it runs, over the pipeline's `on` (oxygenflow#24).
187
    #[serde(default, skip_serializing_if = "Option::is_none")]
188
    pub on: Option<On>,
fc10d9e8 Andy McCall ↶ 189
}
190
c4285535 Andy McCall ↶ 191
impl Job {
a72abeee Andy McCall ↶ 192
    /// The job's environment on top of its sandbox's: what it runs with.
193
    pub fn in_sandbox(&self, sandbox: &Sandbox) -> Job {
194
        let mut job = self.clone();
195
        for (k, v) in &sandbox.env {
196
            job.env.entry(k.clone()).or_insert_with(|| v.clone());
197
        }
e576e086 Andy McCall ↶ 198
        job.runs_on.extend(sandbox.runs_on.iter().cloned());
199
        job.runs_on.sort();
200
        job.runs_on.dedup();
02b127d2 Andy McCall ↶ 201
        job.timeout_s = job.timeout_s.or(sandbox.timeout_s);
a72abeee Andy McCall ↶ 202
        job
203
    }
204
c4285535 Andy McCall ↶ 205
    /// The job with variables that tell it what it builds (`FLOW_STATE`,
206
    /// `FLOW_BOOKMARK` …) added to its environment; a variable the pipeline
207
    /// sets itself keeps the pipeline's value.
208
    pub fn with_flow_env(&self, vars: &[(String, String)]) -> Job {
209
        let mut job = self.clone();
210
        for (k, v) in vars {
211
            job.env.entry(k.clone()).or_insert_with(|| v.clone());
212
        }
213
        job
214
    }
215
}
216
fc10d9e8 Andy McCall ↶ 217
/// An ephemeral scratch mount: empty at the start of every job and gone
d60aac6c Andy McCall ↶ 218
/// after it, so it cannot carry anything between jobs (spec §2.2 I4) --
219
/// unless it names a `cache` (spec §5.4, #28): then a disk kept from an
220
/// earlier passing job may start it, and the receipt says which.
fc10d9e8 Andy McCall ↶ 221
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
222
pub struct Mount {
223
    pub source: MountSource,
224
    /// An absolute path under `/src` or `/tmp`.
225
    pub target: String,
226
    pub size_mib: u32,
d60aac6c Andy McCall ↶ 227
    /// `cache "name"`: keep this disk after a passing job and start later
228
    /// jobs from it (disk mounts only).
229
    #[serde(default, skip_serializing_if = "Option::is_none")]
230
    pub cache: Option<String>,
231
    /// `key "Cargo.lock" ...`: files whose content picks the cache entry.
232
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
233
    pub key: Vec<String>,
fc10d9e8 Andy McCall ↶ 234
}
235
236
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
237
#[serde(rename_all = "lowercase")]
238
pub enum MountSource {
239
    /// `ephemeral://tmpfs`: memory in the sandbox, counted against its memory-limit.
240
    Tmpfs,
241
    /// `ephemeral://disk`: a fresh sparse disk on the runner, for large build trees.
242
    Disk,
243
}
244
245
impl MountSource {
246
    pub fn as_str(self) -> &'static str {
247
        match self {
248
            MountSource::Tmpfs => "ephemeral://tmpfs",
249
            MountSource::Disk => "ephemeral://disk",
250
        }
251
    }
a67b97e9 Andy McCall ↶ 252
}
253
254
/// A file a job produces (spec §5.2): read from the job's disks when it
255
/// ends, kept in the repository as a blob, and named by its receipt.
256
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
257
pub struct Output {
258
    /// The artifact's name, unique in the job.
259
    pub name: String,
260
    /// A clean absolute path under `/src`, not on a tmpfs mount.
261
    pub path: String,
9a6fac2b Andy McCall ↶ 262
    /// `when "always"`: kept from a failing job too, such as a test report
263
    /// (oxygenflow#37). From a failing job it is never in the receipt.
264
    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
265
    pub always: bool,
6d53fb7d Andy McCall ↶ 266
    /// An artifact is one file; a report is a directory with a
267
    /// `report.toml`, kept file by file with a manifest (oxygenhub#127).
268
    #[serde(default, skip_serializing_if = "OutputKind::is_artifact")]
269
    pub kind: OutputKind,
e2c12561 Andy McCall ↶ 270
}
271
6d53fb7d Andy McCall ↶ 272
/// What an [`Output`] is.
273
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
274
#[serde(rename_all = "lowercase")]
275
pub enum OutputKind {
276
    #[default]
277
    Artifact,
278
    Report,
279
}
280
281
impl OutputKind {
282
    pub fn is_artifact(&self) -> bool {
283
        *self == OutputKind::Artifact
284
    }
285
286
    pub fn word(&self) -> &'static str {
287
        match self {
288
            OutputKind::Artifact => "artifact",
289
            OutputKind::Report => "report",
290
        }
291
    }
292
}
293
e2c12561 Andy McCall ↶ 294
/// A secret the job reads as a file (spec §9): never in the environment.
295
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
296
pub struct Secret {
297
    /// Which secret, as the hub names it; unique in the job.
298
    pub name: String,
299
    /// A clean absolute path under `/run/secrets/`, unique in the job;
300
    /// the file is 0400, the job's user's, on an in-memory mount.
301
    pub mount_path: String,
a67b97e9 Andy McCall ↶ 302
}
303
720eb819 Andy McCall ↶ 304
/// An OIDC token the hub mints for the job (oxygenhub#103), naming the
305
/// repository, bookmark, run and job, for `audience`; read as a file.
306
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
307
pub struct Identity {
308
    /// Who the token is for (`sts.amazonaws.com`, a Vault role's audience, …).
309
    pub audience: String,
310
    /// A clean absolute path under `/run/secrets/`, unique among the
311
    /// job's secrets and identities.
312
    pub mount_path: String,
313
}
314
e2c12561 Andy McCall ↶ 315
/// A secret's mount path: under `/run/secrets/`, no `.` or `..`.
316
pub fn secret_path_ok(p: &str) -> bool {
317
    p.strip_prefix("/run/secrets/").is_some_and(|rest| !rest.is_empty() && rest.split('/').all(|c| !c.is_empty() && c != "." && c != ".."))
318
}
319
a67b97e9 Andy McCall ↶ 320
/// Artifacts a job may declare.
321
pub const MAX_OUTPUTS: usize = 16;
322
323
/// Whether `p` is a clean absolute path strictly under `/src`, with nothing
324
/// a debugfs command line would have to quote.
325
fn output_path_ok(p: &str) -> bool {
326
    p.strip_prefix("/src/").is_some_and(|rest| {
327
        !rest.is_empty()
328
            && rest.split('/').all(|c| !c.is_empty() && c != "." && c != "..")
329
            && p.bytes().all(|b| b.is_ascii_graphic() && b != b'"' && b != b'\\')
330
    })
fc10d9e8 Andy McCall ↶ 331
}
332
333
/// Mounts a job may declare.
334
pub const MAX_MOUNTS: usize = 4;
e2c12561 Andy McCall ↶ 335
/// The most secrets a job may mount.
336
pub const MAX_SECRETS: usize = 8;
720eb819 Andy McCall ↶ 337
/// The most identities a job may ask for.
338
pub const MAX_IDENTITIES: usize = 4;
fc10d9e8 Andy McCall ↶ 339
/// Sizes a mount may have: 16 MiB to 64 GiB.
340
pub const MIN_MOUNT_MIB: u64 = 16;
341
pub const MAX_MOUNT_MIB: u64 = 64 * 1024;
342
343
/// Whether `t` is a clean absolute path strictly under `/src` or `/tmp`.
344
fn mount_target_ok(t: &str) -> bool {
345
    let under = t.strip_prefix("/src/").or_else(|| t.strip_prefix("/tmp/"));
346
    under.is_some_and(|rest| !rest.is_empty() && rest.split('/').all(|c| !c.is_empty() && c != "." && c != ".."))
e0918b88 Andy McCall 347
}
348
349
/// One problem in a pipeline, at a 1-based line.
350
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
351
pub struct Problem {
352
    pub line: usize,
353
    pub message: String,
354
}
355
356
impl fmt::Display for Problem {
357
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
358
        write!(f, "line {}: {}", self.line, self.message)
359
    }
360
}
361
362
/// Every problem found, in line order.
363
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)]
364
pub struct Problems(pub Vec<Problem>);
365
366
impl fmt::Display for Problems {
367
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
368
        let lines: Vec<String> = self.0.iter().map(|p| p.to_string()).collect();
369
        f.write_str(&lines.join("\n"))
370
    }
371
}
372
373
const MIN_MEMORY_MIB: u64 = 64;
374
const MAX_MEMORY_MIB: u64 = 16 * 1024;
375
const MAX_CPUS: i128 = 16;
376
377
/// Nodes of the full spec that M0 recognises and refuses.
378
const LATER_TOP: &[&str] = &["publish", "secret"];
e2c12561 Andy McCall ↶ 379
const LATER_JOB: &[&str] = &["attest"];
e0918b88 Andy McCall 380
381
struct Ctx<'a> {
382
    src: &'a str,
383
    problems: Vec<Problem>,
384
}
385
386
impl Ctx<'_> {
387
    fn line(&self, span: SourceSpan) -> usize {
388
        let at = span.offset().min(self.src.len());
389
        self.src[..at].bytes().filter(|b| *b == b'\n').count() + 1
390
    }
391
392
    fn err(&mut self, span: SourceSpan, message: impl Into<String>) {
393
        let line = self.line(span);
394
        self.problems.push(Problem { line, message: message.into() });
395
    }
396
397
    /// The node's arguments as strings; properties and other types are errors.
398
    fn strings(&mut self, node: &KdlNode) -> Vec<String> {
399
        let mut out = Vec::new();
400
        for e in node.entries() {
401
            if let Some(k) = e.name() {
402
                self.err(e.span(), format!("`{}` takes no property `{}`", node.name().value(), k.value()));
403
                continue;
404
            }
405
            match e.value() {
406
                KdlValue::String(s) => out.push(s.clone()),
407
                other => self.err(e.span(), format!("`{}` takes strings, not {}", node.name().value(), kind_of(other))),
408
            }
409
        }
410
        out
411
    }
412
413
    /// Exactly one string argument.
414
    fn one_string(&mut self, node: &KdlNode) -> Option<String> {
415
        let v = self.strings(node);
416
        if v.len() != 1 {
417
            self.err(node.span(), format!("`{}` takes one string", node.name().value()));
418
            return None;
419
        }
420
        v.into_iter().next()
421
    }
422
423
    fn one_integer(&mut self, node: &KdlNode) -> Option<i128> {
424
        match node.entries() {
425
            [e] if e.name().is_none() => match e.value() {
426
                KdlValue::Integer(n) => Some(*n),
427
                other => {
428
                    self.err(e.span(), format!("`{}` takes an integer, not {}", node.name().value(), kind_of(other)));
429
                    None
430
                }
431
            },
432
            _ => {
433
                self.err(node.span(), format!("`{}` takes one integer", node.name().value()));
434
                None
435
            }
436
        }
437
    }
438
439
    fn no_children(&mut self, node: &KdlNode) {
440
        if node.children().is_some() {
441
            self.err(node.span(), format!("`{}` takes no block", node.name().value()));
442
        }
443
    }
444
445
    fn name_ok(&mut self, span: SourceSpan, what: &str, name: &str) -> bool {
446
        let ok = !name.is_empty() && name.len() <= 64 && name.bytes().all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b));
447
        if !ok {
448
            self.err(span, format!("{what} name {name:?} must be 1 to 64 of A-Z a-z 0-9 . _ -"));
449
        }
450
        ok
451
    }
452
}
453
454
use kdl::{KdlDocument, KdlNode, KdlValue};
455
use miette::SourceSpan;
456
457
fn kind_of(v: &KdlValue) -> &'static str {
458
    match v {
459
        KdlValue::String(_) => "a string",
460
        KdlValue::Integer(_) => "an integer",
461
        KdlValue::Float(_) => "a number",
462
        KdlValue::Bool(_) => "a boolean",
463
        KdlValue::Null => "null",
464
    }
465
}
466
467
/// `"512MiB"`, `"4GiB"`, `"65536KiB"` to MiB.
468
fn parse_memory(s: &str) -> Option<u64> {
469
    let (num, unit) = s.split_at(s.find(|c: char| !c.is_ascii_digit())?);
470
    let n: u64 = num.parse().ok()?;
471
    let mib = match unit {
472
        "KiB" => n.is_multiple_of(1024).then_some(n / 1024)?,
473
        "MiB" => n,
474
        "GiB" => n.checked_mul(1024)?,
475
        _ => return None,
476
    };
477
    Some(mib)
478
}
479
b647ab72 Andy McCall ↶ 480
/// The line of the first escape KDL 2.0 does not have but shells and C
481
/// do (`\0NN`, `\e`, `\xNN`), as someone colouring output writes them.
482
fn foreign_escape_line(src: &str) -> Option<usize> {
483
    src.lines()
484
        .position(|l| {
485
            let b = l.as_bytes();
486
            b.windows(2).enumerate().any(|(i, w)| {
487
                w[0] == b'\\'
488
                    && (i == 0 || b[i - 1] != b'\\')
489
                    && (w[1] == b'e' || w[1] == b'x' || (w[1] == b'0' && b.get(i + 2).is_some_and(u8::is_ascii_digit)))
490
            })
491
        })
492
        .map(|i| i + 1)
493
}
494
e0918b88 Andy McCall 495
impl Pipeline {
496
    /// Parse and validate a pipeline document.
497
    pub fn parse(src: &str) -> Result<Pipeline, Problems> {
498
        let doc = match KdlDocument::parse_v2(src) {
499
            Ok(d) => d,
500
            Err(e) => {
501
                let mut ctx = Ctx { src, problems: Vec::new() };
502
                for d in &e.diagnostics {
503
                    let msg = d.message.clone().or_else(|| d.label.clone()).unwrap_or_else(|| "not valid KDL 2.0".into());
504
                    ctx.err(d.span, msg);
505
                }
506
                if ctx.problems.is_empty() {
507
                    ctx.problems.push(Problem { line: 1, message: "not valid KDL 2.0".into() });
b647ab72 Andy McCall ↶ 508
                }
509
                if let Some(line) = foreign_escape_line(src) {
510
                    ctx.problems.push(Problem {
511
                        line,
512
                        message: "KDL strings have no \\033, \\e or \\x escapes: write ESC (for ANSI colours) as \\u{1b}".into(),
513
                    });
e0918b88 Andy McCall 514
                }
515
                return Err(Problems(ctx.problems));
516
            }
517
        };
518
        let mut ctx = Ctx { src, problems: Vec::new() };
519
        let mut schema = None;
520
        let mut sandboxes = BTreeMap::new();
521
        let mut jobs: BTreeMap<String, (Job, SourceSpan)> = BTreeMap::new();
bf7bb1b8 Andy McCall ↶ 522
        let mut on = None;
e0918b88 Andy McCall 523
        for node in doc.nodes() {
524
            match node.name().value() {
bf7bb1b8 Andy McCall ↶ 525
                "on" => {
526
                    if on.is_some() {
527
                        ctx.err(node.span(), "`on` is given twice");
528
                    }
529
                    on = Some(parse_on(&mut ctx, node));
530
                }
e0918b88 Andy McCall 531
                "schema-version" => {
532
                    ctx.no_children(node);
533
                    match ctx.one_integer(node) {
534
                        Some(1) => schema = Some(1),
535
                        Some(n) => ctx.err(node.span(), format!("schema-version {n} is not supported; this runner reads 1")),
536
                        None => {}
537
                    }
538
                }
539
                "sandbox" => {
540
                    if let Some(sb) = parse_sandbox(&mut ctx, node) {
541
                        if sandboxes.contains_key(&sb.name) {
542
                            ctx.err(node.span(), format!("sandbox {:?} is declared twice", sb.name));
543
                        } else {
544
                            sandboxes.insert(sb.name.clone(), sb);
545
                        }
546
                    }
547
                }
548
                "job" => {
549
                    if let Some(job) = parse_job(&mut ctx, node) {
550
                        if jobs.contains_key(&job.name) {
551
                            ctx.err(node.span(), format!("job {:?} is declared twice", job.name));
552
                        } else {
553
                            jobs.insert(job.name.clone(), (job, node.span()));
554
                        }
555
                    }
556
                }
557
                n if LATER_TOP.contains(&n) => ctx.err(node.span(), format!("`{n}` is not in M0 yet")),
bf7bb1b8 Andy McCall ↶ 558
                n => ctx.err(node.span(), format!("unknown node `{n}`; expected schema-version, on, sandbox or job")),
e0918b88 Andy McCall 559
            }
560
        }
561
        if schema.is_none() && !ctx.problems.iter().any(|p| p.message.contains("schema-version")) {
562
            ctx.problems.push(Problem { line: 1, message: "`schema-version 1` is required".into() });
563
        }
564
        if jobs.is_empty() && ctx.problems.is_empty() {
565
            ctx.problems.push(Problem { line: 1, message: "a pipeline needs at least one job".into() });
566
        }
567
        // References and cycles.
568
        for (job, span) in jobs.values() {
569
            if !job.sandbox.is_empty() && !sandboxes.contains_key(&job.sandbox) {
570
                ctx.err(*span, format!("job {:?} uses sandbox {:?}, which is not declared", job.name, job.sandbox));
571
            }
572
            for d in &job.depends_on {
573
                if !jobs.contains_key(d) {
574
                    ctx.err(*span, format!("job {:?} depends on {:?}, which is not declared", job.name, d));
575
                } else if d == &job.name {
576
                    ctx.err(*span, format!("job {:?} depends on itself", job.name));
577
                }
578
            }
579
        }
580
        let graph: BTreeMap<&str, Vec<&str>> =
581
            jobs.values().map(|(j, _)| (j.name.as_str(), j.depends_on.iter().map(String::as_str).collect())).collect();
582
        if let Some(cycle) = find_cycle(&graph) {
583
            let span = jobs[cycle[0]].1;
584
            ctx.err(span, format!("jobs depend on each other in a cycle: {}", cycle.join(" → ")));
585
        }
586
        if !ctx.problems.is_empty() {
587
            ctx.problems.sort_by_key(|p| p.line);
588
            return Err(Problems(ctx.problems));
589
        }
bf7bb1b8 Andy McCall ↶ 590
        Ok(Pipeline { schema_version: 1, sandboxes, jobs: jobs.into_iter().map(|(k, (j, _))| (k, j)).collect(), on })
e0918b88 Andy McCall 591
    }
592
593
    /// Jobs in an order where each comes after what it depends on (ties by name).
594
    pub fn order(&self) -> Vec<&Job> {
595
        let mut done: BTreeSet<&str> = BTreeSet::new();
596
        let mut out = Vec::with_capacity(self.jobs.len());
597
        while out.len() < self.jobs.len() {
598
            let before = out.len();
599
            for j in self.jobs.values() {
600
                if !done.contains(j.name.as_str()) && j.depends_on.iter().all(|d| done.contains(d.as_str())) {
601
                    out.push(j);
602
                }
603
            }
604
            for j in &out[before..] {
605
                done.insert(j.name.as_str());
606
            }
607
            if out.len() == before {
608
                break; // validated pipelines have no cycles
609
            }
610
        }
611
        out
612
    }
613
614
    /// Each job's depth: 0 without dependencies, else one more than its deepest dependency.
615
    pub fn depths(&self) -> BTreeMap<&str, usize> {
616
        let mut depth: BTreeMap<&str, usize> = BTreeMap::new();
617
        for j in self.order() {
618
            let d = j.depends_on.iter().map(|x| depth.get(x.as_str()).copied().unwrap_or(0) + 1).max().unwrap_or(0);
619
            depth.insert(j.name.as_str(), d);
620
        }
621
        depth
622
    }
623
}
624
625
fn find_cycle<'a>(graph: &BTreeMap<&'a str, Vec<&'a str>>) -> Option<Vec<&'a str>> {
626
    #[derive(Clone, Copy, PartialEq)]
627
    enum Mark {
628
        Visiting,
629
        Done,
630
    }
631
    fn visit<'a>(
632
        n: &'a str,
633
        graph: &BTreeMap<&'a str, Vec<&'a str>>,
634
        marks: &mut BTreeMap<&'a str, Mark>,
635
        stack: &mut Vec<&'a str>,
636
    ) -> Option<Vec<&'a str>> {
637
        match marks.get(n) {
638
            Some(Mark::Done) => return None,
639
            Some(Mark::Visiting) => {
640
                let at = stack.iter().position(|s| *s == n).unwrap_or(0);
641
                let mut c = stack[at..].to_vec();
642
                c.push(n);
643
                return Some(c);
644
            }
645
            None => {}
646
        }
647
        marks.insert(n, Mark::Visiting);
648
        stack.push(n);
649
        for d in graph.get(n).into_iter().flatten() {
650
            if graph.contains_key(d) && *d != n {
651
                if let Some(c) = visit(d, graph, marks, stack) {
652
                    return Some(c);
653
                }
654
            }
655
        }
656
        stack.pop();
657
        marks.insert(n, Mark::Done);
658
        None
659
    }
660
    let mut marks = BTreeMap::new();
661
    for n in graph.keys() {
662
        let mut stack = Vec::new();
663
        if let Some(c) = visit(n, graph, &mut marks, &mut stack) {
664
            return Some(c);
665
        }
666
    }
667
    None
668
}
669
670
fn block<'a>(ctx: &mut Ctx, node: &'a KdlNode) -> &'a [KdlNode] {
671
    match node.children() {
672
        Some(d) => d.nodes(),
673
        None => {
674
            ctx.err(node.span(), format!("`{}` needs a block {{ … }}", node.name().value()));
675
            &[]
676
        }
677
    }
678
}
679
680
fn parse_sandbox(ctx: &mut Ctx, node: &KdlNode) -> Option<Sandbox> {
681
    let name = ctx.one_string(node)?;
682
    let named = ctx.name_ok(node.span(), "sandbox", &name);
683
    let (mut runtime, mut rootfs, mut memory, mut cpus, mut network) =
684
        (Some(Runtime::Microvm), None, Some(1024u32), Some(1u32), Some(Network::Isolated));
685
    let mut seen = BTreeSet::new();
a72abeee Andy McCall ↶ 686
    let mut env = BTreeMap::new();
e576e086 Andy McCall ↶ 687
    let mut runs_on = Vec::new();
02b127d2 Andy McCall ↶ 688
    let mut timeout_s = None;
1abadd4a Andy McCall ↶ 689
    let mut arch: Option<String> = None;
e0918b88 Andy McCall 690
    for c in block(ctx, node) {
691
        let key = c.name().value();
e576e086 Andy McCall ↶ 692
        if key != "runs-on" && !seen.insert(key.to_string()) {
e0918b88 Andy McCall 693
            ctx.err(c.span(), format!("`{key}` is given twice"));
a72abeee Andy McCall ↶ 694
        }
695
        if key == "env" {
696
            parse_env(ctx, c, &mut env);
697
            continue;
e0918b88 Andy McCall 698
        }
e576e086 Andy McCall ↶ 699
        if key == "runs-on" {
700
            ctx.no_children(c);
701
            parse_labels(ctx, c, &mut runs_on);
702
            continue;
703
        }
e0918b88 Andy McCall 704
        ctx.no_children(c);
705
        match key {
706
            "runtime" => {
707
                runtime = match ctx.one_string(c).as_deref() {
708
                    Some("microvm") => Some(Runtime::Microvm),
709
                    Some("syscall") | Some("gvisor") => Some(Runtime::Syscall),
710
                    Some("wasm") => {
711
                        ctx.err(c.span(), "runtime \"wasm\" is not in M0 yet; use \"microvm\" or \"syscall\"");
712
                        None
713
                    }
714
                    Some(other) => {
715
                        ctx.err(c.span(), format!("runtime {other:?}: expected \"microvm\", \"syscall\" or \"wasm\""));
716
                        None
717
                    }
718
                    None => None,
719
                }
720
            }
721
            "rootfs" => {
722
                rootfs = match ctx.one_string(c) {
723
                    Some(r) => match r.strip_prefix("cas://blob/") {
724
                        Some(h) if h.len() == 64 && h.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) => {
725
                            Some(h.to_string())
726
                        }
727
                        _ => {
728
                            ctx.err(c.span(), format!("rootfs {r:?}: expected \"cas://blob/\" and 64 lower-case hex digits"));
729
                            None
730
                        }
731
                    },
732
                    None => None,
733
                }
734
            }
1abadd4a Andy McCall ↶ 735
            "arch" => {
736
                arch = match ctx.one_string(c) {
737
                    Some(a) if arch_ok(&a) => Some(a),
738
                    Some(a) => {
739
                        ctx.err(c.span(), format!("arch {a:?} is not an architecture name such as x86_64, aarch64 or riscv64"));
740
                        None
741
                    }
742
                    None => None,
743
                }
744
            }
e0918b88 Andy McCall 745
            "memory-limit" => {
746
                memory = match ctx.one_string(c) {
747
                    Some(m) => match parse_memory(&m) {
748
                        Some(mib) if (MIN_MEMORY_MIB..=MAX_MEMORY_MIB).contains(&mib) => Some(mib as u32),
749
                        Some(_) => {
750
                            ctx.err(c.span(), format!("memory-limit {m:?} must be from 64MiB to 16GiB"));
751
                            None
752
                        }
753
                        None => {
754
                            ctx.err(c.span(), format!("memory-limit {m:?}: expected a whole number of KiB, MiB or GiB, as \"4GiB\""));
755
                            None
756
                        }
757
                    },
758
                    None => None,
759
                }
760
            }
761
            "cpu-limit" => {
762
                cpus = match ctx.one_integer(c) {
763
                    Some(n) if (1..=MAX_CPUS).contains(&n) => Some(n as u32),
764
                    Some(n) => {
765
                        ctx.err(c.span(), format!("cpu-limit {n} must be from 1 to {MAX_CPUS}"));
766
                        None
767
                    }
768
                    None => None,
769
                }
770
            }
771
            "network" => {
772
                network = match ctx.one_string(c).as_deref() {
773
                    Some("isolated") => Some(Network::Isolated),
eb5c4db6 Andy McCall ↶ 774
                    // Loopback only: the guest's `lo` comes up, and it still has
775
                    // no device that reaches anything outside it.
776
                    Some("loopback") => Some(Network::Loopback),
777
                    Some("egress") => {
778
                        ctx.err(c.span(), "network \"egress\" is not in M0; M0 has \"isolated\" and \"loopback\"");
e0918b88 Andy McCall 779
                        None
780
                    }
781
                    Some(other) => {
782
                        ctx.err(c.span(), format!("network {other:?}: expected \"isolated\", \"loopback\" or \"egress\""));
783
                        None
784
                    }
785
                    None => None,
786
                }
787
            }
02b127d2 Andy McCall ↶ 788
            "timeout" => timeout_s = parse_timeout(ctx, c),
e0918b88 Andy McCall 789
            other => ctx.err(c.span(), format!("unknown sandbox setting `{other}`")),
790
        }
791
    }
792
    if rootfs.is_none() && !seen.contains("rootfs") {
793
        ctx.err(node.span(), format!("sandbox {name:?} needs a rootfs"));
794
    }
795
    if !named {
796
        return None;
e576e086 Andy McCall ↶ 797
    }
1abadd4a Andy McCall ↶ 798
    Some(Sandbox {
799
        name,
800
        runtime: runtime?,
801
        rootfs: rootfs?,
802
        memory_mib: memory?,
803
        cpus: cpus?,
804
        network: network?,
805
        env,
806
        runs_on,
807
        timeout_s,
808
        arch,
809
    })
bf7bb1b8 Andy McCall ↶ 810
}
811
812
/// An `on` block: `push` and `tag` with globs (none: any), `manual`,
813
/// `schedule` with five-field cron lines.
814
fn parse_on(ctx: &mut Ctx, node: &KdlNode) -> On {
815
    let mut on = On::default();
816
    if !node.entries().is_empty() {
817
        ctx.err(node.span(), "`on` takes a block: on { push \"main\"; tag \"v*\"; manual }");
818
    }
819
    let globs = |ctx: &mut Ctx, c: &KdlNode| -> Vec<String> {
820
        let g = ctx.strings(c);
821
        for p in &g {
822
            if let Err(e) = globset::Glob::new(p) {
823
                ctx.err(c.span(), format!("{p:?}: {e}"));
824
            }
825
        }
826
        g
827
    };
828
    for c in block(ctx, node) {
829
        ctx.no_children(c);
830
        match c.name().value() {
831
            "push" => on.push.get_or_insert_with(Vec::new).extend(globs(ctx, c)),
832
            "tag" => on.tag.get_or_insert_with(Vec::new).extend(globs(ctx, c)),
833
            "manual" => on.manual = true,
834
            "schedule" => {
835
                for line in ctx.strings(c) {
836
                    if crate::cron::Cron::parse(&line).is_some() {
837
                        on.schedule.push(line);
838
                    } else {
839
                        ctx.err(
840
                            c.span(),
841
                            format!("schedule {line:?}: expected five cron fields (minute hour day month weekday), as \"0 3 * * *\""),
842
                        );
843
                    }
844
                }
845
            }
846
            other => ctx.err(c.span(), format!("unknown `on` event `{other}`; expected push, tag, manual or schedule")),
847
        }
848
    }
849
    if on.push.is_none() && on.tag.is_none() && !on.manual && on.schedule.is_empty() {
850
        ctx.err(node.span(), "`on` names no event, so nothing would run it");
851
    }
852
    on
e576e086 Andy McCall ↶ 853
}
854
02b127d2 Andy McCall ↶ 855
/// The longest a pipeline may ask a job to run.
856
pub const MAX_TIMEOUT_S: u64 = 24 * 3600;
857
858
/// A duration: whole hours, minutes and seconds in that order, each
859
/// optional, as "90m", "2h", "1h30m" or "45s".
860
pub fn parse_duration(s: &str) -> Option<u64> {
861
    let (mut total, mut num, mut last) = (0u64, String::new(), 0u8);
862
    for ch in s.chars() {
863
        if ch.is_ascii_digit() {
864
            num.push(ch);
865
            continue;
866
        }
867
        let (unit, mult) = match ch {
868
            'h' => (1, 3600),
869
            'm' => (2, 60),
870
            's' => (3, 1),
871
            _ => return None,
872
        };
873
        // Each unit once, largest first, each with its number.
874
        if num.is_empty() || unit <= last {
875
            return None;
876
        }
877
        last = unit;
878
        total = total.checked_add(num.parse::<u64>().ok()?.checked_mul(mult)?)?;
879
        num.clear();
880
    }
881
    (num.is_empty() && total > 0).then_some(total)
882
}
883
884
/// A `timeout` line: one duration, from a minute to a day.
885
fn parse_timeout(ctx: &mut Ctx, c: &KdlNode) -> Option<u64> {
886
    let t = ctx.one_string(c)?;
887
    match parse_duration(&t) {
888
        Some(secs) if (60..=MAX_TIMEOUT_S).contains(&secs) => Some(secs),
889
        Some(_) => {
890
            ctx.err(c.span(), format!("timeout {t:?} must be from 1m to 24h"));
891
            None
892
        }
893
        None => {
894
            ctx.err(c.span(), format!("timeout {t:?}: expected hours, minutes and seconds, as \"90m\" or \"1h30m\""));
895
            None
896
        }
897
    }
898
}
899
e576e086 Andy McCall ↶ 900
/// The labels of a `runs-on` line: lower-case letters, digits, `-`, `_`
901
/// and `.`, at most 64 bytes each; into `into`, sorted, once each.
902
fn parse_labels(ctx: &mut Ctx, c: &KdlNode, into: &mut Vec<String>) {
903
    let labels = ctx.strings(c);
904
    if labels.is_empty() {
905
        ctx.err(c.span(), "`runs-on` needs one or more labels, as runs-on \"riscv64\"");
a72abeee Andy McCall ↶ 906
    }
e576e086 Andy McCall ↶ 907
    for l in labels {
908
        let ok = !l.is_empty()
909
            && l.len() <= 64
910
            && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'-' | b'_' | b'.'));
911
        if ok {
912
            into.push(l);
913
        } else {
914
            ctx.err(c.span(), format!("runs-on {l:?}: a label is lower-case letters, digits, -, _ and ., at most 64"));
915
        }
916
    }
917
    into.sort();
918
    into.dedup();
a72abeee Andy McCall ↶ 919
}
920
921
/// An `env` block of `NAME "value"` lines, into `env`.
922
fn parse_env(ctx: &mut Ctx, c: &KdlNode, env: &mut BTreeMap<String, String>) {
923
    if !c.entries().is_empty() {
924
        ctx.err(c.span(), "`env` takes a block of NAME \"value\" lines");
925
    }
926
    for e in block(ctx, c) {
927
        let k = e.name().value().to_string();
928
        let valid =
929
            !k.is_empty() && !k.starts_with(|c: char| c.is_ascii_digit()) && k.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_');
930
        if !valid {
931
            ctx.err(e.span(), format!("environment name {k:?} must be letters, digits and _"));
932
            continue;
933
        }
934
        ctx.no_children(e);
935
        if let Some(v) = ctx.one_string(e) {
936
            if env.insert(k.clone(), v).is_some() {
937
                ctx.err(e.span(), format!("environment {k} is set twice"));
938
            }
939
        }
e0918b88 Andy McCall 940
    }
941
}
942
fc10d9e8 Andy McCall ↶ 943
/// `mount { source "ephemeral://disk"; target "/src/target"; size "8GiB" }`.
944
fn parse_mount(ctx: &mut Ctx, node: &KdlNode) -> Option<Mount> {
945
    let (mut source, mut target, mut size) = (None, None, None);
d60aac6c Andy McCall ↶ 946
    let (mut cache, mut keys) = (None, Vec::new());
fc10d9e8 Andy McCall ↶ 947
    let mut seen = BTreeSet::new();
948
    for c in block(ctx, node) {
949
        let key = c.name().value();
950
        if !seen.insert(key.to_string()) {
951
            ctx.err(c.span(), format!("`{key}` is given twice"));
952
            continue;
953
        }
954
        ctx.no_children(c);
955
        match key {
956
            "source" => {
957
                source = match ctx.one_string(c).as_deref() {
958
                    Some("ephemeral://tmpfs") => Some(MountSource::Tmpfs),
959
                    Some("ephemeral://disk") => Some(MountSource::Disk),
960
                    Some(other) => {
961
                        ctx.err(
962
                            c.span(),
963
                            format!("mount source {other:?} is not in M0 yet; M0 mounts \"ephemeral://tmpfs\" or \"ephemeral://disk\""),
964
                        );
965
                        None
966
                    }
967
                    None => None,
968
                }
969
            }
970
            "target" => {
971
                target = match ctx.one_string(c) {
972
                    Some(t) if mount_target_ok(&t) => Some(t),
973
                    Some(t) => {
974
                        ctx.err(
975
                            c.span(),
976
                            format!("mount target {t:?} must be an absolute path under /src or /tmp, without . or .. or a trailing /"),
977
                        );
978
                        None
979
                    }
980
                    None => None,
981
                }
982
            }
983
            "size" => {
984
                size = match ctx.one_string(c) {
985
                    Some(m) => match parse_memory(&m) {
986
                        Some(mib) if (MIN_MOUNT_MIB..=MAX_MOUNT_MIB).contains(&mib) => Some(mib as u32),
987
                        Some(_) => {
988
                            ctx.err(c.span(), format!("mount size {m:?} must be from 16MiB to 64GiB"));
989
                            None
990
                        }
991
                        None => {
992
                            ctx.err(c.span(), format!("mount size {m:?}: expected a whole number of KiB, MiB or GiB, as \"8GiB\""));
993
                            None
994
                        }
995
                    },
996
                    None => None,
997
                }
998
            }
d60aac6c Andy McCall ↶ 999
            "cache" => {
1000
                cache = match ctx.one_string(c) {
1001
                    Some(n) if !n.is_empty() && n.len() <= 64 && n.bytes().all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)) => {
1002
                        Some(n)
1003
                    }
1004
                    Some(n) => {
1005
                        ctx.err(c.span(), format!("cache name {n:?}: letters, digits, '.', '_' and '-', at most 64"));
1006
                        None
1007
                    }
1008
                    None => None,
1009
                }
1010
            }
1011
            "key" => {
1012
                keys = ctx.strings(c);
1013
                if keys.is_empty() {
1014
                    ctx.err(c.span(), "`key` names one or more files, as \"Cargo.lock\"");
1015
                }
1016
                for k in &keys {
1017
                    if k.starts_with('/') || k.split('/').any(|p| p.is_empty() || p == "." || p == "..") {
1018
                        ctx.err(c.span(), format!("key file {k:?} must be a path in the repository, without . or .."));
1019
                    }
1020
                }
1021
            }
fc10d9e8 Andy McCall ↶ 1022
            "readonly" | "include" => ctx.err(c.span(), format!("mount `{key}` is not in M0 yet")),
1023
            other => ctx.err(c.span(), format!("unknown mount setting `{other}`")),
1024
        }
1025
    }
1026
    for (name, missing) in [("source", source.is_none()), ("target", target.is_none()), ("size", size.is_none())] {
1027
        if missing && !seen.contains(name) {
1028
            ctx.err(node.span(), format!("a mount needs a {name}"));
1029
        }
d60aac6c Andy McCall ↶ 1030
    }
1031
    if source == Some(MountSource::Tmpfs) && cache.is_some() {
1032
        ctx.err(node.span(), "only an ephemeral://disk mount can be a cache");
1033
    }
1034
    if !keys.is_empty() && cache.is_none() {
1035
        ctx.err(node.span(), "`key` goes with `cache`");
fc10d9e8 Andy McCall ↶ 1036
    }
d60aac6c Andy McCall ↶ 1037
    Some(Mount { source: source?, target: target?, size_mib: size?, cache, key: keys })
fc10d9e8 Andy McCall ↶ 1038
}
1039
a67b97e9 Andy McCall ↶ 1040
/// `output { artifact "name" { path "/src/target/release/tool" } }`.
e2c12561 Andy McCall ↶ 1041
/// `secret "deploy-token" { mount-path "/run/secrets/token" }` (spec §9.1).
1042
/// The hub is the vault: the secret of that name set for the repository,
1043
/// its organisation or the site. `vault-ref` is not in this slice.
1044
fn parse_secret(ctx: &mut Ctx, node: &KdlNode, job: &mut Job) {
1045
    let Some(name) = ctx.one_string(node) else {
1046
        ctx.err(node.span(), "`secret` takes a name and a block with mount-path");
1047
        return;
1048
    };
1049
    if !ctx.name_ok(node.span(), "secret", &name) {
1050
        return;
1051
    }
1052
    let mut path = None;
1053
    for c in block(ctx, node) {
1054
        ctx.no_children(c);
1055
        match c.name().value() {
1056
            "mount-path" => match ctx.one_string(c) {
1057
                Some(p) if secret_path_ok(&p) => path = Some(p),
1058
                Some(p) => {
1059
                    ctx.err(c.span(), format!("secret mount-path {p:?} must be an absolute path under /run/secrets/, without . or .."))
1060
                }
1061
                None => {}
1062
            },
1063
            "vault-ref" => ctx.err(c.span(), "`vault-ref` is not in M0; the hub supplies the secret of that name"),
1064
            other => ctx.err(c.span(), format!("unknown secret setting `{other}`")),
1065
        }
1066
    }
1067
    let Some(mount_path) = path else {
1068
        ctx.err(node.span(), format!("secret {name:?} needs a mount-path"));
1069
        return;
1070
    };
1071
    if job.secrets.iter().any(|s| s.name == name) {
1072
        ctx.err(node.span(), format!("secret {name:?} is given twice"));
720eb819 Andy McCall ↶ 1073
    } else if job.secrets.iter().any(|s| s.mount_path == mount_path) || job.identities.iter().any(|i| i.mount_path == mount_path) {
1074
        ctx.err(node.span(), format!("two files mount at {mount_path:?}"));
e2c12561 Andy McCall ↶ 1075
    } else if job.secrets.len() == MAX_SECRETS {
1076
        ctx.err(node.span(), format!("a job has at most {MAX_SECRETS} secrets"));
1077
    } else {
1078
        job.secrets.push(Secret { name, mount_path });
720eb819 Andy McCall ↶ 1079
    }
1080
}
1081
1082
/// `identity "sts.amazonaws.com" { mount-path "/run/secrets/aws-token" }`:
1083
/// the hub mints an OIDC token for that audience and the runner mounts it
1084
/// as a file (oxygenhub#103).
1085
fn parse_identity(ctx: &mut Ctx, node: &KdlNode, job: &mut Job) {
1086
    let Some(audience) = ctx.one_string(node) else {
1087
        ctx.err(node.span(), "`identity` takes an audience and a block with mount-path");
1088
        return;
1089
    };
1090
    let aud_ok = !audience.is_empty() && audience.len() <= 255 && !audience.chars().any(|c| c.is_whitespace() || c.is_control());
1091
    if !aud_ok {
1092
        ctx.err(node.span(), format!("identity audience {audience:?} must be 1 to 255 characters without spaces"));
1093
        return;
1094
    }
1095
    let mut path = None;
1096
    for c in block(ctx, node) {
1097
        ctx.no_children(c);
1098
        match c.name().value() {
1099
            "mount-path" => match ctx.one_string(c) {
1100
                Some(p) if secret_path_ok(&p) => path = Some(p),
1101
                Some(p) => {
1102
                    ctx.err(c.span(), format!("identity mount-path {p:?} must be an absolute path under /run/secrets/, without . or .."))
1103
                }
1104
                None => {}
1105
            },
1106
            other => ctx.err(c.span(), format!("unknown identity setting `{other}`")),
1107
        }
1108
    }
1109
    let Some(mount_path) = path else {
1110
        ctx.err(node.span(), format!("identity {audience:?} needs a mount-path"));
1111
        return;
1112
    };
1113
    if job.identities.iter().any(|i| i.audience == audience) {
1114
        ctx.err(node.span(), format!("identity {audience:?} is given twice"));
1115
    } else if job.identities.iter().any(|i| i.mount_path == mount_path) || job.secrets.iter().any(|s| s.mount_path == mount_path) {
1116
        ctx.err(node.span(), format!("two files mount at {mount_path:?}"));
1117
    } else if job.identities.len() == MAX_IDENTITIES {
1118
        ctx.err(node.span(), format!("a job has at most {MAX_IDENTITIES} identities"));
1119
    } else {
1120
        job.identities.push(Identity { audience, mount_path });
e2c12561 Andy McCall ↶ 1121
    }
1122
}
1123
a67b97e9 Andy McCall ↶ 1124
fn parse_output(ctx: &mut Ctx, node: &KdlNode, job: &mut Job) {
1125
    if !node.entries().is_empty() {
6d53fb7d Andy McCall ↶ 1126
        ctx.err(node.span(), "`output` takes a block of artifacts and reports");
a67b97e9 Andy McCall ↶ 1127
    }
1128
    for a in block(ctx, node) {
6d53fb7d Andy McCall ↶ 1129
        let kind = match a.name().value() {
1130
            "artifact" => OutputKind::Artifact,
1131
            "report" => OutputKind::Report,
1132
            other => {
1133
                ctx.err(
1134
                    a.span(),
1135
                    format!("unknown output `{other}`; outputs are `artifact \"name\" {{ path \"/src/file\" }}` and `report \"name\" {{ path \"/src/dir\" }}`"),
1136
                );
1137
                continue;
1138
            }
1139
        };
1140
        let word = kind.word();
a67b97e9 Andy McCall ↶ 1141
        let Some(name) = ctx.one_string(a) else { continue };
6d53fb7d Andy McCall ↶ 1142
        if !ctx.name_ok(a.span(), word, &name) {
a67b97e9 Andy McCall ↶ 1143
            continue;
1144
        }
1145
        let mut path = None;
9a6fac2b Andy McCall ↶ 1146
        let mut always = false;
a67b97e9 Andy McCall ↶ 1147
        for c in block(ctx, a) {
1148
            ctx.no_children(c);
1149
            match c.name().value() {
1150
                "path" => match ctx.one_string(c) {
1151
                    Some(p) if output_path_ok(&p) => path = Some(p),
6d53fb7d Andy McCall ↶ 1152
                    Some(p) => ctx
1153
                        .err(c.span(), format!("{word} path {p:?} must be an absolute path under /src, without . or .., quotes or spaces")),
a67b97e9 Andy McCall ↶ 1154
                    None => {}
1155
                },
6d53fb7d Andy McCall ↶ 1156
                "format" if kind == OutputKind::Artifact => {
a67b97e9 Andy McCall ↶ 1157
                    if ctx.one_string(c).as_deref() != Some("blob") {
1158
                        ctx.err(c.span(), "artifact format is \"blob\"");
1159
                    }
1160
                }
9a6fac2b Andy McCall ↶ 1161
                "when" => match ctx.one_string(c).as_deref() {
1162
                    Some("passed") => always = false,
1163
                    Some("always") => always = true,
6d53fb7d Andy McCall ↶ 1164
                    Some(_) => ctx.err(c.span(), format!("{word} `when` is \"passed\" (the default) or \"always\"")),
9a6fac2b Andy McCall ↶ 1165
                    None => {}
1166
                },
6d53fb7d Andy McCall ↶ 1167
                other => ctx.err(c.span(), format!("unknown {word} setting `{other}`")),
a67b97e9 Andy McCall ↶ 1168
            }
1169
        }
1170
        let Some(path) = path else {
6d53fb7d Andy McCall ↶ 1171
            ctx.err(a.span(), format!("{word} {name:?} needs a path"));
a67b97e9 Andy McCall ↶ 1172
            continue;
1173
        };
1174
        if job.outputs.iter().any(|o| o.name == name) {
6d53fb7d Andy McCall ↶ 1175
            ctx.err(a.span(), format!("output {name:?} is given twice (artifacts and reports share names)"));
a67b97e9 Andy McCall ↶ 1176
        } else if job.outputs.len() == MAX_OUTPUTS {
6d53fb7d Andy McCall ↶ 1177
            ctx.err(a.span(), format!("a job has at most {MAX_OUTPUTS} artifacts and reports"));
a67b97e9 Andy McCall ↶ 1178
        } else {
6d53fb7d Andy McCall ↶ 1179
            job.outputs.push(Output { name, path, always, kind });
a67b97e9 Andy McCall ↶ 1180
        }
1181
    }
1182
}
1183
e0918b88 Andy McCall 1184
fn parse_job(ctx: &mut Ctx, node: &KdlNode) -> Option<Job> {
1185
    let name = ctx.one_string(node)?;
1186
    let named = ctx.name_ok(node.span(), "job", &name);
fc10d9e8 Andy McCall ↶ 1187
    let mut job = Job {
1188
        name: name.clone(),
1189
        sandbox: String::new(),
1190
        scope: vec![],
1191
        depends_on: vec![],
1192
        env: BTreeMap::new(),
1193
        exec: vec![],
1194
        mounts: vec![],
e2c12561 Andy McCall ↶ 1195
        secrets: vec![],
720eb819 Andy McCall ↶ 1196
        identities: vec![],
a67b97e9 Andy McCall ↶ 1197
        outputs: vec![],
e576e086 Andy McCall ↶ 1198
        runs_on: vec![],
02b127d2 Andy McCall ↶ 1199
        timeout_s: None,
bf7bb1b8 Andy McCall ↶ 1200
        on: None,
fc10d9e8 Andy McCall ↶ 1201
    };
e0918b88 Andy McCall 1202
    let mut seen = BTreeSet::new();
a67b97e9 Andy McCall ↶ 1203
    let mut outputs_at = None;
e0918b88 Andy McCall 1204
    for c in block(ctx, node) {
1205
        let key = c.name().value();
720eb819 Andy McCall ↶ 1206
        if key != "scope"
1207
            && key != "depends-on"
1208
            && key != "mount"
1209
            && key != "secret"
1210
            && key != "identity"
1211
            && key != "runs-on"
1212
            && !seen.insert(key.to_string())
1213
        {
e0918b88 Andy McCall 1214
            ctx.err(c.span(), format!("`{key}` is given twice"));
1215
        }
1216
        match key {
1217
            "sandbox" => {
1218
                ctx.no_children(c);
1219
                job.sandbox = ctx.one_string(c).unwrap_or_default();
1220
            }
1221
            "scope" => {
1222
                ctx.no_children(c);
1223
                for g in ctx.strings(c) {
1224
                    match globset::Glob::new(&g) {
1225
                        Ok(_) => job.scope.push(g),
1226
                        Err(e) => ctx.err(c.span(), format!("scope {g:?}: {e}")),
1227
                    }
1228
                }
1229
            }
1230
            "depends-on" => {
1231
                ctx.no_children(c);
1232
                job.depends_on.extend(ctx.strings(c));
e576e086 Andy McCall ↶ 1233
            }
1234
            "runs-on" => {
1235
                ctx.no_children(c);
1236
                parse_labels(ctx, c, &mut job.runs_on);
02b127d2 Andy McCall ↶ 1237
            }
1238
            "timeout" => {
1239
                ctx.no_children(c);
1240
                job.timeout_s = parse_timeout(ctx, c);
e0918b88 Andy McCall 1241
            }
bf7bb1b8 Andy McCall ↶ 1242
            "on" => job.on = Some(parse_on(ctx, c)),
a72abeee Andy McCall ↶ 1243
            "env" => parse_env(ctx, c, &mut job.env),
e0918b88 Andy McCall 1244
            "exec" => {
1245
                ctx.no_children(c);
1246
                job.exec = ctx.strings(c);
1247
                if job.exec.is_empty() {
1248
                    ctx.err(c.span(), "`exec` needs a program and its arguments");
1249
                } else if job.exec.iter().any(|a| a.contains('\0')) {
1250
                    ctx.err(c.span(), "`exec` arguments cannot contain NUL");
1251
                }
1252
            }
fc10d9e8 Andy McCall ↶ 1253
            "mount" => {
1254
                if !c.entries().is_empty() {
1255
                    ctx.err(c.span(), "`mount` takes a block of source, target and size");
1256
                }
1257
                if let Some(m) = parse_mount(ctx, c) {
1258
                    if job.mounts.len() == MAX_MOUNTS {
1259
                        ctx.err(c.span(), format!("a job has at most {MAX_MOUNTS} mounts"));
1260
                    } else if let Some(other) = job.mounts.iter().find(|o| {
1261
                        o.target == m.target
1262
                            || m.target.starts_with(&format!("{}/", o.target))
1263
                            || o.target.starts_with(&format!("{}/", m.target))
1264
                    }) {
1265
                        ctx.err(c.span(), format!("mount {:?} overlaps mount {:?}", m.target, other.target));
1266
                    } else {
1267
                        job.mounts.push(m);
1268
                    }
1269
                }
1270
            }
a67b97e9 Andy McCall ↶ 1271
            "output" => {
1272
                outputs_at = Some(c.span());
1273
                parse_output(ctx, c, &mut job);
1274
            }
e2c12561 Andy McCall ↶ 1275
            "secret" => parse_secret(ctx, c, &mut job),
720eb819 Andy McCall ↶ 1276
            "identity" => parse_identity(ctx, c, &mut job),
e0918b88 Andy McCall 1277
            n if LATER_JOB.contains(&n) => ctx.err(c.span(), format!("`{n}` is not in M0 yet")),
1278
            other => ctx.err(c.span(), format!("unknown job setting `{other}`")),
1279
        }
1280
    }
fc10d9e8 Andy McCall ↶ 1281
    job.mounts.sort_by(|a, b| a.target.cmp(&b.target));
a67b97e9 Andy McCall ↶ 1282
    job.outputs.sort_by(|a, b| a.name.cmp(&b.name));
e2c12561 Andy McCall ↶ 1283
    job.secrets.sort_by(|a, b| a.name.cmp(&b.name));
720eb819 Andy McCall ↶ 1284
    job.identities.sort_by(|a, b| a.audience.cmp(&b.audience));
a67b97e9 Andy McCall ↶ 1285
    // A tmpfs is gone when the job ends; nothing on it can be kept.
1286
    for o in &job.outputs {
1287
        if let Some(m) = job.mounts.iter().find(|m| m.source == MountSource::Tmpfs && o.path.starts_with(&format!("{}/", m.target))) {
1288
            ctx.err(
1289
                outputs_at.unwrap_or(node.span()),
1290
                format!(
1291
                    "artifact {:?} is on the tmpfs mount {:?}, which is gone when the job ends; use ephemeral://disk",
1292
                    o.name, m.target
1293
                ),
1294
            );
1295
        }
1296
    }
e0918b88 Andy McCall 1297
    if job.sandbox.is_empty() && !seen.contains("sandbox") {
1298
        ctx.err(node.span(), format!("job {name:?} needs a sandbox"));
1299
    }
1300
    if job.exec.is_empty() && !seen.contains("exec") {
1301
        ctx.err(node.span(), format!("job {name:?} needs an exec"));
1302
    }
1303
    job.depends_on.sort();
1304
    job.depends_on.dedup();
1305
    // Kept even when incomplete, so its references are checked too; the
1306
    // problems above already make the pipeline invalid.
1307
    named.then_some(job)
1308
}