OXYGENHUB
# a4b3f8cb9aae oxygenflow / crates / flow-core / src / tests.rs
29.7 KiB · Rust · last changed A build cache: disk mounts that carry a build forward 22 days ago source blame history raw
1
use crate::*;
2
3
const ROOTFS: &str = "9b5c2a1e8f3d4e7a6b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a";
4
5
fn doc(body: &str) -> String {
6
    format!(
7
        "schema-version 1\n\nsandbox \"alpine\" {{\n    runtime \"microvm\"\n    rootfs \"cas://blob/{ROOTFS}\"\n    memory-limit \"512MiB\"\n    cpu-limit 2\n    network \"isolated\"\n}}\n\n{body}"
8
    )
9
}
10
11
fn problems(src: &str) -> Vec<(usize, String)> {
12
    Pipeline::parse(src).unwrap_err().0.into_iter().map(|p| (p.line, p.message)).collect()
13
}
14
15
#[test]
16
fn parses_the_m0_example() {
17
    let p = Pipeline::parse(&doc(r#"job "lint" {
18
    sandbox "alpine"
19
    scope "src/**" "README.md"
20
    env {
21
        LANG "C.UTF-8"
22
    }
23
    exec "grep" "-rn" "TODO" "src"
24
}
25
26
job "test" {
27
    sandbox "alpine"
28
    depends-on "lint"
29
    exec "sh" "-eu" "test.sh"
30
}
31
"#))
32
    .unwrap();
33
    let sb = &p.sandboxes["alpine"];
34
    assert_eq!((sb.runtime, sb.memory_mib, sb.cpus, sb.network), (Runtime::Microvm, 512, 2, Network::Isolated));
35
    assert_eq!(sb.rootfs, ROOTFS);
36
    let lint = &p.jobs["lint"];
37
    assert_eq!(lint.exec, ["grep", "-rn", "TODO", "src"]);
38
    assert_eq!(lint.scope, ["src/**", "README.md"]);
39
    assert_eq!(lint.env["LANG"], "C.UTF-8");
40
    assert_eq!(p.jobs["test"].depends_on, ["lint"]);
41
    assert_eq!(p.order().iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), ["lint", "test"]);
42
    assert_eq!(p.depths()["test"], 1);
43
}
44
45
#[test]
46
fn defaults_and_gib() {
47
    let p = Pipeline::parse(&format!(
48
        "schema-version 1\nsandbox \"s\" {{\n  rootfs \"cas://blob/{ROOTFS}\"\n  memory-limit \"2GiB\"\n}}\njob \"a\" {{\n  sandbox \"s\"\n  exec \"true\"\n}}\n"
49
    ))
50
    .unwrap();
51
    let s = &p.sandboxes["s"];
52
    assert_eq!((s.runtime, s.memory_mib, s.cpus, s.network), (Runtime::Microvm, 2048, 1, Network::Isolated));
53
}
54
55
#[test]
56
fn shell_metacharacters_stay_literal_argv() {
57
    // Spec §10.1: every byte reaches argv as written; nothing adds a shell.
58
    let p = Pipeline::parse(&doc(r#"job "x" { sandbox "alpine"; exec "echo" "a; rm -rf / && $(id) | `x`" "b\nc"; }"#)).unwrap();
59
    assert_eq!(p.jobs["x"].exec, ["echo", "a; rm -rf / && $(id) | `x`", "b\nc"]);
60
}
61
62
#[test]
63
fn problems_carry_lines_and_are_all_reported() {
64
    let src = doc(r#"job "a" {
65
    sandbox "missing"
66
    exec "true"
67
    mount { source "oxy://workspace"; }
68
}
69
job "b" {
70
    sandbox "alpine"
71
    depends-on "nope"
72
    exec
73
}
74
flavour "x"
75
"#);
76
    let ps = problems(&src);
77
    let has = |line: usize, text: &str| ps.iter().any(|(l, m)| *l == line && m.contains(text));
78
    assert!(has(11, "which is not declared") && has(11, "\"missing\""), "{ps:?}");
79
    assert!(has(14, "mount source \"oxy://workspace\" is not in M0 yet"), "{ps:?}");
80
    assert!(has(16, "\"nope\""), "{ps:?}");
81
    assert!(has(19, "`exec` needs a program"), "{ps:?}");
82
    assert!(has(21, "unknown node `flavour`"), "{ps:?}");
83
}
84
85
#[test]
86
fn typed_values_are_not_coerced() {
87
    let src = format!(
88
        "schema-version 1\nsandbox \"s\" {{\n  rootfs \"cas://blob/{ROOTFS}\"\n  cpu-limit \"2\"\n  memory-limit \"1.5GiB\"\n}}\njob \"a\" {{\n  sandbox \"s\"\n  exec \"echo\" 1\n}}\n"
89
    );
90
    let ps = problems(&src);
91
    assert!(ps.iter().any(|(l, m)| *l == 4 && m.contains("integer, not a string")), "{ps:?}");
92
    assert!(ps.iter().any(|(l, m)| *l == 5 && m.contains("whole number")), "{ps:?}");
93
    assert!(ps.iter().any(|(l, m)| *l == 9 && m.contains("strings, not an integer")), "{ps:?}");
94
}
95
96
#[test]
97
fn m0_refuses_what_it_cannot_honour() {
98
    let src = format!("schema-version 1\nsandbox \"s\" {{\n  runtime \"wasm\"\n  rootfs \"cas://blob/{ROOTFS}\"\n  network \"egress\"\n}}\njob \"a\" {{ sandbox \"s\"; exec \"true\"; }}\n");
99
    let ps = problems(&src);
100
    assert!(ps.iter().any(|(l, m)| *l == 3 && m.contains("wasm")), "{ps:?}");
101
    assert!(ps.iter().any(|(l, m)| *l == 5 && m.contains("isolated")), "{ps:?}");
102
    assert!(problems("schema-version 2\n").iter().any(|(_, m)| m.contains("schema-version 2")));
103
    assert!(problems(&format!(
104
        "schema-version 1\nsandbox \"s\" {{ rootfs \"cas://blob/{}\"; }}\njob \"a\" {{ sandbox \"s\"; exec \"true\"; }}",
105
        &ROOTFS[..60]
106
    ))
107
    .iter()
108
    .any(|(_, m)| m.contains("64 lower-case hex")));
109
}
110
111
#[test]
112
fn cycles_are_found() {
113
    let ps = problems(&doc(
114
        "job \"a\" { sandbox \"alpine\"; depends-on \"c\"; exec \"true\"; }\njob \"b\" { sandbox \"alpine\"; depends-on \"a\"; exec \"true\"; }\njob \"c\" { sandbox \"alpine\"; depends-on \"b\"; exec \"true\"; }\njob \"d\" { sandbox \"alpine\"; depends-on \"d\"; exec \"true\"; }\n",
115
    ));
116
    assert!(ps.iter().any(|(_, m)| m.contains("cycle: a → c → b → a")), "{ps:?}");
117
    assert!(ps.iter().any(|(_, m)| m.contains("\"d\" depends on itself")), "{ps:?}");
118
}
119
120
#[test]
121
fn invalid_kdl_has_a_line() {
122
    let ps = problems("schema-version 1\njob \"a\" {\n  exec \"unterminated\n}\n");
123
    assert!(!ps.is_empty() && ps[0].0 >= 2, "{ps:?}");
124
}
125
126
#[test]
127
fn canonical_form_ignores_layout_and_order() {
128
    let a = Pipeline::parse(&doc(
129
        "job \"b\" { sandbox \"alpine\"; exec \"true\"; }\njob \"a\" {\n  sandbox \"alpine\"\n  depends-on \"b\"\n  exec \"x\"\n}\n",
130
    ))
131
    .unwrap();
132
    let b = Pipeline::parse(&doc("// comment\njob \"a\" { exec \"x\"; depends-on \"b\"; sandbox \"alpine\"; }\n\n\njob \"b\" { exec \"true\"; sandbox \"alpine\"; }\n")).unwrap();
133
    assert_eq!(a.canonical(), b.canonical());
134
    assert_eq!(a.hash(), b.hash());
135
    let c = Pipeline::parse(&doc(
136
        "job \"b\" { sandbox \"alpine\"; exec \"true\"; }\njob \"a\" { sandbox \"alpine\"; depends-on \"b\"; exec \"y\"; }\n",
137
    ))
138
    .unwrap();
139
    assert_ne!(a.hash(), c.hash());
140
    let sb = &a.sandboxes["alpine"];
141
    assert_ne!(a.jobs["a"].hash(sb), c.jobs["a"].hash(sb));
142
    assert_eq!(a.jobs["b"].hash(sb), c.jobs["b"].hash(sb), "an unchanged job keeps its hash");
143
    // The canonical form round-trips through Oxygen's strict decoder.
144
    oxy_core::cbor::decode(&a.canonical()).unwrap();
145
}
146
147
#[test]
148
fn scope_tree_hash_sees_only_the_scope() {
149
    let f = |b: u8| [b; 32];
150
    let scope = vec!["src/**".to_string()];
151
    let base = scope_tree_hash(&scope, [("src/a.rs", f(1)), ("README.md", f(2))]);
152
    assert_eq!(base, scope_tree_hash(&scope, [("README.md", f(9)), ("src/a.rs", f(1))]), "outside the scope, and order, do not count");
153
    assert_ne!(base, scope_tree_hash(&scope, [("src/a.rs", f(3)), ("README.md", f(2))]));
154
    assert_ne!(scope_tree_hash(&[], [("README.md", f(2))]), scope_tree_hash(&[], [("README.md", f(9))]), "no scope is the whole tree");
155
}
156
157
#[test]
158
fn receipts_hash_as_oxygen_objects() {
159
    let r = Receipt {
160
        job_name: "lint".into(),
161
        pipeline_hash: [1; 32],
162
        scope_tree_hash: [2; 32],
163
        inputs: vec![],
164
        environment: [3; 32],
165
        outputs: vec![],
166
        exit_code: 0,
167
        duration_ms: 1234,
168
    };
169
    let body = r.encode();
170
    let mut want = format!("oxy1 receipt {}\0", body.len()).into_bytes();
171
    want.extend_from_slice(&body);
172
    assert_eq!(r.hash(), *blake3::hash(&want).as_bytes());
173
    let json = serde_json::to_string(&r).unwrap();
174
    assert_eq!(serde_json::from_str::<Receipt>(&json).unwrap(), r);
175
    // The same receipt as the Oxygen object a runner signs.
176
    assert_eq!(r.to_oxy().hash().0, r.hash());
177
    assert_eq!(r.to_oxy().encode(), body);
178
    let mut r2 = r.clone();
179
    r2.exit_code = 1;
180
    assert_ne!(r.hash(), r2.hash());
181
}
182
183
#[test]
184
fn ephemeral_mounts_parse_sort_and_hash() {
185
    let plain = Pipeline::parse(&doc("job \"b\" {\n    sandbox \"alpine\"\n    exec \"cargo\" \"build\"\n}\n")).unwrap();
186
    let p = Pipeline::parse(&doc(r#"job "b" {
187
    sandbox "alpine"
188
    mount {
189
        source "ephemeral://tmpfs"
190
        target "/tmp/cache"
191
        size "256MiB"
192
    }
193
    mount {
194
        source "ephemeral://disk"
195
        target "/src/target"
196
        size "8GiB"
197
    }
198
    exec "cargo" "build"
199
}
200
"#))
201
    .unwrap();
202
    let b = &p.jobs["b"];
203
    assert_eq!(
204
        b.mounts,
205
        vec![
206
            Mount { source: MountSource::Disk, target: "/src/target".into(), size_mib: 8192, cache: None, key: vec![] },
207
            Mount { source: MountSource::Tmpfs, target: "/tmp/cache".into(), size_mib: 256, cache: None, key: vec![] },
208
        ],
209
        "sorted by target"
210
    );
211
    assert_ne!(p.hash(), plain.hash(), "mounts are part of the job");
212
    // A job without mounts encodes as before mounts existed: no field 6.
213
    let v = oxy_core::cbor::decode(&plain.canonical()).unwrap();
214
    let text = format!("{v:?}");
215
    assert!(!text.contains("ephemeral"), "{text}");
216
    let json = serde_json::to_value(&plain.jobs["b"]).unwrap();
217
    assert!(json.get("mounts").is_none(), "absent from the API too");
218
    let back: Job = serde_json::from_value(json).unwrap();
219
    assert!(back.mounts.is_empty());
220
}
221
222
#[test]
223
fn a_disk_mount_can_be_a_cache_keyed_by_files() {
224
    let src = |extra: &str| {
225
        doc(&format!(
226
            "job \"b\" {{\n    sandbox \"alpine\"\n    mount {{\n        source \"ephemeral://disk\"\n        target \"/src/target\"\n        size \"8GiB\"\n{extra}    }}\n    exec \"cargo\" \"build\"\n}}\n"
227
        ))
228
    };
229
    let plain = Pipeline::parse(&src("")).unwrap();
230
    let cached = Pipeline::parse(&src("        cache \"target\"\n        key \"Cargo.lock\" \"rust-toolchain.toml\"\n")).unwrap();
231
    let m = &cached.jobs["b"].mounts[0];
232
    assert_eq!((m.cache.as_deref(), m.key.clone()), (Some("target"), vec!["Cargo.lock".to_string(), "rust-toolchain.toml".to_string()]));
233
    assert_ne!(cached.hash(), plain.hash(), "a cache is part of the job");
234
    let keyless = Pipeline::parse(&src("        cache \"target\"\n")).unwrap();
235
    assert_ne!(keyless.hash(), cached.hash());
236
    let ps = problems(&src("        cache \"no spaces\"\n"));
237
    assert!(ps.iter().any(|(_, m)| m.contains("cache name")), "{ps:?}");
238
    let ps = problems(&src("        key \"Cargo.lock\"\n"));
239
    assert!(ps.iter().any(|(_, m)| m.contains("goes with `cache`")), "{ps:?}");
240
    let ps = problems(&src("        cache \"t\"\n        key \"../x\"\n"));
241
    assert!(ps.iter().any(|(_, m)| m.contains("path in the repository")), "{ps:?}");
242
    let tmpfs = doc("job \"b\" {\n    sandbox \"alpine\"\n    mount {\n        source \"ephemeral://tmpfs\"\n        target \"/tmp/c\"\n        size \"64MiB\"\n        cache \"c\"\n    }\n    exec \"true\"\n}\n");
243
    assert!(problems(&tmpfs).iter().any(|(_, m)| m.contains("only an ephemeral://disk")));
244
}
245
246
#[test]
247
fn mounts_are_checked() {
248
    let job = |mount: &str| doc(&format!("job \"b\" {{\n    sandbox \"alpine\"\n    mount {{\n{mount}\n    }}\n    exec \"true\"\n}}\n"));
249
    let has = |src: String, want: &str| {
250
        let ps = problems(&src);
251
        assert!(ps.iter().any(|(_, m)| m.contains(want)), "{want:?} not in {ps:?}");
252
    };
253
    has(job("source \"oxy://workspace\"\ntarget \"/src/x\"\nsize \"1GiB\""), "not in M0 yet");
254
    has(job("source \"ephemeral://disk\"\ntarget \"/etc/x\"\nsize \"1GiB\""), "under /src or /tmp");
255
    has(job("source \"ephemeral://disk\"\ntarget \"/src\"\nsize \"1GiB\""), "under /src or /tmp");
256
    has(job("source \"ephemeral://disk\"\ntarget \"/src/../etc\"\nsize \"1GiB\""), "without . or ..");
257
    has(job("source \"ephemeral://disk\"\ntarget \"/src/x/\"\nsize \"1GiB\""), "trailing /");
258
    has(job("source \"ephemeral://disk\"\ntarget \"/src/x\"\nsize \"8MiB\""), "from 16MiB to 64GiB");
259
    has(job("source \"ephemeral://disk\"\ntarget \"/src/x\"\nsize \"65GiB\""), "from 16MiB to 64GiB");
260
    has(job("source \"ephemeral://disk\"\ntarget \"/src/x\"\nsize \"lots\""), "whole number");
261
    has(job("source \"ephemeral://disk\"\ntarget \"/src/x\""), "needs a size");
262
    has(job("source \"ephemeral://disk\"\ntarget \"/src/x\"\nsize \"1GiB\"\nreadonly #true"), "`readonly` is not in M0 yet");
263
    has(job("source \"ephemeral://disk\"\nsource \"ephemeral://tmpfs\"\ntarget \"/src/x\"\nsize \"1GiB\""), "given twice");
264
    let two = |a: &str, b: &str| {
265
        doc(&format!(
266
            "job \"b\" {{\n    sandbox \"alpine\"\n    mount {{ source \"ephemeral://disk\"; target \"{a}\"; size \"1GiB\"; }}\n    mount {{ source \"ephemeral://disk\"; target \"{b}\"; size \"1GiB\"; }}\n    exec \"true\"\n}}\n"
267
        ))
268
    };
269
    has(two("/src/t", "/src/t/deps"), "overlaps");
270
    has(two("/src/t", "/src/t"), "overlaps");
271
    assert!(Pipeline::parse(&two("/src/t", "/src/tt")).is_ok(), "a shared prefix is not nesting");
272
    let five: String =
273
        (0..5).map(|i| format!("    mount {{ source \"ephemeral://tmpfs\"; target \"/tmp/m{i}\"; size \"16MiB\"; }}\n")).collect();
274
    has(doc(&format!("job \"b\" {{\n    sandbox \"alpine\"\n{five}    exec \"true\"\n}}\n")), "at most 4 mounts");
275
}
276
277
#[test]
278
fn loopback_is_in_m0_and_egress_is_not() {
279
    let src = |net: &str| {
280
        format!("schema-version 1\nsandbox \"s\" {{\n  rootfs \"cas://blob/{ROOTFS}\"\n  network \"{net}\"\n}}\njob \"a\" {{ sandbox \"s\"; exec \"true\"; }}\n")
281
    };
282
    let p = Pipeline::parse(&src("loopback")).unwrap();
283
    assert_eq!(p.sandboxes["s"].network, Network::Loopback);
284
    assert_ne!(p.hash(), Pipeline::parse(&src("isolated")).unwrap().hash(), "the network is part of the sandbox");
285
    assert!(problems(&src("egress")).iter().any(|(l, m)| *l == 4 && m.contains("not in M0")));
286
}
287
288
#[test]
289
fn secrets_mount_as_files_and_are_part_of_the_job() {
290
    let plain = Pipeline::parse(&doc("job \"d\" {\n    sandbox \"alpine\"\n    exec \"deploy\"\n}\n")).unwrap();
291
    let p = Pipeline::parse(&doc(r#"job "d" {
292
    sandbox "alpine"
293
    secret "token" {
294
        mount-path "/run/secrets/token"
295
    }
296
    secret "aws" {
297
        mount-path "/run/secrets/aws/credentials"
298
    }
299
    exec "deploy"
300
}"#))
301
    .unwrap();
302
    let d = &p.jobs["d"];
303
    assert_eq!(
304
        d.secrets,
305
        vec![
306
            Secret { name: "aws".into(), mount_path: "/run/secrets/aws/credentials".into() },
307
            Secret { name: "token".into(), mount_path: "/run/secrets/token".into() },
308
        ],
309
        "sorted by name"
310
    );
311
    assert_ne!(p.hash(), plain.hash(), "secrets are part of the job");
312
    for (bad, why) in [
313
        ("secret \"t\" {\n    mount-path \"/tmp/t\"\n}", "under /run/secrets/"),
314
        ("secret \"t\" {\n    mount-path \"/run/secrets/../x\"\n}", "under /run/secrets/"),
315
        ("secret \"t\" {\n}", "needs a mount-path"),
316
        ("secret \"t\" {\n    mount-path \"/run/secrets/a\"\n}\nsecret \"t\" {\n    mount-path \"/run/secrets/b\"\n}", "given twice"),
317
        (
318
            "secret \"t\" {\n    mount-path \"/run/secrets/a\"\n}\nsecret \"u\" {\n    mount-path \"/run/secrets/a\"\n}",
319
            "two files mount at",
320
        ),
321
        ("secret \"t\" {\n    vault-ref \"x\"\n    mount-path \"/run/secrets/a\"\n}", "vault-ref"),
322
    ] {
323
        let e = Pipeline::parse(&doc(&format!("job \"d\" {{\n    sandbox \"alpine\"\n    {bad}\n    exec \"deploy\"\n}}\n"))).unwrap_err();
324
        assert!(e.to_string().contains(why), "{bad}: {e}");
325
    }
326
}
327
328
#[test]
329
fn identities_are_asked_for_by_audience_and_mount_like_secrets() {
330
    let plain = Pipeline::parse(&doc("job \"d\" {\n    sandbox \"alpine\"\n    exec \"deploy\"\n}\n")).unwrap();
331
    let p = Pipeline::parse(&doc(r#"job "d" {
332
    sandbox "alpine"
333
    identity "sts.amazonaws.com" {
334
        mount-path "/run/secrets/aws-token"
335
    }
336
    identity "https://vault.example" {
337
        mount-path "/run/secrets/vault-token"
338
    }
339
    secret "key" {
340
        mount-path "/run/secrets/key"
341
    }
342
    exec "deploy"
343
}"#))
344
    .unwrap();
345
    let d = &p.jobs["d"];
346
    assert_eq!(
347
        d.identities.iter().map(|i| i.audience.as_str()).collect::<Vec<_>>(),
348
        ["https://vault.example", "sts.amazonaws.com"],
349
        "sorted"
350
    );
351
    assert_ne!(p.hash(), plain.hash(), "identities are part of the job");
352
    for (bad, why) in [
353
        ("identity \"a b\" {\n    mount-path \"/run/secrets/x\"\n}", "without spaces"),
354
        ("identity \"a\" {\n    mount-path \"/tmp/x\"\n}", "under /run/secrets/"),
355
        ("identity \"a\" {\n}", "needs a mount-path"),
356
        ("identity \"a\" {\n    mount-path \"/run/secrets/x\"\n}\nidentity \"a\" {\n    mount-path \"/run/secrets/y\"\n}", "given twice"),
357
        (
358
            "identity \"a\" {\n    mount-path \"/run/secrets/x\"\n}\nsecret \"s\" {\n    mount-path \"/run/secrets/x\"\n}",
359
            "two files mount at",
360
        ),
361
    ] {
362
        let e = Pipeline::parse(&doc(&format!("job \"d\" {{\n    sandbox \"alpine\"\n    {bad}\n    exec \"deploy\"\n}}\n"))).unwrap_err();
363
        assert!(e.to_string().contains(why), "{bad}: {e}");
364
    }
365
}
366
367
#[test]
368
fn a_sandbox_may_name_its_architecture() {
369
    let plain = Pipeline::parse(&doc("job \"b\" {\n    sandbox \"alpine\"\n    exec \"make\"\n}\n")).unwrap();
370
    assert_eq!(plain.sandboxes["alpine"].arch, None);
371
    let src =
372
        doc("job \"b\" {\n    sandbox \"alpine\"\n    exec \"make\"\n}\n").replace("memory-limit", "arch \"riscv64\"\n    memory-limit");
373
    let p = Pipeline::parse(&src).unwrap();
374
    assert_eq!(p.sandboxes["alpine"].arch.as_deref(), Some("riscv64"));
375
    assert_ne!(p.hash(), plain.hash(), "the architecture is part of the sandbox");
376
    let bad = src.replace("riscv64", "RISC V");
377
    assert!(Pipeline::parse(&bad).unwrap_err().to_string().contains("not an architecture name"));
378
    assert!(arch_ok("x86_64") && arch_ok("aarch64") && !arch_ok("x") && !arch_ok("A64"));
379
}
380
381
#[test]
382
fn jobs_are_told_what_they_build() {
383
    let p = Pipeline::parse(&doc(r#"job "build" {
384
    sandbox "alpine"
385
    env {
386
        FLOW_BOOKMARK "pinned-by-the-pipeline"
387
    }
388
    exec "make"
389
}"#))
390
    .unwrap();
391
    let job = p.jobs["build"].clone();
392
    let claim = crate::api::Claim {
393
        job_id: 7,
394
        run_id: 40,
395
        repo: "andy/demo".into(),
396
        clone_url: "https://hub/andy/demo".into(),
397
        token: "t".into(),
398
        heads: vec!["aa".into(), "bb".into()],
399
        pipeline_hash: "00".into(),
400
        job: job.clone(),
401
        sandbox: p.sandboxes["alpine"].clone(),
402
        timeout_s: 60,
403
        subrepos: vec![],
404
        state: "cc".into(),
405
        bookmark: "feature/x".into(),
406
        run_number: 3,
407
        env: [("CC".to_string(), "clang".to_string()), ("FLOW_REPO".to_string(), "spoofed".to_string())].into_iter().collect(),
408
        secrets: vec![],
409
    };
410
    let env = job.with_flow_env(&claim.flow_env()).env;
411
    assert_eq!(env["CC"], "clang", "the hub's variables reach the job");
412
    assert_eq!(env["FLOW_REPO"], "andy/demo", "a hub variable cannot pose as a FLOW_ fact");
413
    assert_eq!(env["FLOW_HEADS"], "aa+bb");
414
    assert_eq!(env["FLOW_STATE"], "cc");
415
    assert_eq!(env["FLOW_RUN"], "3", "the run's number in its repository, not the hub's id");
416
    assert_eq!(env["FLOW_JOB"], "build");
417
    assert_eq!(env["FLOW_BOOKMARK"], "pinned-by-the-pipeline", "the pipeline's own value wins");
418
    // Unknown values (an older hub) are left out, not empty.
419
    let old = crate::api::Claim {
420
        state: String::new(),
421
        bookmark: String::new(),
422
        run_number: 0,
423
        env: Default::default(),
424
        secrets: vec![],
425
        ..claim
426
    };
427
    let names: Vec<String> = old.flow_env().into_iter().map(|(k, _)| k).collect();
428
    assert_eq!(names, ["FLOW_REPO", "FLOW_HEADS", "FLOW_JOB"]);
429
}
430
431
#[test]
432
fn a_sandbox_env_is_where_jobs_start() {
433
    let src = |sandbox_env: &str| {
434
        format!(
435
            "schema-version 1\nsandbox \"mos\" {{\n    rootfs \"cas://blob/{ROOTFS}\"\n{sandbox_env}}}\njob \"build\" {{\n    sandbox \"mos\"\n    env {{\n        X16EMU \"/opt/job\"\n    }}\n    exec \"make\"\n}}\n"
436
        )
437
    };
438
    let with =
439
        Pipeline::parse(&src("    env {\n        PATH \"/opt/llvm-mos/bin:/usr/bin:/bin\"\n        X16EMU \"/opt/x16emu\"\n    }\n"))
440
            .unwrap();
441
    let sb = &with.sandboxes["mos"];
442
    assert_eq!(sb.env["PATH"], "/opt/llvm-mos/bin:/usr/bin:/bin");
443
    let job = with.jobs["build"].in_sandbox(sb);
444
    assert_eq!(job.env["PATH"], "/opt/llvm-mos/bin:/usr/bin:/bin", "from the sandbox");
445
    assert_eq!(job.env["X16EMU"], "/opt/job", "the job's own value wins");
446
    // The sandbox's environment is part of the pipeline's hash; a sandbox
447
    // without one hashes as before.
448
    let without = Pipeline::parse(&src("")).unwrap();
449
    assert!(without.sandboxes["mos"].env.is_empty());
450
    assert_ne!(with.hash(), without.hash());
451
    let bad = problems(&src("    env {\n        BAD-NAME \"x\"\n    }\n"));
452
    assert!(bad.iter().any(|(_, m)| m.contains("letters, digits and _")), "{bad:?}");
453
}
454
455
#[test]
456
fn artifacts_are_declared_checked_and_hashed() {
457
    let src = |out: &str| {
458
        doc(&format!(
459
            "job \"build\" {{\n    sandbox \"alpine\"\n    mount {{\n        source \"ephemeral://disk\"\n        target \"/src/target\"\n        size \"1GiB\"\n    }}\n    mount {{\n        source \"ephemeral://tmpfs\"\n        target \"/src/tmp\"\n        size \"64MiB\"\n    }}\n    exec \"make\"\n{out}}}\n"
460
        ))
461
    };
462
    let p = Pipeline::parse(&src(
463
        "    output {\n        artifact \"tool\" {\n            path \"/src/target/release/tool\"\n            format \"blob\"\n        }\n        artifact \"docs\" {\n            path \"/src/docs/tool.1\"\n        }\n    }\n",
464
    ))
465
    .unwrap();
466
    let names: Vec<(&str, &str)> = p.jobs["build"].outputs.iter().map(|o| (o.name.as_str(), o.path.as_str())).collect();
467
    assert_eq!(names, [("docs", "/src/docs/tool.1"), ("tool", "/src/target/release/tool")], "sorted by name");
468
    // Artifacts are part of the job's hash; a job without them hashes as before.
469
    let plain = Pipeline::parse(&src("")).unwrap();
470
    let sb = &p.sandboxes["alpine"];
471
    assert_ne!(p.jobs["build"].hash(sb), plain.jobs["build"].hash(sb));
472
    assert!(plain.jobs["build"].outputs.is_empty());
473
    let json = serde_json::to_string(&plain.jobs["build"]).unwrap();
474
    assert!(!json.contains("outputs"), "no outputs field when there are none: {json}");
475
476
    let bad = |out: &str| problems(&src(out)).into_iter().map(|(_, m)| m).collect::<Vec<_>>().join("\n");
477
    assert!(bad("    output {\n        artifact \"x\" {\n            path \"/etc/passwd\"\n        }\n    }\n")
478
        .contains("must be an absolute path under /src"));
479
    assert!(bad("    output {\n        artifact \"x\" {\n            path \"/src/../etc\"\n        }\n    }\n")
480
        .contains("must be an absolute path under /src"));
481
    assert!(bad("    output {\n        artifact \"x\" {\n            path \"/src/tmp/a\"\n        }\n    }\n").contains("tmpfs"));
482
    assert!(bad("    output {\n        artifact \"x\" {\n        }\n    }\n").contains("needs a path"));
483
    assert!(bad("    output {\n        artifact \"x\" {\n            path \"/src/a\"\n        }\n        artifact \"x\" {\n            path \"/src/b\"\n        }\n    }\n")
484
        .contains("given twice"));
485
    assert!(bad("    output {\n        artifact \"x\" {\n            path \"/src/a\"\n            format \"tar\"\n        }\n    }\n")
486
        .contains("format"));
487
488
    // oxygenflow#37: `when "always"` keeps a failing job's report; the
489
    // default hashes and serialises as before.
490
    let p = Pipeline::parse(&src(
491
        "    output {\n        artifact \"junit\" {\n            path \"/src/target/junit.xml\"\n            when \"always\"\n        }\n        artifact \"tool\" {\n            path \"/src/target/tool\"\n            when \"passed\"\n        }\n    }\n",
492
    ))
493
    .unwrap();
494
    let outs = &p.jobs["build"].outputs;
495
    assert!(outs[0].always && outs[0].name == "junit");
496
    assert!(!outs[1].always);
497
    let json = serde_json::to_string(&outs[1]).unwrap();
498
    assert!(!json.contains("always"), "{json}");
499
    assert!(serde_json::to_string(&outs[0]).unwrap().contains("\"always\":true"));
500
    assert!(bad("    output {\n        artifact \"x\" {\n            path \"/src/a\"\n            when \"failed\"\n        }\n    }\n")
501
        .contains("\"passed\" (the default) or \"always\""));
502
503
    // oxygenhub#127: a report is a directory output; it shares names with
504
    // artifacts, and its kind is part of the job's hash.
505
    let as_kind = |kind: &str| {
506
        Pipeline::parse(&src(&format!("    output {{\n        {kind} \"perf\" {{\n            path \"/src/target/report\"\n            when \"always\"\n        }}\n    }}\n"))).unwrap()
507
    };
508
    let (report, artifact) = (as_kind("report"), as_kind("artifact"));
509
    let o = &report.jobs["build"].outputs[0];
510
    assert!(o.kind == OutputKind::Report && o.always && o.path == "/src/target/report");
511
    assert!(serde_json::to_string(o).unwrap().contains("\"kind\":\"report\""));
512
    assert!(!serde_json::to_string(&artifact.jobs["build"].outputs[0]).unwrap().contains("kind"));
513
    assert_ne!(report.jobs["build"].hash(sb), artifact.jobs["build"].hash(sb));
514
    assert!(bad("    output {\n        artifact \"x\" {\n            path \"/src/a\"\n        }\n        report \"x\" {\n            path \"/src/b\"\n        }\n    }\n")
515
        .contains("given twice"));
516
    assert!(bad("    output {\n        report \"x\" {\n        }\n    }\n").contains("needs a path"));
517
}
518
519
/// metalgear feedback 19 (oxygenflow#17): `\033` in a string is not KDL;
520
/// the problems say to write ESC as `\u{1b}`, which parses.
521
#[test]
522
fn ansi_escapes_are_written_as_unicode() {
523
    let bad = problems(&doc("job \"a\" {\n    sandbox \"alpine\"\n    exec \"printf\" \"\\033[32mok\\033[0m\"\n}\n"));
524
    assert!(bad.iter().any(|(line, m)| *line == 13 && m.contains("\\u{1b}")), "{bad:?}");
525
    let p = Pipeline::parse(&doc("job \"a\" {\n    sandbox \"alpine\"\n    exec \"printf\" \"\\u{1b}[32mok\\u{1b}[0m\"\n}\n")).unwrap();
526
    assert_eq!(p.jobs["a"].exec[1], "\u{1b}[32mok\u{1b}[0m");
527
}
528
529
#[test]
530
fn runs_on_asks_for_runner_labels() {
531
    let src = doc("job \"cross\" {\n    sandbox \"alpine\"\n    runs-on \"riscv64\" \"big-disk\"\n    exec \"make\"\n}\n")
532
        .replace("    network", "    runs-on \"gpu\"\n    network");
533
    let p = Pipeline::parse(&src).unwrap();
534
    let job = &p.jobs["cross"];
535
    assert_eq!(job.runs_on, vec!["big-disk", "riscv64"], "sorted");
536
    let sandbox = &p.sandboxes["alpine"];
537
    assert_eq!(sandbox.runs_on, vec!["gpu"]);
538
    assert_eq!(job.in_sandbox(sandbox).runs_on, vec!["big-disk", "gpu", "riscv64"], "with its sandbox's");
539
    // A pipeline without runs-on serialises as before, so its jobs' hashes stay.
540
    let plain = Pipeline::parse(&doc("job \"x\" {\n    sandbox \"alpine\"\n    exec \"true\"\n}\n")).unwrap();
541
    assert!(!serde_json::to_string(&plain.jobs["x"]).unwrap().contains("runs_on"));
542
    let bad = problems(&doc("job \"x\" {\n    sandbox \"alpine\"\n    runs-on \"RISCV\"\n    exec \"true\"\n}\n"));
543
    assert!(bad.iter().any(|(_, m)| m.contains("a label is lower-case")), "{bad:?}");
544
    let none = problems(&doc("job \"y\" {\n    sandbox \"alpine\"\n    runs-on\n    exec \"true\"\n}\n"));
545
    assert!(none.iter().any(|(_, m)| m.contains("needs one or more labels")), "{none:?}");
546
}
547
548
/// oxygenflow#23: `timeout` on a job, or on its sandbox for every job in
549
/// it, as a duration from a minute to a day.
550
#[test]
551
fn a_job_says_how_long_it_may_run() {
552
    use crate::pipeline::{parse_duration, Pipeline};
553
    assert_eq!(parse_duration("90m"), Some(5400));
554
    assert_eq!(parse_duration("1h30m"), Some(5400));
555
    assert_eq!(parse_duration("2h"), Some(7200));
556
    assert_eq!(parse_duration("45s"), Some(45));
557
    for bad in ["", "90", "m", "30m1h", "1h1h", "2d", "-1h"] {
558
        assert_eq!(parse_duration(bad), None, "{bad:?}");
559
    }
560
    let root = "a".repeat(64);
561
    let text = format!(
562
        "schema-version 1\nsandbox \"s\" {{\n    runtime \"microvm\"\n    rootfs \"cas://blob/{root}\"\n    memory-limit \"1GiB\"\n    cpu-limit 1\n    network \"isolated\"\n    timeout \"2h\"\n}}\njob \"long\" {{\n    sandbox \"s\"\n    timeout \"3h30m\"\n    exec \"true\"\n}}\njob \"plain\" {{\n    sandbox \"s\"\n    exec \"true\"\n}}\n"
563
    );
564
    let p = Pipeline::parse(&text).unwrap();
565
    let job = |n: &str| p.jobs[n].in_sandbox(&p.sandboxes["s"]);
566
    assert_eq!(job("long").timeout_s, Some(12600), "the job's own");
567
    assert_eq!(job("plain").timeout_s, Some(7200), "the sandbox's");
568
    let bad = text.replace("timeout \"3h30m\"", "timeout \"2d\"");
569
    assert!(Pipeline::parse(&bad).unwrap_err().to_string().contains("timeout"));
570
    let long = text.replace("timeout \"3h30m\"", "timeout \"25h\"");
571
    assert!(Pipeline::parse(&long).unwrap_err().to_string().contains("from 1m to 24h"));
572
}
573
574
/// oxygenflow#24: `on` says what starts a job, at pipeline or job level;
575
/// a job without one runs on every push and by hand, as before.
576
#[test]
577
fn on_says_what_starts_a_job() {
578
    use crate::pipeline::{Event, Pipeline};
579
    let root = "a".repeat(64);
580
    let text = format!(
581
        "schema-version 1\non {{\n    push \"main\" \"feature/*\"\n    manual\n}}\nsandbox \"s\" {{\n    runtime \"microvm\"\n    rootfs \"cas://blob/{root}\"\n    memory-limit \"1GiB\"\n    cpu-limit 1\n    network \"isolated\"\n}}\njob \"test\" {{\n    sandbox \"s\"\n    exec \"true\"\n}}\njob \"dist\" {{\n    sandbox \"s\"\n    on {{\n        tag \"v*\"\n        manual\n    }}\n    exec \"true\"\n}}\njob \"nightly\" {{\n    sandbox \"s\"\n    on {{\n        schedule \"0 3 * * *\"\n    }}\n    exec \"true\"\n}}\n"
582
    );
583
    let p = Pipeline::parse(&text).unwrap();
584
    let runs = |job: &str, e: Event| p.on_of(&p.jobs[job]).matches(e);
585
    assert!(runs("test", Event::Push("main")) && runs("test", Event::Push("feature/9-x")));
586
    assert!(!runs("test", Event::Push("develop")), "not a bookmark the pipeline names");
587
    assert!(!runs("test", Event::Tag("v1.0.0")) && runs("test", Event::Manual));
588
    assert!(runs("dist", Event::Tag("v0.3.1")) && !runs("dist", Event::Tag("nightly")));
589
    assert!(!runs("dist", Event::Push("main")) && runs("dist", Event::Manual));
590
    assert!(runs("nightly", Event::Schedule) && !runs("nightly", Event::Manual));
591
    // Without any `on`, as before.
592
    let plain = text.replace("on {\n    push \"main\" \"feature/*\"\n    manual\n}\n", "");
593
    let p = Pipeline::parse(&plain).unwrap();
594
    assert!(p.on_of(&p.jobs["test"]).matches(Event::Push("anything")) && p.on_of(&p.jobs["test"]).matches(Event::Manual));
595
    for (bad, why) in [
596
        ("on {\n}\n", "names no event"),
597
        ("on {\n    when \"x\"\n}\n", "unknown `on` event"),
598
        ("on {\n    schedule \"0 3 * *\"\n}\n", "five cron fields"),
599
    ] {
600
        let t = format!("{bad}{plain}");
601
        let e = Pipeline::parse(&t).unwrap_err().to_string();
602
        assert!(e.contains(why), "{bad:?}: {e}");
603
    }
604
}