OXYGENHUB
◆ main oxygenhub / .flow / pipeline.kdl
9.0 KiB · last changed New repository page with board and starting files; assetlinks.json; hung tests stopped; changelog on the WASM tier (closes #360, closes #365, closes #370) 1 day ago source blame history raw
1
// OxygenHub's pipeline (OxygenFlow, docs in the oxygenflow repository).
2
// Every push to a bookmark on OxygenHub runs it in Firecracker microVMs on
3
// Earth; the jobs appear as checks on the change. oxygen and oxygenflow come
4
// from the pinned subrepos in deps/, which the runner puts in the workspace.
5
schema-version 1
6
7
// Cheap checks on the WASM tier (oxygenflow#96): OxygenFlow's checks module,
8
// copied from oxygenflow/.flow/checks.wasm (built there from
9
// crates/flow-checks; copy it again when it changes). No image and no KVM,
10
// so any runner with the WASM tier takes it: a phone, plex, windows-1.
11
sandbox "wasm" {
12
    runtime "wasm"
13
    memory-limit "64MiB"
14
}
15
16
// The changelog's shape, and a section for the version Cargo.toml says, so a
17
// release never finds its notes missing.
18
job "changelog" {
19
    sandbox "wasm"
20
    scope "CHANGELOG.md" "Cargo.toml" ".flow/checks.wasm"
21
    exec ".flow/checks.wasm" "changelog" "CHANGELOG.md" "Cargo.toml"
22
}
23
24
// The OxygenFlow build image: Rust with rustfmt and clippy, Node, every
25
// crate of Cargo.lock vendored, and this workspace's dependencies prebuilt
26
// in /opt/warm/oxygenhub, which .flow/warm-target.sh copies onto a job's
27
// target disk (oxygenflow/images/rust). Rebuild and re-pin it when
28
// Cargo.lock changes; until then only the changed crates are compiled.
29
sandbox "rust" {
30
    runtime "microvm"
31
    rootfs "cas://blob/836461b613019f62399a960c61a25c482eb74e30054ff2cd44d5594c3305872b"
32
    memory-limit "8GiB"
33
    cpu-limit 4
34
    // Tests serve and connect on 127.0.0.1; nothing leaves the guest.
35
    network "loopback"
36
    // CI builds are thrown away: no incremental state, and no debug info
37
    // (the release profile keeps line tables for the binary Earth runs,
38
    // which is built outside CI), so less to compile, link and write.
39
    env {
40
        CARGO_INCREMENTAL "0"
41
        CARGO_PROFILE_DEV_DEBUG "0"
42
        CARGO_PROFILE_TEST_DEBUG "0"
43
        CARGO_PROFILE_RELEASE_DEBUG "0"
44
        // One build identity for every CI state, so a cached target stays
45
        // fresh from one push to the next (oxygenflow#52); dist sets the
46
        // release's own.
47
        OXY_BUILD "ci"
48
    }
49
}
50
51
sandbox "rust-small" {
52
    runtime "microvm"
53
    rootfs "cas://blob/836461b613019f62399a960c61a25c482eb74e30054ff2cd44d5594c3305872b"
54
    memory-limit "1GiB"
55
    cpu-limit 1
56
    network "isolated"
57
}
58
59
// The riscv64 build image on narvi-1 (oxygenflow#69): Ubuntu 24.04 riscv64
60
// with Rust 1.97.1 and the same vendored crates, in the syscall tier (the
61
// board has no hardware virtualisation, so no microVM). `runs-on` keeps it
62
// on a RISC-V runner.
63
sandbox "rust-riscv64" {
64
    runtime "syscall"
65
    rootfs "cas://blob/b4edba9173ddf63aa1df46222e6a59b265071b19f2815c89d0153b593835c616"
66
    runs-on "riscv64"
67
    memory-limit "4GiB"
68
    cpu-limit 6
69
    network "loopback"
70
    env {
71
        CARGO_INCREMENTAL "0"
72
        CARGO_PROFILE_RELEASE_DEBUG "0"
73
        CARGO_BUILD_JOBS "4"
74
    }
75
}
76
77
job "fmt" {
78
    sandbox "rust-small"
79
    // Only this workspace's packages: --all would also check the path
80
    // dependencies in deps/, which follow their own formatting.
81
    exec "cargo" "fmt" "--check" "-p" "oxygenhub" "-p" "hub-api" "-p" "oh"
82
}
83
84
job "clippy" {
85
    sandbox "rust"
86
    depends-on "fmt"
87
    mount {
88
        source "ephemeral://disk"
89
        target "/src/target"
90
        size "24GiB"
91
        // Kept from this job's last passing run (oxygenflow#28): cargo
92
        // builds only what changed since.
93
        cache "target"
94
        key "Cargo.lock"
95
    }
96
    exec "sh" ".flow/warm-target.sh" "cargo" "clippy" "--workspace" "--all-targets" "--locked" "--" "-D" "warnings"
97
}
98
99
job "test" {
100
    sandbox "rust"
101
    depends-on "fmt"
102
    mount {
103
        source "ephemeral://disk"
104
        target "/src/target"
105
        // 40 GiB (2026-10-20), as test-postgres: its kept target filled 24 GiB.
106
        size "40GiB"
107
        // Kept from this job's last passing run (oxygenflow#28): cargo
108
        // builds only what changed since.
109
        cache "target"
110
        key "Cargo.lock"
111
    }
112
    // Under cargo-nextest (#307): a JUnit report for the hub's Tests tab,
113
    // kept whether or not the tests pass.
114
    exec "sh" ".flow/warm-target.sh" "cargo" "nextest" "run" "--workspace" "--locked" "--profile" "ci"
115
    output {
116
        artifact "junit" {
117
            path "/src/target/nextest/ci/junit.xml"
118
            when "always"
119
        }
120
    }
121
}
122
123
// Line coverage (#307) for the Coverage tab: the tests again, instrumented,
124
// in a job of its own beside `test`, so neither waits for the other and the
125
// instrumented build keeps its own cached target. Not a required check.
126
job "coverage" {
127
    sandbox "rust"
128
    depends-on "fmt"
129
    mount {
130
        source "ephemeral://disk"
131
        target "/src/target"
132
        size "48GiB"
133
        cache "coverage-target"
134
        key "Cargo.lock"
135
    }
136
    exec "cargo" "llvm-cov" "nextest" "--workspace" "--locked" "--profile" "ci" "--lcov" "--output-path" "/src/target/lcov.info"
137
    output {
138
        artifact "coverage" {
139
            path "/src/target/lcov.info"
140
        }
141
    }
142
}
143
144
// The hub's tests again on PostgreSQL (#211, docs/postgres.md), against a
145
// server the job starts in its own guest.
146
job "test-postgres" {
147
    sandbox "rust"
148
    depends-on "fmt"
149
    mount {
150
        source "ephemeral://disk"
151
        target "/src/target"
152
        // 40 GiB (2026-10-20): its kept target holds the postgres-feature test
153
        // binaries beside older ones, and filled 24 GiB (the linker then dies
154
        // with SIGBUS writing its mmapped output).
155
        size "40GiB"
156
        cache "target"
157
        key "Cargo.lock"
158
    }
159
    exec "sh" ".flow/test-postgres.sh"
160
}
161
162
job "build" {
163
    sandbox "rust"
164
    depends-on "fmt"
165
    mount {
166
        source "ephemeral://disk"
167
        target "/src/target"
168
        size "24GiB"
169
        // Kept from this job's last passing run (oxygenflow#28): cargo
170
        // builds only what changed since.
171
        cache "target"
172
        key "Cargo.lock"
173
    }
174
    exec "sh" ".flow/warm-target.sh" "cargo" "build" "--release" "--locked"
175
}
176
177
// Release files for every platform (oxygen#59): scripts/dist.sh in the
178
// build image's cross toolchains. A push runs it only when it touches
179
// these paths (a release moves Cargo.toml's version); "Run pipeline" and
180
// reruns always do. The files are kept as this run's artifacts, named by
181
// its signed receipt, and attached to the release from there, so the
182
// release's trust page shows each as built here.
183
job "dist" {
184
    sandbox "rust"
185
    depends-on "fmt"
186
    scope "Cargo.toml" "scripts/dist.sh" "packaging/**"
187
    mount {
188
        source "ephemeral://disk"
189
        target "/src/target"
190
        size "24GiB"
191
        // Kept from this job's last passing run (oxygenflow#28): cargo
192
        // builds only what changed since.
193
        cache "target"
194
        key "Cargo.lock"
195
    }
196
    env {
197
        DIST_FLAT "/src/target/dist-flat"
198
        ZIG_GLOBAL_CACHE_DIR "/src/target/zig-cache"
199
    }
200
    exec "bash" "scripts/dist.sh"
201
    output {
202
        artifact "oh-linux-x86_64.tar.gz" {
203
            path "/src/target/dist-flat/oh-linux-x86_64.tar.gz"
204
        }
205
        artifact "oh-linux-arm64.tar.gz" {
206
            path "/src/target/dist-flat/oh-linux-arm64.tar.gz"
207
        }
208
        artifact "oh-linux-riscv64.tar.gz" {
209
            path "/src/target/dist-flat/oh-linux-riscv64.tar.gz"
210
        }
211
        artifact "oh-macos-x86_64.tar.gz" {
212
            path "/src/target/dist-flat/oh-macos-x86_64.tar.gz"
213
        }
214
        artifact "oh-macos-arm64.tar.gz" {
215
            path "/src/target/dist-flat/oh-macos-arm64.tar.gz"
216
        }
217
        artifact "oh-windows-x86_64.zip" {
218
            path "/src/target/dist-flat/oh-windows-x86_64.zip"
219
        }
220
        artifact "oh_amd64.deb" {
221
            path "/src/target/dist-flat/oh_amd64.deb"
222
        }
223
        artifact "oh_arm64.deb" {
224
            path "/src/target/dist-flat/oh_arm64.deb"
225
        }
226
        artifact "oh_riscv64.deb" {
227
            path "/src/target/dist-flat/oh_riscv64.deb"
228
        }
229
        artifact "oxygenhub-server-linux-x86_64.tar.gz" {
230
            path "/src/target/dist-flat/oxygenhub-server-linux-x86_64.tar.gz"
231
        }
232
        artifact "oh-src.tar.gz" {
233
            path "/src/target/dist-flat/oh-src.tar.gz"
234
        }
235
        artifact "SHA256SUMS" {
236
            path "/src/target/dist-flat/SHA256SUMS"
237
        }
238
    }
239
}
240
241
// The native RISC-V check (oxygenflow#69): oh built and run on narvi-1, a
242
// real riscv64 board, so each release is known to run there and not only
243
// to cross-compile (dist). Like dist, a push runs it only when it touches
244
// these paths; "Run pipeline" and reruns always do. Its target directory
245
// is cached between passing runs (oxygenflow#70).
246
job "riscv64" {
247
    sandbox "rust-riscv64"
248
    depends-on "fmt"
249
    scope "Cargo.toml" "scripts/dist.sh" "packaging/**" ".flow/riscv64-check.sh"
250
    timeout "3h"
251
    mount {
252
        source "ephemeral://disk"
253
        target "/src/target"
254
        size "16GiB"
255
        cache "target"
256
        key "Cargo.lock"
257
    }
258
    exec "sh" ".flow/riscv64-check.sh"
259
    output {
260
        artifact "oh-linux-riscv64-native" {
261
            path "/src/target/release/oh"
262
        }
263
    }
264
}