OXYGENHUB
◆ v0.1.0 oxygenhub / crates / hub / src / flow.rs
30.2 KiB · Rust · last changed Pin oxygen with the Linux-scale work; read change metadata alone 1 month ago source blame history raw
42fa9554 Andy McCall ↶ 1
//! OxygenFlow on the hub (hub spec §13.2): runs started by pushes, the
2
//! runner API, job tokens and the checks that mirror jobs. Pipelines are
3
//! parsed by `flow-core`, the same code `flow check` runs.
4
5
use crate::api::ApiError;
6
use crate::db::{self, flow as fdb};
7
use crate::repos::RepoName;
8
use crate::Shared;
9
use axum::extract::{Path, State};
10
use axum::http::{HeaderMap, StatusCode};
11
use axum::response::{IntoResponse, Response};
12
use axum::Json;
13
use flow_core::api::{Claim, ClaimRequest, Finish, LogBatch, MAX_RECORDS, MAX_RECORD_TEXT};
14
use flow_core::{Pipeline, PIPELINE_PATH};
15
use oxy_core::hash::ChangeHash;
16
17
/// Prefix of job tokens, so the wire protocol can tell them from personal tokens.
18
pub const JOB_TOKEN_PREFIX: &str = "flowjob_";
19
/// A job token is refused this long after its job started, whatever its state.
20
pub const JOB_TOKEN_MAX_AGE_MS: i64 = 2 * 3600 * 1000;
21
9008bf6c Andy McCall ↶ 22
/// Whether pushes to this repository start runs: its own setting when a
23
/// site administrator made one, else hub.toml's `[flow] repos`.
24
pub fn enabled(app: &Shared, row: &db::repos::RepoRow) -> bool {
25
    match row.flow.as_str() {
26
        "on" => true,
27
        "off" => false,
28
        _ => app.config.flow.allows(&row.owner, &row.name),
29
    }
30
}
31
42fa9554 Andy McCall ↶ 32
/// A bookmark moved by a push: start a run when the repository is allowed
33
/// and the new state has a pipeline.
34
pub async fn on_push(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, state: &str, heads: &[ChangeHash], actor: &str) {
9008bf6c Andy McCall ↶ 35
    if bookmark.starts_with("tags/") || heads.is_empty() || !enabled(app, row) {
42fa9554 Andy McCall ↶ 36
        return;
37
    }
1edac4b6 Andy McCall ↶ 38
    if let Err(e) = start_run(app, row, bookmark, state, heads, actor, "push").await {
39
        tracing::warn!("flow: starting a run of {}/{}: {e}", row.owner, row.name);
40
    }
41
}
42
43
/// "Run pipeline" (hub spec §13.2): run a bookmark's current state without
44
/// moving it. Returns the new run's number.
45
pub async fn run_bookmark(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, actor: &str) -> Result<i64, String> {
46
    if !enabled(app, row) {
47
        return Err("OxygenFlow is not enabled for this repository".into());
48
    }
49
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
50
    let b = bookmark.to_string();
51
    let found = app
52
        .repos
53
        .read(&repo, move |r, _| {
54
            let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) };
55
            Ok(Some((st, r.store().get_state(&st)?.heads)))
56
        })
57
        .await
58
        .map_err(|e| e.to_string())?;
59
    let Some((state, heads)) = found else { return Err(format!("no bookmark {bookmark}")) };
60
    if heads.is_empty() {
61
        return Err(format!("{bookmark} has no changes yet"));
62
    }
63
    start_run(app, row, bookmark, &state.to_hex(), &heads, actor, "manual")
64
        .await?
65
        .ok_or_else(|| format!("{bookmark} has no {PIPELINE_PATH}"))
66
}
67
4f692533 Andy McCall ↶ 68
/// "Run pipeline" on one committed change: the state whose only head is
69
/// `change`, whatever bookmarks hold it. Returns the new run's number.
70
pub async fn run_change(app: &Shared, row: &db::repos::RepoRow, change: &str, actor: &str) -> Result<i64, String> {
71
    if !enabled(app, row) {
72
        return Err("OxygenFlow is not enabled for this repository".into());
73
    }
74
    let h = ChangeHash::from_hex(change.trim().trim_start_matches('#')).ok_or("give the full hash of a change")?;
75
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
df15b747 Andy McCall ↶ 76
    let known = app.repos.read(&repo, move |r, _| Ok(r.store().get_change_meta(&h).is_ok())).await.map_err(|e| e.to_string())?;
4f692533 Andy McCall ↶ 77
    if !known {
78
        return Err(format!("no change {} in this repository", &change.trim()[..12.min(change.trim().len())]));
79
    }
80
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(vec![h]).encode());
81
    start_run(app, row, "", &state.to_hex(), &[h], actor, "manual")
82
        .await?
83
        .ok_or_else(|| format!("that change's state has no {PIPELINE_PATH}"))
84
}
85
1edac4b6 Andy McCall ↶ 86
/// Re-run a finished run: the same state, bookmark and pipeline, as a new
87
/// run whose checks replace the old ones. Returns the new run's number.
88
pub async fn rerun(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<i64, String> {
89
    if !enabled(app, row) {
90
        return Err("OxygenFlow is not enabled for this repository".into());
91
    }
92
    if run.live() {
93
        return Err(format!("run #{} is still {}", run.number, run.status));
94
    }
95
    let heads: Vec<ChangeHash> = run.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
96
    if heads.is_empty() {
97
        return Err("the run names no changes".into());
98
    }
99
    start_run(app, row, &run.bookmark, &run.state, &heads, actor, &format!("rerun:{}", run.number))
100
        .await?
101
        .ok_or_else(|| format!("that state has no {PIPELINE_PATH}"))
102
}
103
104
/// Start a run of the pipeline in the state `heads` make (whose state
105
/// hash is `state`), recorded on `bookmark`: for a push, "Run pipeline"
106
/// (`trigger` = `manual`) and re-runs (`rerun:N`). `actor` is the account
107
/// whose access the run's subrepo reads are checked against. Returns the
108
/// run's number, or `None` when the state has no pipeline. The caller
109
/// checks that Flow is enabled.
110
pub async fn start_run(
111
    app: &Shared,
112
    row: &db::repos::RepoRow,
113
    bookmark: &str,
114
    state: &str,
115
    heads: &[ChangeHash],
116
    actor: &str,
117
    trigger: &str,
118
) -> Result<Option<i64>, String> {
119
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
42fa9554 Andy McCall ↶ 120
    let h = heads.to_vec();
121
    let text = app
122
        .repos
123
        .read(&repo, move |r, cache| {
124
            let loaded = cache.load(r, &h)?;
125
            Ok(match loaded.snapshot.files.get(PIPELINE_PATH).map(|f| &f.content) {
126
                Some(oxy_core::state::FileContent::Text { bytes, .. }) => Some(String::from_utf8_lossy(bytes).into_owned()),
127
                _ => None,
128
            })
129
        })
1edac4b6 Andy McCall ↶ 130
        .await
131
        .map_err(|e| format!("reading the pipeline: {e}"))?;
132
    let Some(text) = text else { return Ok(None) };
42fa9554 Andy McCall ↶ 133
    let mut hex: Vec<String> = heads.iter().map(|h| h.to_hex()).collect();
134
    hex.sort();
135
    let heads_s = hex.join("+");
1edac4b6 Andy McCall ↶ 136
    let base = db::flow::NewRun { repo_id: row.id, bookmark, state, heads: &heads_s, pipeline_hash: "", actor, trigger, error: None };
137
    let id = match Pipeline::parse(&text) {
42fa9554 Andy McCall ↶ 138
        Err(problems) => {
139
            let msg = problems.to_string();
1edac4b6 Andy McCall ↶ 140
            db::flow::create_run(&app.db, db::flow::NewRun { error: Some(&msg), ..base }, &[]).await.map_err(|e| e.to_string())?
42fa9554 Andy McCall ↶ 141
        }
142
        Ok(p) => {
143
            let depths = p.depths();
144
            let jobs: Vec<fdb::NewJob> = p
145
                .order()
146
                .into_iter()
147
                .map(|j| {
148
                    let sb = &p.sandboxes[&j.sandbox];
149
                    fdb::NewJob {
150
                        name: j.name.clone(),
151
                        depth: depths[j.name.as_str()] as i64,
152
                        depends: j.depends_on.clone(),
153
                        spec: serde_json::to_string(j).unwrap_or_default(),
154
                        sandbox: serde_json::to_string(sb).unwrap_or_default(),
155
                        runtime: sb.runtime.as_str().into(),
156
                    }
157
                })
158
                .collect();
159
            let hash = hex::encode(p.hash());
1edac4b6 Andy McCall ↶ 160
            let id =
161
                db::flow::create_run(&app.db, db::flow::NewRun { pipeline_hash: &hash, ..base }, &jobs).await.map_err(|e| e.to_string())?;
162
            if let Ok(Some(run)) = fdb::run_by_id(&app.db, id).await {
42fa9554 Andy McCall ↶ 163
                for j in &jobs {
a0d4bbfb Andy McCall ↶ 164
                    report_check(app, row, state, run.number, &j.name, "queued", "waiting for a runner").await;
42fa9554 Andy McCall ↶ 165
                }
166
            }
167
            id
168
        }
169
    };
1edac4b6 Andy McCall ↶ 170
    let run = fdb::run_by_id(&app.db, id).await.map_err(|e| e.to_string())?.ok_or("the run vanished")?;
171
    Ok(Some(run.number))
42fa9554 Andy McCall ↶ 172
}
173
a0d4bbfb Andy McCall ↶ 174
/// Record a job's check; it links to the run's page by the run's number.
175
async fn report_check(app: &Shared, row: &db::repos::RepoRow, state: &str, run_number: i64, job: &str, status: &str, summary: &str) {
176
    let url = format!("{}/{}/{}/flow/{run_number}", app.config.base_url(), row.owner, row.name);
42fa9554 Andy McCall ↶ 177
    let name = format!("flow/{job}");
178
    let c = db::hooks::upsert_check(
179
        &app.db,
180
        row.id,
181
        db::hooks::CheckReport { state, name: &name, status, summary, url: &url, reporter: None, seal: "" },
182
    )
183
    .await;
184
    if let Ok(c) = c {
185
        let api = crate::api::check_of(&c);
186
        app.events.proposal(row.id, hub_api::ProposalEvent { number: 0, kind: "check".into(), check: Some(api.clone()) });
187
        crate::hooks::emit(app, row, "check", "flow", serde_json::json!({ "check": api })).await;
188
    }
189
}
190
191
/// The check status a job's status shows as.
192
fn check_status(job_status: &str) -> &'static str {
193
    match job_status {
194
        "queued" => "queued",
195
        "running" => "running",
196
        "passed" => "passed",
197
        "skipped" => "skipped",
198
        _ => "failed",
199
    }
200
}
201
b6b7e4cf Andy McCall ↶ 202
/// The runner a bearer token belongs to: a registered runner, or one
203
/// configured with a hub.toml token (tracked from its first request). A
204
/// disabled runner is refused.
205
async fn runner_auth(app: &Shared, headers: &HeaderMap) -> Result<db::runners::Runner, ApiError> {
42fa9554 Andy McCall ↶ 206
    let Some(t) = oxy_server::bearer_token(headers) else {
207
        return Err(ApiError(StatusCode::UNAUTHORIZED, "a runner token is required".into()));
b6b7e4cf Andy McCall ↶ 208
    };
209
    let hash = blake3::hash(t.as_bytes()).to_hex().to_string();
210
    let runner = match db::runners::by_token(&app.db, &hash).await? {
211
        Some(r) => r,
212
        None if app.config.flow.runner_tokens.iter().any(|x| crate::auth::mac_equal(x, t)) => {
213
            db::runners::configured(&app.db, &hash, "runner").await?
214
        }
215
        None => return Err(ApiError(StatusCode::UNAUTHORIZED, "not a runner token".into())),
42fa9554 Andy McCall ↶ 216
    };
b6b7e4cf Andy McCall ↶ 217
    if runner.state == "disabled" {
218
        return Err(ApiError(StatusCode::FORBIDDEN, format!("runner {} is disabled on this hub", runner.name)));
42fa9554 Andy McCall ↶ 219
    }
b6b7e4cf Andy McCall ↶ 220
    Ok(runner)
42fa9554 Andy McCall ↶ 221
}
222
b6b7e4cf Andy McCall ↶ 223
/// `POST /api/v1/flow/runners/register`: a one-time registration token
224
/// from Admin → Runners buys a runner credential, binding the runner's
225
/// signing identity (hub spec §13.2).
226
pub async fn register(
227
    State(app): State<Shared>,
228
    Json(req): Json<flow_core::api::Register>,
229
) -> Result<Json<flow_core::api::Registered>, ApiError> {
230
    let bytes = hex::decode(req.identity.trim()).map_err(|_| ApiError(StatusCode::BAD_REQUEST, "identity: not hex".into()))?;
231
    let identity = oxy_core::identity::Identity::decode(&bytes).map_err(|e| ApiError(StatusCode::BAD_REQUEST, format!("identity: {e}")))?;
232
    let name: String = req.name.chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')).take(64).collect();
233
    if name.is_empty() {
234
        return Err(ApiError(StatusCode::BAD_REQUEST, "give the runner a name of letters, digits, - _ or .".into()));
235
    }
236
    let credential = format!("flowrunner_{}", crate::auth::random_hex(32));
237
    let hash = blake3::hash(credential.as_bytes()).to_hex().to_string();
238
    let reg = blake3::hash(req.token.trim().as_bytes()).to_hex().to_string();
239
    let runner =
240
        db::runners::register(&app.db, &reg, &name, &hash, &identity.hash().to_hex(), &identity.name, "").await?.ok_or_else(|| {
241
            ApiError(StatusCode::FORBIDDEN, "that registration token is unknown, used or expired; create one on Admin → Runners".into())
242
        })?;
243
    db::runners::seen(&app.db, runner.id, None, Some(&req.info)).await?;
244
    tracing::info!("flow: runner {} registered (key {})", runner.name, &runner.key[..12]);
245
    Ok(Json(flow_core::api::Registered { id: runner.id, name: runner.name, credential }))
246
}
247
42fa9554 Andy McCall ↶ 248
/// `POST /api/v1/flow/claim`.
249
pub async fn claim(State(app): State<Shared>, headers: HeaderMap, Json(req): Json<ClaimRequest>) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 250
    let me = runner_auth(&app, &headers).await?;
251
    let claimed: String = req.runner.chars().filter(|c| !c.is_control()).take(64).collect();
252
    db::runners::seen(&app.db, me.id, Some(&claimed), req.info.as_ref()).await?;
253
    // A draining runner finishes what it has and takes nothing new.
254
    if me.state == "draining" {
255
        return Ok(StatusCode::NO_CONTENT.into_response());
256
    }
257
    let runner = if me.source == "registered" { me.name.clone() } else { claimed };
42fa9554 Andy McCall ↶ 258
    let secret = format!("{JOB_TOKEN_PREFIX}{}", crate::auth::random_hex(32));
76bc5a05 Andy McCall ↶ 259
    let room = fdb::Room { memory_mib: req.free_memory_mib, cpus: req.free_cpus };
260
    let Some(job) = fdb::claim_within(&app.db, &req.runtimes, &runner, &crate::auth::token_hash(&secret), room).await? else {
42fa9554 Andy McCall ↶ 261
        return Ok(StatusCode::NO_CONTENT.into_response());
262
    };
b6b7e4cf Andy McCall ↶ 263
    db::runners::set_job_runner(&app.db, job.id, me.id).await?;
42fa9554 Andy McCall ↶ 264
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
265
    let row =
266
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
a0d4bbfb Andy McCall ↶ 267
    report_check(&app, &row, &run.state, run.number, &job.name, "running", &format!("running on {runner}")).await;
42fa9554 Andy McCall ↶ 268
    let bad = |e: serde_json::Error| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}"));
269
    let claim = Claim {
270
        job_id: job.id,
271
        run_id: run.id,
272
        repo: format!("{}/{}", row.owner, row.name),
273
        clone_url: format!("{}/{}/{}", app.config.base_url(), row.owner, row.name),
274
        token: secret,
275
        heads: run.head_list().into_iter().map(String::from).collect(),
276
        pipeline_hash: run.pipeline_hash.clone(),
277
        job: serde_json::from_str(&job.spec).map_err(bad)?,
278
        sandbox: serde_json::from_str(&job.sandbox).map_err(bad)?,
279
        timeout_s: app.config.flow.job_timeout_minutes.max(1) * 60,
fd3db644 Andy McCall ↶ 280
        subrepos: vec![],
9a065bc0 Andy McCall ↶ 281
        state: run.state.clone(),
282
        bookmark: run.bookmark.clone(),
283
        run_number: run.number,
42fa9554 Andy McCall ↶ 284
    };
fd3db644 Andy McCall ↶ 285
    let mut claim = claim;
286
    let (inputs, ids) = subrepo_inputs(&app, &row, &run).await;
287
    if !ids.is_empty() {
288
        fdb::set_also_read(&app.db, job.id, &ids).await?;
289
    }
290
    claim.subrepos = inputs;
42fa9554 Andy McCall ↶ 291
    Ok(Json(claim).into_response())
fd3db644 Andy McCall ↶ 292
}
293
25091e5c Andy McCall ↶ 294
/// The Oxygen subrepo pins in `name`'s state `heads`: (path, repository id, pinned heads).
295
async fn pins_in(
296
    app: &Shared,
297
    name: crate::repos::RepoName,
298
    heads: Vec<oxy_core::hash::ChangeHash>,
299
) -> Vec<(String, [u8; 16], Vec<oxy_core::hash::ChangeHash>)> {
300
    app.repos
fd3db644 Andy McCall ↶ 301
        .read(&name, move |repo, cache| {
302
            let loaded = cache.load(repo, &heads)?;
303
            let mut out = Vec::new();
304
            for (path, f) in &loaded.snapshot.files {
305
                if f.kind != oxy_core::objects::NodeKind::Subrepo {
306
                    continue;
307
                }
308
                if let oxy_core::state::FileContent::Opaque { blobs } = &f.content {
309
                    if let Some(b) = blobs.first() {
310
                        if let Ok(oxy_core::subrepo::Pin::Oxy { repo: id, heads, .. }) = repo
311
                            .read_blob(b)
312
                            .map_err(crate::repos::RepoError::Oxy)
313
                            .and_then(|bytes| oxy_core::subrepo::Pin::decode(&bytes).map_err(|e| crate::repos::RepoError::Oxy(e.into())))
314
                        {
315
                            out.push((path.clone(), id, heads));
316
                        }
317
                    }
318
                }
319
            }
320
            Ok(out)
321
        })
322
        .await
25091e5c Andy McCall ↶ 323
        .unwrap_or_default()
324
}
325
326
/// How deep subrepos of subrepos are followed.
327
const MAX_SUBREPO_DEPTH: usize = 3;
328
329
/// The Oxygen subrepos the run's state pins, and those their states pin in
330
/// turn (parents first, paths joined), that live on this hub and that the
331
/// run's pusher may read: what the job's workspace needs beside its own
332
/// tree. A pin the hub cannot serve is left out, with what it pins; the job
333
/// then fails on the missing files with an ordinary error.
334
async fn subrepo_inputs(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>) {
335
    let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![]) };
336
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
fd3db644 Andy McCall ↶ 337
    let pusher = db::accounts::account_by_name(&app.db, &run.actor).await.ok().flatten();
25091e5c Andy McCall ↶ 338
    let rows = db::repos::list(&app.db).await.unwrap_or_default();
fd3db644 Andy McCall ↶ 339
    let mut inputs = Vec::new();
340
    let mut ids = Vec::new();
25091e5c Andy McCall ↶ 341
    // (path prefix, repository, heads, depth), parents before children.
342
    let mut queue = std::collections::VecDeque::from([(String::new(), name, heads, 0usize)]);
343
    while let Some((prefix, name, heads, depth)) = queue.pop_front() {
344
        if depth == MAX_SUBREPO_DEPTH {
345
            continue;
346
        }
347
        for (path, id, pinned) in pins_in(app, name, heads).await {
348
            let path = if prefix.is_empty() { path } else { format!("{prefix}/{path}") };
349
            for other in &rows {
350
                let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue };
351
                let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false);
352
                if !matches {
353
                    continue;
354
                }
355
                let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten();
356
                if crate::auth::roles::allows(role, crate::auth::Access::Read) {
357
                    inputs.push(flow_core::api::SubrepoInput {
358
                        path: path.clone(),
359
                        repo: format!("{}/{}", other.owner, other.name),
360
                        clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name),
361
                        heads: pinned.iter().map(|h| h.to_hex()).collect(),
362
                    });
363
                    if !ids.contains(&other.id) {
364
                        ids.push(other.id);
365
                    }
366
                    queue.push_back((path.clone(), other_name, pinned.clone(), depth + 1));
367
                }
368
                break;
fd3db644 Andy McCall ↶ 369
            }
370
        }
371
    }
372
    (inputs, ids)
42fa9554 Andy McCall ↶ 373
}
374
375
async fn running_job(app: &Shared, id: i64) -> Result<fdb::Job, ApiError> {
376
    match fdb::job(&app.db, id).await? {
377
        Some(j) if j.status == "running" => Ok(j),
378
        Some(_) => Err(ApiError(StatusCode::CONFLICT, "this job is not running".into())),
379
        None => Err(ApiError(StatusCode::NOT_FOUND, "no such job".into())),
380
    }
381
}
382
383
/// `POST /api/v1/flow/jobs/{id}/log`.
384
pub async fn log(
385
    State(app): State<Shared>,
386
    headers: HeaderMap,
387
    Path(id): Path<i64>,
388
    Json(mut batch): Json<LogBatch>,
389
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 390
    runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 391
    let job = running_job(&app, id).await?;
392
    for r in &mut batch.records {
393
        if r.text.len() > MAX_RECORD_TEXT {
394
            let mut cut = MAX_RECORD_TEXT;
395
            while !r.text.is_char_boundary(cut) {
396
                cut -= 1;
397
            }
398
            r.text.truncate(cut);
399
        }
400
        if !matches!(r.stream.as_str(), "out" | "err" | "console" | "flow") {
401
            r.stream = "console".into();
402
        }
403
    }
404
    let kept = fdb::append_log(&app.db, job.id, &batch.records, MAX_RECORDS as i64).await?;
405
    if kept < batch.records.len() && job.log_count < MAX_RECORDS as i64 {
406
        tracing::info!("flow: job {} reached the log limit", job.id);
407
    }
408
    Ok(StatusCode::NO_CONTENT)
409
}
410
a3c8288b Andy McCall ↶ 411
/// Largest artifact the hub keeps; the runner refuses larger ones first.
412
pub const MAX_ARTIFACT_BYTES: usize = 512 << 20;
413
414
/// `POST /api/v1/flow/jobs/{id}/artifacts/{name}`: the bytes of an artifact
415
/// the job declares (OxygenFlow spec §5.2), kept as a blob of the run's
416
/// repository. Answers `{"blob"}`.
417
pub async fn artifact(
418
    State(app): State<Shared>,
419
    headers: HeaderMap,
420
    Path((id, name)): Path<(i64, String)>,
421
    body: axum::body::Bytes,
422
) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 423
    runner_auth(&app, &headers).await?;
a3c8288b Andy McCall ↶ 424
    let job = running_job(&app, id).await?;
425
    let spec: flow_core::Job =
426
        serde_json::from_str(&job.spec).map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}")))?;
427
    let Some(out) = spec.outputs.iter().find(|o| o.name == name) else {
428
        return Err(ApiError(StatusCode::BAD_REQUEST, format!("job {} declares no artifact {name:?}", job.name)));
429
    };
430
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
431
    let row =
432
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
433
    let repo = RepoName::new(&row.owner, &row.name).ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "bad repository name".into()))?;
434
    let content = blake3::hash(&body).to_hex().to_string();
435
    let size = body.len() as i64;
436
    let blob = app
437
        .repos
438
        .write(&repo, move |r, _| Ok(r.store_blob(&body)?))
439
        .await
440
        .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("keep the artifact: {e}")))?;
441
    let a = fdb::Artifact { job_id: job.id, name, path: out.path.clone(), blob: blob.to_hex(), size, content, created_ms: db::now_ms() };
442
    fdb::add_artifact(&app.db, &a).await?;
443
    Ok(Json(serde_json::json!({ "blob": a.blob })).into_response())
444
}
445
42fa9554 Andy McCall ↶ 446
/// `POST /api/v1/flow/jobs/{id}/finish`.
447
pub async fn finish(
448
    State(app): State<Shared>,
449
    headers: HeaderMap,
450
    Path(id): Path<i64>,
451
    Json(f): Json<Finish>,
452
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 453
    let me = runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 454
    let job = running_job(&app, id).await?;
455
    let status = match (f.exit_code, &f.error) {
456
        (Some(0), None) => "passed",
457
        (Some(_), _) => "failed",
458
        (None, _) => "error",
459
    };
460
    let (receipt, receipt_hash) = match &f.receipt {
461
        Some(r) if r.job_name == job.name && Some(r.exit_code as i32) == f.exit_code => {
a3c8288b Andy McCall ↶ 462
            // The receipt names exactly the artifacts uploaded, in name order.
463
            let kept: Vec<String> = fdb::artifacts(&app.db, job.id).await?.into_iter().map(|a| a.blob).collect();
464
            if r.outputs.iter().map(hex::encode).collect::<Vec<_>>() != kept {
465
                return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt names other outputs than the artifacts uploaded".into()));
466
            }
42fa9554 Andy McCall ↶ 467
            (serde_json::to_string(r).unwrap_or_default(), hex::encode(r.hash()))
468
        }
469
        Some(_) => return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt does not match the job".into())),
470
        None => (String::new(), String::new()),
471
    };
472
    let error = f.error.clone().unwrap_or_default();
473
    let changed = fdb::finish(
474
        &app.db,
475
        job.id,
476
        fdb::Outcome { status, exit_code: f.exit_code.map(i64::from), error: &error, receipt: &receipt, receipt_hash: &receipt_hash },
477
    )
478
    .await?;
479
    if let Some(run) = fdb::run_by_id(&app.db, job.run_id).await? {
480
        if let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? {
cc19697f Andy McCall ↶ 481
            if let (Some(r), Some(signed)) = (&f.receipt, &f.signed_receipt) {
b6b7e4cf Andy McCall ↶ 482
                let (attest, signer, note) = match keep_receipt(&app, &me, &row, &run, r, signed).await {
cc19697f Andy McCall ↶ 483
                    Ok((attest, signer)) => (attest, signer, String::new()),
484
                    Err(e) => {
485
                        tracing::warn!("flow: job {} receipt not kept: {e}", job.id);
486
                        (String::new(), String::new(), e)
487
                    }
488
                };
489
                fdb::set_receipt(&app.db, job.id, &attest, &signer, &note).await?;
490
            }
42fa9554 Andy McCall ↶ 491
            for j in changed {
492
                let summary = match j.status.as_str() {
493
                    "passed" => format!("passed in {}", duration(&j)),
494
                    "failed" => format!("exit {} after {}", j.exit_code.unwrap_or(-1), duration(&j)),
495
                    "skipped" => "skipped: a dependency did not pass".into(),
496
                    _ => format!("could not run: {}", j.error),
497
                };
a0d4bbfb Andy McCall ↶ 498
                report_check(&app, &row, &run.state, run.number, &j.name, check_status(&j.status), &summary).await;
42fa9554 Andy McCall ↶ 499
            }
500
        }
501
    }
502
    Ok(StatusCode::NO_CONTENT)
cc19697f Andy McCall ↶ 503
}
504
505
/// Check a runner's signed receipt for a job of `run` and keep it in the
506
/// repository (Oxygen spec §14.6): the attestation's hex and the runner's
507
/// name, or why it was not kept. It must be the job's receipt, for the
508
/// run's state and repository, signed by the key the runner's token first
509
/// signed with.
510
pub async fn keep_receipt(
511
    app: &Shared,
b6b7e4cf Andy McCall ↶ 512
    runner: &db::runners::Runner,
cc19697f Andy McCall ↶ 513
    row: &db::repos::RepoRow,
514
    run: &fdb::Run,
515
    receipt: &flow_core::Receipt,
516
    signed_hex: &str,
517
) -> Result<(String, String), String> {
518
    use oxy_core::receipt::SignedReceipt;
519
    let bytes = hex::decode(signed_hex).map_err(|_| "the signed receipt is not hex".to_string())?;
520
    let sr = SignedReceipt::decode(&bytes).map_err(|e| format!("not a signed receipt: {e}"))?;
521
    sr.verify().map_err(|e| format!("it does not verify: {e}"))?;
522
    if sr.receipt.hash().0 != receipt.hash() {
523
        return Err("it signs another receipt than the job's".into());
524
    }
525
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
526
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(heads).encode());
527
    if sr.state() != state {
528
        return Err("it is for another state than the run's".into());
529
    }
530
    let key = sr.runner.hash().to_hex();
b6b7e4cf Andy McCall ↶ 531
    let bound = db::runners::bind_key(&app.db, runner.id, &key, &sr.runner.name).await.map_err(|e| e.to_string())?;
cc19697f Andy McCall ↶ 532
    if bound != key {
533
        return Err(format!("signed by {}, not the key this runner signs with ({})", &key[..12], &bound[..12.min(bound.len())]));
534
    }
535
    let name = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
536
    let signer = sr.runner.name.clone();
537
    let attest = app.repos.write(&name, move |repo, _| Ok(repo.add_receipt(&sr)?)).await.map_err(|e| e.to_string())?;
538
    Ok((attest.to_hex(), signer))
42fa9554 Andy McCall ↶ 539
}
540
541
/// A job's wall time, as the pages show it.
542
pub fn duration(j: &fdb::Job) -> String {
543
    match (j.started_ms, j.finished_ms) {
544
        (Some(a), Some(b)) => human_ms(b - a),
545
        (Some(a), None) => human_ms(db::now_ms() - a),
546
        _ => "–".into(),
547
    }
548
}
549
550
pub fn human_ms(ms: i64) -> String {
551
    let ms = ms.max(0);
552
    if ms < 1000 {
553
        format!("{ms} ms")
554
    } else if ms < 60_000 {
555
        format!("{:.1} s", ms as f64 / 1000.0)
556
    } else {
557
        format!("{} min {} s", ms / 60_000, (ms % 60_000) / 1000)
558
    }
559
}
bf3d0754 Andy McCall ↶ 560
561
/// Where a live run stands among the runners' work: what its viewers may
562
/// know of it. Jobs of other repositories are named only when those are
563
/// public; the rest are counted.
564
#[derive(Clone, Debug, Default, PartialEq, Eq)]
565
pub struct QueueInfo {
566
    /// 1-based place of the run's first queued job among all queued jobs.
567
    pub position: Option<usize>,
568
    /// Jobs running now, and queued jobs ahead of the run's first one.
569
    pub running: Vec<String>,
570
    pub ahead: Vec<String>,
571
    /// Of those, jobs in repositories the page may not name.
572
    pub hidden: usize,
573
}
574
575
impl QueueInfo {
576
    /// "3rd in line" and the like.
577
    pub fn place(&self) -> Option<String> {
578
        self.position.map(|n| {
579
            let suffix = match (n % 10, n % 100) {
580
                (1, 11) | (2, 12) | (3, 13) => "th",
581
                (1, _) => "st",
582
                (2, _) => "nd",
583
                (3, _) => "rd",
584
                _ => "th",
585
            };
586
            format!("{n}{suffix} in line")
587
        })
588
    }
589
}
590
591
/// The queue as `run` (of repository `repo_id`) sees it, from `queue`.
592
pub fn queue_info(queue: &[fdb::QueueEntry], repo_id: i64, run_id: i64) -> QueueInfo {
593
    let mut info = QueueInfo::default();
594
    let first = queue.iter().filter(|e| e.status == "queued").position(|e| e.run_id == run_id);
595
    info.position = first.map(|i| i + 1);
596
    let label = |e: &fdb::QueueEntry| format!("{}/{} #{} {}", e.owner, e.name, e.run_number, e.job);
597
    let mut queued_seen = 0;
598
    for e in queue {
599
        if e.run_id == run_id {
600
            continue;
601
        }
602
        let ahead = match e.status.as_str() {
603
            "running" => true,
604
            _ => {
605
                queued_seen += 1;
606
                first.is_some_and(|f| queued_seen <= f)
607
            }
608
        };
609
        if !ahead {
610
            continue;
611
        }
612
        if e.repo_id != repo_id && e.visibility != "public" {
613
            info.hidden += 1;
614
        } else if e.status == "running" {
615
            info.running.push(label(e));
616
        } else {
617
            info.ahead.push(label(e));
618
        }
619
    }
620
    info
621
}
622
623
#[cfg(test)]
624
mod queue_tests {
625
    use super::*;
626
627
    fn e(job_id: i64, run_id: i64, repo_id: i64, vis: &str, job: &str, status: &str) -> fdb::QueueEntry {
628
        fdb::QueueEntry {
629
            job_id,
630
            run_id,
631
            repo_id,
632
            owner: "o".into(),
633
            name: format!("r{repo_id}"),
634
            visibility: vis.into(),
635
            run_number: run_id,
636
            job: job.into(),
637
            status: status.into(),
638
        }
639
    }
640
641
    #[test]
642
    fn a_run_learns_its_place_and_what_is_ahead_of_it() {
643
        // Running first, then queued in claim order (as `fdb::queue` returns them).
644
        let q = vec![
645
            e(1, 10, 1, "public", "build", "running"),
646
            e(2, 11, 2, "private", "test", "running"),
647
            e(3, 10, 1, "public", "test", "queued"),
648
            e(4, 12, 3, "private", "lint", "queued"),
649
            e(5, 13, 4, "public", "build", "queued"),
650
            e(6, 13, 4, "public", "test", "queued"),
651
            e(7, 14, 5, "public", "late", "queued"),
652
        ];
653
        let info = queue_info(&q, 4, 13);
654
        assert_eq!(info.position, Some(3));
655
        assert_eq!(info.place().as_deref(), Some("3rd in line"));
656
        assert_eq!(info.running, ["o/r1 #10 build"]);
657
        assert_eq!(info.ahead, ["o/r1 #10 test"], "queued ahead, not behind");
658
        assert_eq!(info.hidden, 2, "private repositories are counted, not named");
659
        // A private repository's own jobs are named to its viewers.
660
        assert_eq!(queue_info(&q, 3, 12).ahead, ["o/r1 #10 test"]);
661
        assert_eq!(queue_info(&q, 3, 12).position, Some(2));
662
        // A run with nothing queued has no place.
663
        assert_eq!(queue_info(&q, 1, 99).position, None);
664
        let ordinal = |n| QueueInfo { position: Some(n), ..Default::default() }.place().unwrap();
665
        assert_eq!(
666
            [ordinal(1), ordinal(2), ordinal(11), ordinal(22), ordinal(113)],
667
            ["1st in line", "2nd in line", "11th in line", "22nd in line", "113th in line"]
668
        );
669
    }
670
}