OXYGENHUB
# c4c87f481d05 oxygenhub / crates / hub / src / flow.rs
30.5 KiB · Rust · last changed Flow: jobs that ask for runner labels go to runners with them (refs #41, part 3) 1 month ago source blame history raw
42fa9554 Andy McCall ↶ 1
//! OxygenFlow on the hub (hub spec §13.2): runs started by pushes, the
2
//! runner API, job tokens and the checks that mirror jobs. Pipelines are
3
//! parsed by `flow-core`, the same code `flow check` runs.
4
5
use crate::api::ApiError;
6
use crate::db::{self, flow as fdb};
7
use crate::repos::RepoName;
8
use crate::Shared;
9
use axum::extract::{Path, State};
10
use axum::http::{HeaderMap, StatusCode};
11
use axum::response::{IntoResponse, Response};
12
use axum::Json;
13
use flow_core::api::{Claim, ClaimRequest, Finish, LogBatch, MAX_RECORDS, MAX_RECORD_TEXT};
14
use flow_core::{Pipeline, PIPELINE_PATH};
15
use oxy_core::hash::ChangeHash;
16
17
/// Prefix of job tokens, so the wire protocol can tell them from personal tokens.
18
pub const JOB_TOKEN_PREFIX: &str = "flowjob_";
19
/// A job token is refused this long after its job started, whatever its state.
20
pub const JOB_TOKEN_MAX_AGE_MS: i64 = 2 * 3600 * 1000;
21
9008bf6c Andy McCall ↶ 22
/// Whether pushes to this repository start runs: its own setting when a
23
/// site administrator made one, else hub.toml's `[flow] repos`.
24
pub fn enabled(app: &Shared, row: &db::repos::RepoRow) -> bool {
25
    match row.flow.as_str() {
26
        "on" => true,
27
        "off" => false,
28
        _ => app.config.flow.allows(&row.owner, &row.name),
29
    }
30
}
31
42fa9554 Andy McCall ↶ 32
/// A bookmark moved by a push: start a run when the repository is allowed
33
/// and the new state has a pipeline.
34
pub async fn on_push(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, state: &str, heads: &[ChangeHash], actor: &str) {
9008bf6c Andy McCall ↶ 35
    if bookmark.starts_with("tags/") || heads.is_empty() || !enabled(app, row) {
42fa9554 Andy McCall ↶ 36
        return;
37
    }
1edac4b6 Andy McCall ↶ 38
    if let Err(e) = start_run(app, row, bookmark, state, heads, actor, "push").await {
39
        tracing::warn!("flow: starting a run of {}/{}: {e}", row.owner, row.name);
40
    }
41
}
42
43
/// "Run pipeline" (hub spec §13.2): run a bookmark's current state without
44
/// moving it. Returns the new run's number.
45
pub async fn run_bookmark(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, actor: &str) -> Result<i64, String> {
46
    if !enabled(app, row) {
47
        return Err("OxygenFlow is not enabled for this repository".into());
48
    }
49
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
50
    let b = bookmark.to_string();
51
    let found = app
52
        .repos
53
        .read(&repo, move |r, _| {
54
            let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) };
55
            Ok(Some((st, r.store().get_state(&st)?.heads)))
56
        })
57
        .await
58
        .map_err(|e| e.to_string())?;
59
    let Some((state, heads)) = found else { return Err(format!("no bookmark {bookmark}")) };
60
    if heads.is_empty() {
61
        return Err(format!("{bookmark} has no changes yet"));
62
    }
63
    start_run(app, row, bookmark, &state.to_hex(), &heads, actor, "manual")
64
        .await?
65
        .ok_or_else(|| format!("{bookmark} has no {PIPELINE_PATH}"))
66
}
67
4f692533 Andy McCall ↶ 68
/// "Run pipeline" on one committed change: the state whose only head is
69
/// `change`, whatever bookmarks hold it. Returns the new run's number.
70
pub async fn run_change(app: &Shared, row: &db::repos::RepoRow, change: &str, actor: &str) -> Result<i64, String> {
71
    if !enabled(app, row) {
72
        return Err("OxygenFlow is not enabled for this repository".into());
73
    }
74
    let h = ChangeHash::from_hex(change.trim().trim_start_matches('#')).ok_or("give the full hash of a change")?;
75
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
df15b747 Andy McCall ↶ 76
    let known = app.repos.read(&repo, move |r, _| Ok(r.store().get_change_meta(&h).is_ok())).await.map_err(|e| e.to_string())?;
4f692533 Andy McCall ↶ 77
    if !known {
78
        return Err(format!("no change {} in this repository", &change.trim()[..12.min(change.trim().len())]));
79
    }
80
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(vec![h]).encode());
81
    start_run(app, row, "", &state.to_hex(), &[h], actor, "manual")
82
        .await?
83
        .ok_or_else(|| format!("that change's state has no {PIPELINE_PATH}"))
84
}
85
1edac4b6 Andy McCall ↶ 86
/// Re-run a finished run: the same state, bookmark and pipeline, as a new
87
/// run whose checks replace the old ones. Returns the new run's number.
88
pub async fn rerun(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<i64, String> {
89
    if !enabled(app, row) {
90
        return Err("OxygenFlow is not enabled for this repository".into());
91
    }
92
    if run.live() {
93
        return Err(format!("run #{} is still {}", run.number, run.status));
94
    }
95
    let heads: Vec<ChangeHash> = run.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
96
    if heads.is_empty() {
97
        return Err("the run names no changes".into());
98
    }
99
    start_run(app, row, &run.bookmark, &run.state, &heads, actor, &format!("rerun:{}", run.number))
100
        .await?
101
        .ok_or_else(|| format!("that state has no {PIPELINE_PATH}"))
102
}
103
104
/// Start a run of the pipeline in the state `heads` make (whose state
105
/// hash is `state`), recorded on `bookmark`: for a push, "Run pipeline"
106
/// (`trigger` = `manual`) and re-runs (`rerun:N`). `actor` is the account
107
/// whose access the run's subrepo reads are checked against. Returns the
108
/// run's number, or `None` when the state has no pipeline. The caller
109
/// checks that Flow is enabled.
110
pub async fn start_run(
111
    app: &Shared,
112
    row: &db::repos::RepoRow,
113
    bookmark: &str,
114
    state: &str,
115
    heads: &[ChangeHash],
116
    actor: &str,
117
    trigger: &str,
118
) -> Result<Option<i64>, String> {
119
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
42fa9554 Andy McCall ↶ 120
    let h = heads.to_vec();
121
    let text = app
122
        .repos
123
        .read(&repo, move |r, cache| {
124
            let loaded = cache.load(r, &h)?;
125
            Ok(match loaded.snapshot.files.get(PIPELINE_PATH).map(|f| &f.content) {
126
                Some(oxy_core::state::FileContent::Text { bytes, .. }) => Some(String::from_utf8_lossy(bytes).into_owned()),
127
                _ => None,
128
            })
129
        })
1edac4b6 Andy McCall ↶ 130
        .await
131
        .map_err(|e| format!("reading the pipeline: {e}"))?;
132
    let Some(text) = text else { return Ok(None) };
42fa9554 Andy McCall ↶ 133
    let mut hex: Vec<String> = heads.iter().map(|h| h.to_hex()).collect();
134
    hex.sort();
135
    let heads_s = hex.join("+");
1edac4b6 Andy McCall ↶ 136
    let base = db::flow::NewRun { repo_id: row.id, bookmark, state, heads: &heads_s, pipeline_hash: "", actor, trigger, error: None };
137
    let id = match Pipeline::parse(&text) {
42fa9554 Andy McCall ↶ 138
        Err(problems) => {
139
            let msg = problems.to_string();
1edac4b6 Andy McCall ↶ 140
            db::flow::create_run(&app.db, db::flow::NewRun { error: Some(&msg), ..base }, &[]).await.map_err(|e| e.to_string())?
42fa9554 Andy McCall ↶ 141
        }
142
        Ok(p) => {
143
            let depths = p.depths();
144
            let jobs: Vec<fdb::NewJob> = p
145
                .order()
146
                .into_iter()
147
                .map(|j| {
148
                    let sb = &p.sandboxes[&j.sandbox];
149
                    fdb::NewJob {
150
                        name: j.name.clone(),
151
                        depth: depths[j.name.as_str()] as i64,
152
                        depends: j.depends_on.clone(),
153
                        spec: serde_json::to_string(j).unwrap_or_default(),
154
                        sandbox: serde_json::to_string(sb).unwrap_or_default(),
155
                        runtime: sb.runtime.as_str().into(),
156
                    }
157
                })
158
                .collect();
159
            let hash = hex::encode(p.hash());
1edac4b6 Andy McCall ↶ 160
            let id =
161
                db::flow::create_run(&app.db, db::flow::NewRun { pipeline_hash: &hash, ..base }, &jobs).await.map_err(|e| e.to_string())?;
162
            if let Ok(Some(run)) = fdb::run_by_id(&app.db, id).await {
42fa9554 Andy McCall ↶ 163
                for j in &jobs {
a0d4bbfb Andy McCall ↶ 164
                    report_check(app, row, state, run.number, &j.name, "queued", "waiting for a runner").await;
42fa9554 Andy McCall ↶ 165
                }
166
            }
167
            id
168
        }
169
    };
1edac4b6 Andy McCall ↶ 170
    let run = fdb::run_by_id(&app.db, id).await.map_err(|e| e.to_string())?.ok_or("the run vanished")?;
171
    Ok(Some(run.number))
42fa9554 Andy McCall ↶ 172
}
173
a0d4bbfb Andy McCall ↶ 174
/// Record a job's check; it links to the run's page by the run's number.
175
async fn report_check(app: &Shared, row: &db::repos::RepoRow, state: &str, run_number: i64, job: &str, status: &str, summary: &str) {
176
    let url = format!("{}/{}/{}/flow/{run_number}", app.config.base_url(), row.owner, row.name);
42fa9554 Andy McCall ↶ 177
    let name = format!("flow/{job}");
178
    let c = db::hooks::upsert_check(
179
        &app.db,
180
        row.id,
181
        db::hooks::CheckReport { state, name: &name, status, summary, url: &url, reporter: None, seal: "" },
182
    )
183
    .await;
184
    if let Ok(c) = c {
185
        let api = crate::api::check_of(&c);
186
        app.events.proposal(row.id, hub_api::ProposalEvent { number: 0, kind: "check".into(), check: Some(api.clone()) });
187
        crate::hooks::emit(app, row, "check", "flow", serde_json::json!({ "check": api })).await;
188
    }
189
}
190
191
/// The check status a job's status shows as.
192
fn check_status(job_status: &str) -> &'static str {
193
    match job_status {
194
        "queued" => "queued",
195
        "running" => "running",
196
        "passed" => "passed",
197
        "skipped" => "skipped",
198
        _ => "failed",
199
    }
200
}
201
b6b7e4cf Andy McCall ↶ 202
/// The runner a bearer token belongs to: a registered runner, or one
203
/// configured with a hub.toml token (tracked from its first request). A
204
/// disabled runner is refused.
205
async fn runner_auth(app: &Shared, headers: &HeaderMap) -> Result<db::runners::Runner, ApiError> {
42fa9554 Andy McCall ↶ 206
    let Some(t) = oxy_server::bearer_token(headers) else {
207
        return Err(ApiError(StatusCode::UNAUTHORIZED, "a runner token is required".into()));
b6b7e4cf Andy McCall ↶ 208
    };
209
    let hash = blake3::hash(t.as_bytes()).to_hex().to_string();
210
    let runner = match db::runners::by_token(&app.db, &hash).await? {
211
        Some(r) => r,
212
        None if app.config.flow.runner_tokens.iter().any(|x| crate::auth::mac_equal(x, t)) => {
213
            db::runners::configured(&app.db, &hash, "runner").await?
214
        }
215
        None => return Err(ApiError(StatusCode::UNAUTHORIZED, "not a runner token".into())),
42fa9554 Andy McCall ↶ 216
    };
b6b7e4cf Andy McCall ↶ 217
    if runner.state == "disabled" {
218
        return Err(ApiError(StatusCode::FORBIDDEN, format!("runner {} is disabled on this hub", runner.name)));
42fa9554 Andy McCall ↶ 219
    }
b6b7e4cf Andy McCall ↶ 220
    Ok(runner)
42fa9554 Andy McCall ↶ 221
}
222
b6b7e4cf Andy McCall ↶ 223
/// `POST /api/v1/flow/runners/register`: a one-time registration token
224
/// from Admin → Runners buys a runner credential, binding the runner's
225
/// signing identity (hub spec §13.2).
226
pub async fn register(
227
    State(app): State<Shared>,
228
    Json(req): Json<flow_core::api::Register>,
229
) -> Result<Json<flow_core::api::Registered>, ApiError> {
230
    let bytes = hex::decode(req.identity.trim()).map_err(|_| ApiError(StatusCode::BAD_REQUEST, "identity: not hex".into()))?;
231
    let identity = oxy_core::identity::Identity::decode(&bytes).map_err(|e| ApiError(StatusCode::BAD_REQUEST, format!("identity: {e}")))?;
232
    let name: String = req.name.chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')).take(64).collect();
233
    if name.is_empty() {
234
        return Err(ApiError(StatusCode::BAD_REQUEST, "give the runner a name of letters, digits, - _ or .".into()));
235
    }
236
    let credential = format!("flowrunner_{}", crate::auth::random_hex(32));
237
    let hash = blake3::hash(credential.as_bytes()).to_hex().to_string();
238
    let reg = blake3::hash(req.token.trim().as_bytes()).to_hex().to_string();
239
    let runner =
240
        db::runners::register(&app.db, &reg, &name, &hash, &identity.hash().to_hex(), &identity.name, "").await?.ok_or_else(|| {
241
            ApiError(StatusCode::FORBIDDEN, "that registration token is unknown, used or expired; create one on Admin → Runners".into())
242
        })?;
243
    db::runners::seen(&app.db, runner.id, None, Some(&req.info)).await?;
244
    tracing::info!("flow: runner {} registered (key {})", runner.name, &runner.key[..12]);
245
    Ok(Json(flow_core::api::Registered { id: runner.id, name: runner.name, credential }))
246
}
247
42fa9554 Andy McCall ↶ 248
/// `POST /api/v1/flow/claim`.
249
pub async fn claim(State(app): State<Shared>, headers: HeaderMap, Json(req): Json<ClaimRequest>) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 250
    let me = runner_auth(&app, &headers).await?;
251
    let claimed: String = req.runner.chars().filter(|c| !c.is_control()).take(64).collect();
252
    db::runners::seen(&app.db, me.id, Some(&claimed), req.info.as_ref()).await?;
253
    // A draining runner finishes what it has and takes nothing new.
254
    if me.state == "draining" {
255
        return Ok(StatusCode::NO_CONTENT.into_response());
256
    }
257
    let runner = if me.source == "registered" { me.name.clone() } else { claimed };
42fa9554 Andy McCall ↶ 258
    let secret = format!("{JOB_TOKEN_PREFIX}{}", crate::auth::random_hex(32));
76bc5a05 Andy McCall ↶ 259
    let room = fdb::Room { memory_mib: req.free_memory_mib, cpus: req.free_cpus };
c4c87f48 Andy McCall ↶ 260
    // The labels it reports now, or those it registered with.
261
    let labels: Vec<String> = match &req.info {
262
        Some(i) => i.labels.clone(),
263
        None => me.label_list().into_iter().map(String::from).collect(),
264
    };
265
    let Some(job) =
266
        fdb::claim_within(&app.db, &req.runtimes, &runner, &crate::auth::token_hash(&secret), room, &me.scope(), &labels).await?
267
    else {
42fa9554 Andy McCall ↶ 268
        return Ok(StatusCode::NO_CONTENT.into_response());
269
    };
b6b7e4cf Andy McCall ↶ 270
    db::runners::set_job_runner(&app.db, job.id, me.id).await?;
42fa9554 Andy McCall ↶ 271
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
272
    let row =
273
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
a0d4bbfb Andy McCall ↶ 274
    report_check(&app, &row, &run.state, run.number, &job.name, "running", &format!("running on {runner}")).await;
42fa9554 Andy McCall ↶ 275
    let bad = |e: serde_json::Error| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}"));
276
    let claim = Claim {
277
        job_id: job.id,
278
        run_id: run.id,
279
        repo: format!("{}/{}", row.owner, row.name),
280
        clone_url: format!("{}/{}/{}", app.config.base_url(), row.owner, row.name),
281
        token: secret,
282
        heads: run.head_list().into_iter().map(String::from).collect(),
283
        pipeline_hash: run.pipeline_hash.clone(),
284
        job: serde_json::from_str(&job.spec).map_err(bad)?,
285
        sandbox: serde_json::from_str(&job.sandbox).map_err(bad)?,
286
        timeout_s: app.config.flow.job_timeout_minutes.max(1) * 60,
fd3db644 Andy McCall ↶ 287
        subrepos: vec![],
9a065bc0 Andy McCall ↶ 288
        state: run.state.clone(),
289
        bookmark: run.bookmark.clone(),
290
        run_number: run.number,
42fa9554 Andy McCall ↶ 291
    };
fd3db644 Andy McCall ↶ 292
    let mut claim = claim;
293
    let (inputs, ids) = subrepo_inputs(&app, &row, &run).await;
294
    if !ids.is_empty() {
295
        fdb::set_also_read(&app.db, job.id, &ids).await?;
296
    }
297
    claim.subrepos = inputs;
42fa9554 Andy McCall ↶ 298
    Ok(Json(claim).into_response())
fd3db644 Andy McCall ↶ 299
}
300
25091e5c Andy McCall ↶ 301
/// The Oxygen subrepo pins in `name`'s state `heads`: (path, repository id, pinned heads).
302
async fn pins_in(
303
    app: &Shared,
304
    name: crate::repos::RepoName,
305
    heads: Vec<oxy_core::hash::ChangeHash>,
306
) -> Vec<(String, [u8; 16], Vec<oxy_core::hash::ChangeHash>)> {
307
    app.repos
fd3db644 Andy McCall ↶ 308
        .read(&name, move |repo, cache| {
309
            let loaded = cache.load(repo, &heads)?;
310
            let mut out = Vec::new();
311
            for (path, f) in &loaded.snapshot.files {
312
                if f.kind != oxy_core::objects::NodeKind::Subrepo {
313
                    continue;
314
                }
315
                if let oxy_core::state::FileContent::Opaque { blobs } = &f.content {
316
                    if let Some(b) = blobs.first() {
317
                        if let Ok(oxy_core::subrepo::Pin::Oxy { repo: id, heads, .. }) = repo
318
                            .read_blob(b)
319
                            .map_err(crate::repos::RepoError::Oxy)
320
                            .and_then(|bytes| oxy_core::subrepo::Pin::decode(&bytes).map_err(|e| crate::repos::RepoError::Oxy(e.into())))
321
                        {
322
                            out.push((path.clone(), id, heads));
323
                        }
324
                    }
325
                }
326
            }
327
            Ok(out)
328
        })
329
        .await
25091e5c Andy McCall ↶ 330
        .unwrap_or_default()
331
}
332
333
/// How deep subrepos of subrepos are followed.
334
const MAX_SUBREPO_DEPTH: usize = 3;
335
336
/// The Oxygen subrepos the run's state pins, and those their states pin in
337
/// turn (parents first, paths joined), that live on this hub and that the
338
/// run's pusher may read: what the job's workspace needs beside its own
339
/// tree. A pin the hub cannot serve is left out, with what it pins; the job
340
/// then fails on the missing files with an ordinary error.
341
async fn subrepo_inputs(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>) {
342
    let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![]) };
343
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
fd3db644 Andy McCall ↶ 344
    let pusher = db::accounts::account_by_name(&app.db, &run.actor).await.ok().flatten();
25091e5c Andy McCall ↶ 345
    let rows = db::repos::list(&app.db).await.unwrap_or_default();
fd3db644 Andy McCall ↶ 346
    let mut inputs = Vec::new();
347
    let mut ids = Vec::new();
25091e5c Andy McCall ↶ 348
    // (path prefix, repository, heads, depth), parents before children.
349
    let mut queue = std::collections::VecDeque::from([(String::new(), name, heads, 0usize)]);
350
    while let Some((prefix, name, heads, depth)) = queue.pop_front() {
351
        if depth == MAX_SUBREPO_DEPTH {
352
            continue;
353
        }
354
        for (path, id, pinned) in pins_in(app, name, heads).await {
355
            let path = if prefix.is_empty() { path } else { format!("{prefix}/{path}") };
356
            for other in &rows {
357
                let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue };
358
                let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false);
359
                if !matches {
360
                    continue;
361
                }
362
                let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten();
363
                if crate::auth::roles::allows(role, crate::auth::Access::Read) {
364
                    inputs.push(flow_core::api::SubrepoInput {
365
                        path: path.clone(),
366
                        repo: format!("{}/{}", other.owner, other.name),
367
                        clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name),
368
                        heads: pinned.iter().map(|h| h.to_hex()).collect(),
369
                    });
370
                    if !ids.contains(&other.id) {
371
                        ids.push(other.id);
372
                    }
373
                    queue.push_back((path.clone(), other_name, pinned.clone(), depth + 1));
374
                }
375
                break;
fd3db644 Andy McCall ↶ 376
            }
377
        }
378
    }
379
    (inputs, ids)
42fa9554 Andy McCall ↶ 380
}
381
382
async fn running_job(app: &Shared, id: i64) -> Result<fdb::Job, ApiError> {
383
    match fdb::job(&app.db, id).await? {
384
        Some(j) if j.status == "running" => Ok(j),
385
        Some(_) => Err(ApiError(StatusCode::CONFLICT, "this job is not running".into())),
386
        None => Err(ApiError(StatusCode::NOT_FOUND, "no such job".into())),
387
    }
388
}
389
390
/// `POST /api/v1/flow/jobs/{id}/log`.
391
pub async fn log(
392
    State(app): State<Shared>,
393
    headers: HeaderMap,
394
    Path(id): Path<i64>,
395
    Json(mut batch): Json<LogBatch>,
396
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 397
    runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 398
    let job = running_job(&app, id).await?;
399
    for r in &mut batch.records {
400
        if r.text.len() > MAX_RECORD_TEXT {
401
            let mut cut = MAX_RECORD_TEXT;
402
            while !r.text.is_char_boundary(cut) {
403
                cut -= 1;
404
            }
405
            r.text.truncate(cut);
406
        }
407
        if !matches!(r.stream.as_str(), "out" | "err" | "console" | "flow") {
408
            r.stream = "console".into();
409
        }
410
    }
411
    let kept = fdb::append_log(&app.db, job.id, &batch.records, MAX_RECORDS as i64).await?;
412
    if kept < batch.records.len() && job.log_count < MAX_RECORDS as i64 {
413
        tracing::info!("flow: job {} reached the log limit", job.id);
414
    }
415
    Ok(StatusCode::NO_CONTENT)
416
}
417
a3c8288b Andy McCall ↶ 418
/// Largest artifact the hub keeps; the runner refuses larger ones first.
419
pub const MAX_ARTIFACT_BYTES: usize = 512 << 20;
420
421
/// `POST /api/v1/flow/jobs/{id}/artifacts/{name}`: the bytes of an artifact
422
/// the job declares (OxygenFlow spec §5.2), kept as a blob of the run's
423
/// repository. Answers `{"blob"}`.
424
pub async fn artifact(
425
    State(app): State<Shared>,
426
    headers: HeaderMap,
427
    Path((id, name)): Path<(i64, String)>,
428
    body: axum::body::Bytes,
429
) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 430
    runner_auth(&app, &headers).await?;
a3c8288b Andy McCall ↶ 431
    let job = running_job(&app, id).await?;
432
    let spec: flow_core::Job =
433
        serde_json::from_str(&job.spec).map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}")))?;
434
    let Some(out) = spec.outputs.iter().find(|o| o.name == name) else {
435
        return Err(ApiError(StatusCode::BAD_REQUEST, format!("job {} declares no artifact {name:?}", job.name)));
436
    };
437
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
438
    let row =
439
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
440
    let repo = RepoName::new(&row.owner, &row.name).ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "bad repository name".into()))?;
441
    let content = blake3::hash(&body).to_hex().to_string();
442
    let size = body.len() as i64;
443
    let blob = app
444
        .repos
445
        .write(&repo, move |r, _| Ok(r.store_blob(&body)?))
446
        .await
447
        .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("keep the artifact: {e}")))?;
448
    let a = fdb::Artifact { job_id: job.id, name, path: out.path.clone(), blob: blob.to_hex(), size, content, created_ms: db::now_ms() };
449
    fdb::add_artifact(&app.db, &a).await?;
450
    Ok(Json(serde_json::json!({ "blob": a.blob })).into_response())
451
}
452
42fa9554 Andy McCall ↶ 453
/// `POST /api/v1/flow/jobs/{id}/finish`.
454
pub async fn finish(
455
    State(app): State<Shared>,
456
    headers: HeaderMap,
457
    Path(id): Path<i64>,
458
    Json(f): Json<Finish>,
459
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 460
    let me = runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 461
    let job = running_job(&app, id).await?;
462
    let status = match (f.exit_code, &f.error) {
463
        (Some(0), None) => "passed",
464
        (Some(_), _) => "failed",
465
        (None, _) => "error",
466
    };
467
    let (receipt, receipt_hash) = match &f.receipt {
468
        Some(r) if r.job_name == job.name && Some(r.exit_code as i32) == f.exit_code => {
a3c8288b Andy McCall ↶ 469
            // The receipt names exactly the artifacts uploaded, in name order.
470
            let kept: Vec<String> = fdb::artifacts(&app.db, job.id).await?.into_iter().map(|a| a.blob).collect();
471
            if r.outputs.iter().map(hex::encode).collect::<Vec<_>>() != kept {
472
                return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt names other outputs than the artifacts uploaded".into()));
473
            }
42fa9554 Andy McCall ↶ 474
            (serde_json::to_string(r).unwrap_or_default(), hex::encode(r.hash()))
475
        }
476
        Some(_) => return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt does not match the job".into())),
477
        None => (String::new(), String::new()),
478
    };
479
    let error = f.error.clone().unwrap_or_default();
480
    let changed = fdb::finish(
481
        &app.db,
482
        job.id,
483
        fdb::Outcome { status, exit_code: f.exit_code.map(i64::from), error: &error, receipt: &receipt, receipt_hash: &receipt_hash },
484
    )
485
    .await?;
486
    if let Some(run) = fdb::run_by_id(&app.db, job.run_id).await? {
487
        if let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? {
cc19697f Andy McCall ↶ 488
            if let (Some(r), Some(signed)) = (&f.receipt, &f.signed_receipt) {
b6b7e4cf Andy McCall ↶ 489
                let (attest, signer, note) = match keep_receipt(&app, &me, &row, &run, r, signed).await {
cc19697f Andy McCall ↶ 490
                    Ok((attest, signer)) => (attest, signer, String::new()),
491
                    Err(e) => {
492
                        tracing::warn!("flow: job {} receipt not kept: {e}", job.id);
493
                        (String::new(), String::new(), e)
494
                    }
495
                };
496
                fdb::set_receipt(&app.db, job.id, &attest, &signer, &note).await?;
497
            }
42fa9554 Andy McCall ↶ 498
            for j in changed {
499
                let summary = match j.status.as_str() {
500
                    "passed" => format!("passed in {}", duration(&j)),
501
                    "failed" => format!("exit {} after {}", j.exit_code.unwrap_or(-1), duration(&j)),
502
                    "skipped" => "skipped: a dependency did not pass".into(),
503
                    _ => format!("could not run: {}", j.error),
504
                };
a0d4bbfb Andy McCall ↶ 505
                report_check(&app, &row, &run.state, run.number, &j.name, check_status(&j.status), &summary).await;
42fa9554 Andy McCall ↶ 506
            }
507
        }
508
    }
509
    Ok(StatusCode::NO_CONTENT)
cc19697f Andy McCall ↶ 510
}
511
512
/// Check a runner's signed receipt for a job of `run` and keep it in the
513
/// repository (Oxygen spec §14.6): the attestation's hex and the runner's
514
/// name, or why it was not kept. It must be the job's receipt, for the
515
/// run's state and repository, signed by the key the runner's token first
516
/// signed with.
517
pub async fn keep_receipt(
518
    app: &Shared,
b6b7e4cf Andy McCall ↶ 519
    runner: &db::runners::Runner,
cc19697f Andy McCall ↶ 520
    row: &db::repos::RepoRow,
521
    run: &fdb::Run,
522
    receipt: &flow_core::Receipt,
523
    signed_hex: &str,
524
) -> Result<(String, String), String> {
525
    use oxy_core::receipt::SignedReceipt;
526
    let bytes = hex::decode(signed_hex).map_err(|_| "the signed receipt is not hex".to_string())?;
527
    let sr = SignedReceipt::decode(&bytes).map_err(|e| format!("not a signed receipt: {e}"))?;
528
    sr.verify().map_err(|e| format!("it does not verify: {e}"))?;
529
    if sr.receipt.hash().0 != receipt.hash() {
530
        return Err("it signs another receipt than the job's".into());
531
    }
532
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
533
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(heads).encode());
534
    if sr.state() != state {
535
        return Err("it is for another state than the run's".into());
536
    }
537
    let key = sr.runner.hash().to_hex();
b6b7e4cf Andy McCall ↶ 538
    let bound = db::runners::bind_key(&app.db, runner.id, &key, &sr.runner.name).await.map_err(|e| e.to_string())?;
cc19697f Andy McCall ↶ 539
    if bound != key {
540
        return Err(format!("signed by {}, not the key this runner signs with ({})", &key[..12], &bound[..12.min(bound.len())]));
541
    }
542
    let name = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
543
    let signer = sr.runner.name.clone();
544
    let attest = app.repos.write(&name, move |repo, _| Ok(repo.add_receipt(&sr)?)).await.map_err(|e| e.to_string())?;
545
    Ok((attest.to_hex(), signer))
42fa9554 Andy McCall ↶ 546
}
547
548
/// A job's wall time, as the pages show it.
549
pub fn duration(j: &fdb::Job) -> String {
550
    match (j.started_ms, j.finished_ms) {
551
        (Some(a), Some(b)) => human_ms(b - a),
552
        (Some(a), None) => human_ms(db::now_ms() - a),
553
        _ => "–".into(),
554
    }
555
}
556
557
pub fn human_ms(ms: i64) -> String {
558
    let ms = ms.max(0);
559
    if ms < 1000 {
560
        format!("{ms} ms")
561
    } else if ms < 60_000 {
562
        format!("{:.1} s", ms as f64 / 1000.0)
563
    } else {
564
        format!("{} min {} s", ms / 60_000, (ms % 60_000) / 1000)
565
    }
566
}
bf3d0754 Andy McCall ↶ 567
568
/// Where a live run stands among the runners' work: what its viewers may
569
/// know of it. Jobs of other repositories are named only when those are
570
/// public; the rest are counted.
571
#[derive(Clone, Debug, Default, PartialEq, Eq)]
572
pub struct QueueInfo {
573
    /// 1-based place of the run's first queued job among all queued jobs.
574
    pub position: Option<usize>,
575
    /// Jobs running now, and queued jobs ahead of the run's first one.
576
    pub running: Vec<String>,
577
    pub ahead: Vec<String>,
578
    /// Of those, jobs in repositories the page may not name.
579
    pub hidden: usize,
580
}
581
582
impl QueueInfo {
583
    /// "3rd in line" and the like.
584
    pub fn place(&self) -> Option<String> {
585
        self.position.map(|n| {
586
            let suffix = match (n % 10, n % 100) {
587
                (1, 11) | (2, 12) | (3, 13) => "th",
588
                (1, _) => "st",
589
                (2, _) => "nd",
590
                (3, _) => "rd",
591
                _ => "th",
592
            };
593
            format!("{n}{suffix} in line")
594
        })
595
    }
596
}
597
598
/// The queue as `run` (of repository `repo_id`) sees it, from `queue`.
599
pub fn queue_info(queue: &[fdb::QueueEntry], repo_id: i64, run_id: i64) -> QueueInfo {
600
    let mut info = QueueInfo::default();
601
    let first = queue.iter().filter(|e| e.status == "queued").position(|e| e.run_id == run_id);
602
    info.position = first.map(|i| i + 1);
603
    let label = |e: &fdb::QueueEntry| format!("{}/{} #{} {}", e.owner, e.name, e.run_number, e.job);
604
    let mut queued_seen = 0;
605
    for e in queue {
606
        if e.run_id == run_id {
607
            continue;
608
        }
609
        let ahead = match e.status.as_str() {
610
            "running" => true,
611
            _ => {
612
                queued_seen += 1;
613
                first.is_some_and(|f| queued_seen <= f)
614
            }
615
        };
616
        if !ahead {
617
            continue;
618
        }
619
        if e.repo_id != repo_id && e.visibility != "public" {
620
            info.hidden += 1;
621
        } else if e.status == "running" {
622
            info.running.push(label(e));
623
        } else {
624
            info.ahead.push(label(e));
625
        }
626
    }
627
    info
628
}
629
630
#[cfg(test)]
631
mod queue_tests {
632
    use super::*;
633
634
    fn e(job_id: i64, run_id: i64, repo_id: i64, vis: &str, job: &str, status: &str) -> fdb::QueueEntry {
635
        fdb::QueueEntry {
636
            job_id,
637
            run_id,
638
            repo_id,
639
            owner: "o".into(),
640
            name: format!("r{repo_id}"),
641
            visibility: vis.into(),
642
            run_number: run_id,
643
            job: job.into(),
644
            status: status.into(),
b66e70b7 Andy McCall ↶ 645
            runner: String::new(),
c4c87f48 Andy McCall ↶ 646
            spec: String::new(),
647
            sandbox: String::new(),
bf3d0754 Andy McCall ↶ 648
        }
649
    }
650
651
    #[test]
652
    fn a_run_learns_its_place_and_what_is_ahead_of_it() {
653
        // Running first, then queued in claim order (as `fdb::queue` returns them).
654
        let q = vec![
655
            e(1, 10, 1, "public", "build", "running"),
656
            e(2, 11, 2, "private", "test", "running"),
657
            e(3, 10, 1, "public", "test", "queued"),
658
            e(4, 12, 3, "private", "lint", "queued"),
659
            e(5, 13, 4, "public", "build", "queued"),
660
            e(6, 13, 4, "public", "test", "queued"),
661
            e(7, 14, 5, "public", "late", "queued"),
662
        ];
663
        let info = queue_info(&q, 4, 13);
664
        assert_eq!(info.position, Some(3));
665
        assert_eq!(info.place().as_deref(), Some("3rd in line"));
666
        assert_eq!(info.running, ["o/r1 #10 build"]);
667
        assert_eq!(info.ahead, ["o/r1 #10 test"], "queued ahead, not behind");
668
        assert_eq!(info.hidden, 2, "private repositories are counted, not named");
669
        // A private repository's own jobs are named to its viewers.
670
        assert_eq!(queue_info(&q, 3, 12).ahead, ["o/r1 #10 test"]);
671
        assert_eq!(queue_info(&q, 3, 12).position, Some(2));
672
        // A run with nothing queued has no place.
673
        assert_eq!(queue_info(&q, 1, 99).position, None);
674
        let ordinal = |n| QueueInfo { position: Some(n), ..Default::default() }.place().unwrap();
675
        assert_eq!(
676
            [ordinal(1), ordinal(2), ordinal(11), ordinal(22), ordinal(113)],
677
            ["1st in line", "2nd in line", "11th in line", "22nd in line", "113th in line"]
678
        );
679
    }
680
}