OXYGENHUB
# d4792b4a4629 oxygenhub / crates / hub / src / flow.rs
42.4 KiB · Rust · last changed Cancel runs from oh, the run page, and newer pushes 1 month ago source blame history raw
42fa9554 Andy McCall ↶ 1
//! OxygenFlow on the hub (hub spec §13.2): runs started by pushes, the
2
//! runner API, job tokens and the checks that mirror jobs. Pipelines are
3
//! parsed by `flow-core`, the same code `flow check` runs.
4
5
use crate::api::ApiError;
6
use crate::db::{self, flow as fdb};
7
use crate::repos::RepoName;
8
use crate::Shared;
9
use axum::extract::{Path, State};
10
use axum::http::{HeaderMap, StatusCode};
11
use axum::response::{IntoResponse, Response};
12
use axum::Json;
13
use flow_core::api::{Claim, ClaimRequest, Finish, LogBatch, MAX_RECORDS, MAX_RECORD_TEXT};
14
use flow_core::{Pipeline, PIPELINE_PATH};
15
use oxy_core::hash::ChangeHash;
16
17
/// Prefix of job tokens, so the wire protocol can tell them from personal tokens.
18
pub const JOB_TOKEN_PREFIX: &str = "flowjob_";
19
/// A job token is refused this long after its job started, whatever its state.
20
pub const JOB_TOKEN_MAX_AGE_MS: i64 = 2 * 3600 * 1000;
21
9008bf6c Andy McCall ↶ 22
/// Whether pushes to this repository start runs: its own setting when a
23
/// site administrator made one, else hub.toml's `[flow] repos`.
24
pub fn enabled(app: &Shared, row: &db::repos::RepoRow) -> bool {
25
    match row.flow.as_str() {
26
        "on" => true,
27
        "off" => false,
28
        _ => app.config.flow.allows(&row.owner, &row.name),
29
    }
30
}
31
42fa9554 Andy McCall ↶ 32
/// A bookmark moved by a push: start a run when the repository is allowed
33
/// and the new state has a pipeline.
34
pub async fn on_push(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, state: &str, heads: &[ChangeHash], actor: &str) {
c889911b Andy McCall ↶ 35
    if heads.is_empty() || !enabled(app, row) {
42fa9554 Andy McCall ↶ 36
        return;
37
    }
c889911b Andy McCall ↶ 38
    // A pushed tag runs the jobs whose `on` names it (oxygenflow#24).
39
    let started = match bookmark.strip_prefix("tags/") {
40
        Some(tag) => start_run_with(app, row, bookmark, state, heads, actor, &format!("tag:{tag}"), None).await,
41
        None => {
42
            let touched = touched_since_last_run(app, row, bookmark, heads).await;
745d225d Andy McCall ↶ 43
            let started = start_run_scoped(app, row, bookmark, state, heads, actor, "push", touched.as_deref()).await;
44
            if let Ok(Some(n)) = started {
45
                supersede(app, row, bookmark, n).await;
46
            }
47
            started
c889911b Andy McCall ↶ 48
        }
49
    };
50
    if let Err(e) = started {
1edac4b6 Andy McCall ↶ 51
        tracing::warn!("flow: starting a run of {}/{}: {e}", row.owner, row.name);
52
    }
53
}
54
55
/// "Run pipeline" (hub spec §13.2): run a bookmark's current state without
56
/// moving it. Returns the new run's number.
57
pub async fn run_bookmark(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, actor: &str) -> Result<i64, String> {
58
    if !enabled(app, row) {
59
        return Err("OxygenFlow is not enabled for this repository".into());
60
    }
61
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
62
    let b = bookmark.to_string();
63
    let found = app
64
        .repos
65
        .read(&repo, move |r, _| {
66
            let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) };
67
            Ok(Some((st, r.store().get_state(&st)?.heads)))
68
        })
69
        .await
70
        .map_err(|e| e.to_string())?;
71
    let Some((state, heads)) = found else { return Err(format!("no bookmark {bookmark}")) };
72
    if heads.is_empty() {
73
        return Err(format!("{bookmark} has no changes yet"));
74
    }
75
    start_run(app, row, bookmark, &state.to_hex(), &heads, actor, "manual")
76
        .await?
77
        .ok_or_else(|| format!("{bookmark} has no {PIPELINE_PATH}"))
78
}
79
4f692533 Andy McCall ↶ 80
/// "Run pipeline" on one committed change: the state whose only head is
81
/// `change`, whatever bookmarks hold it. Returns the new run's number.
82
pub async fn run_change(app: &Shared, row: &db::repos::RepoRow, change: &str, actor: &str) -> Result<i64, String> {
83
    if !enabled(app, row) {
84
        return Err("OxygenFlow is not enabled for this repository".into());
85
    }
86
    let h = ChangeHash::from_hex(change.trim().trim_start_matches('#')).ok_or("give the full hash of a change")?;
87
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
df15b747 Andy McCall ↶ 88
    let known = app.repos.read(&repo, move |r, _| Ok(r.store().get_change_meta(&h).is_ok())).await.map_err(|e| e.to_string())?;
4f692533 Andy McCall ↶ 89
    if !known {
90
        return Err(format!("no change {} in this repository", &change.trim()[..12.min(change.trim().len())]));
91
    }
92
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(vec![h]).encode());
93
    start_run(app, row, "", &state.to_hex(), &[h], actor, "manual")
94
        .await?
95
        .ok_or_else(|| format!("that change's state has no {PIPELINE_PATH}"))
96
}
97
1edac4b6 Andy McCall ↶ 98
/// Re-run a finished run: the same state, bookmark and pipeline, as a new
99
/// run whose checks replace the old ones. Returns the new run's number.
100
pub async fn rerun(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<i64, String> {
101
    if !enabled(app, row) {
102
        return Err("OxygenFlow is not enabled for this repository".into());
103
    }
104
    if run.live() {
105
        return Err(format!("run #{} is still {}", run.number, run.status));
106
    }
107
    let heads: Vec<ChangeHash> = run.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
108
    if heads.is_empty() {
109
        return Err("the run names no changes".into());
110
    }
c889911b Andy McCall ↶ 111
    // A rerun repeats its run's event: the jobs it had are the jobs it gets.
112
    let trigger = format!("rerun:{}:{}", run.number, original_event(run));
113
    start_run(app, row, &run.bookmark, &run.state, &heads, actor, &trigger)
1edac4b6 Andy McCall ↶ 114
        .await?
115
        .ok_or_else(|| format!("that state has no {PIPELINE_PATH}"))
745d225d Andy McCall ↶ 116
}
117
118
/// Cancel a run still going (oxygenflow#32), its checks with it.
119
pub async fn cancel(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<(), String> {
120
    if !run.live() {
121
        return Err(format!("run #{} is already {}", run.number, run.status));
122
    }
123
    cancel_with(app, run, &format!("cancelled by {actor}")).await.map_err(|e| {
124
        tracing::warn!("flow: cancelling run #{} of {}/{}: {e}", run.number, row.owner, row.name);
125
        "the run could not be cancelled".to_string()
126
    })
127
}
128
129
async fn cancel_with(app: &Shared, run: &fdb::Run, why: &str) -> sqlx::Result<()> {
130
    let changed = fdb::cancel_run(&app.db, run.id, why).await?;
131
    report_changed(app, run.id, changed).await
c889911b Andy McCall ↶ 132
}
133
745d225d Andy McCall ↶ 134
/// A push makes the push runs still going on its bookmark moot: they are
135
/// cancelled, and the new run's scope reaches back past them.
136
async fn supersede(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, number: i64) {
137
    let Ok(older) = fdb::live_push_runs_before(&app.db, row.id, bookmark, number).await else { return };
138
    for run in older {
139
        if let Err(e) = cancel_with(app, &run, &format!("superseded by run #{number}")).await {
140
            tracing::warn!("flow: superseding run #{} of {}/{}: {e}", run.number, row.owner, row.name);
141
        }
142
    }
143
}
144
c889911b Andy McCall ↶ 145
/// The event a run's trigger stands for: `push`, `tag:NAME`, `manual`,
146
/// `schedule`, and `rerun:N:EVENT` repeating EVENT.
147
fn event_of<'a>(trigger: &'a str, bookmark: &'a str) -> flow_core::pipeline::Event<'a> {
148
    use flow_core::pipeline::Event;
149
    let t = match trigger.strip_prefix("rerun:") {
150
        Some(rest) => rest.split_once(':').map(|(_, e)| e).unwrap_or("manual"),
151
        None => trigger,
152
    };
153
    if let Some(tag) = t.strip_prefix("tag:") {
154
        Event::Tag(tag)
155
    } else if t == "push" {
156
        Event::Push(bookmark)
157
    } else if t == "schedule" {
158
        Event::Schedule
159
    } else {
160
        Event::Manual
161
    }
1edac4b6 Andy McCall ↶ 162
}
163
c889911b Andy McCall ↶ 164
/// The event to repeat when rerunning `run`.
165
fn original_event(run: &fdb::Run) -> String {
166
    let t = run.trigger.as_str();
167
    match t.strip_prefix("rerun:") {
168
        Some(rest) => rest.split_once(':').map(|(_, e)| e.to_string()).unwrap_or_else(|| "manual".into()),
169
        None => t.to_string(),
170
    }
171
}
172
d7ce41ac Andy McCall ↶ 173
/// The paths the changes a push brought touch: the changes in `heads`'
174
/// state that the bookmark's previous run did not have, and the files their
175
/// edits and tree operations name, as the new state places them. `None`
176
/// (run every job) for the bookmark's first run, or when it cannot be told.
177
async fn touched_since_last_run(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, heads: &[ChangeHash]) -> Option<Vec<String>> {
178
    let last = fdb::latest_on(&app.db, row.id, bookmark).await.ok()??;
179
    let old: Vec<ChangeHash> = last.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
180
    if old.is_empty() {
181
        return None;
182
    }
183
    let repo = RepoName::new(&row.owner, &row.name)?;
184
    let new = heads.to_vec();
185
    app.repos
186
        .read(&repo, move |r, cache| {
187
            let fresh = r.negotiate(&old, &new)?;
188
            let loaded = cache.load(r, &new)?;
189
            let by_node: std::collections::HashMap<oxy_core::ids::NodeId, &String> =
190
                loaded.snapshot.files.iter().map(|(p, f)| (f.node, p)).collect();
191
            let mut paths = std::collections::BTreeSet::new();
192
            for h in fresh {
193
                let (c, _) = r.store().get_change_lazy(&h)?;
194
                let nodes = c.edits.iter().map(|e| e.node()).chain(c.tree_ops.iter().map(|o| o.node()));
195
                for n in nodes {
196
                    if let Some(p) = by_node.get(&n) {
197
                        paths.insert((*p).clone());
198
                    }
199
                }
200
            }
201
            Ok(paths.into_iter().collect::<Vec<String>>())
202
        })
203
        .await
204
        .ok()
205
}
206
207
/// Whether a job runs for a push that touched `touched`: always without a
208
/// scope or without knowing what was touched, else when a path matches.
209
fn in_scope(scope: &[String], touched: Option<&[String]>) -> bool {
210
    let Some(touched) = touched else { return true };
211
    if scope.is_empty() {
212
        return true;
213
    }
214
    let mut b = globset::GlobSetBuilder::new();
215
    for g in scope {
216
        match globset::Glob::new(g) {
217
            Ok(g) => {
218
                b.add(g);
219
            }
220
            Err(_) => return true,
221
        }
222
    }
223
    let Ok(set) = b.build() else { return true };
224
    touched.iter().any(|p| set.is_match(p))
225
}
226
227
/// [`start_run`] for a push: jobs whose `scope` the push did not touch are
228
/// left out of the run, unless a job that runs depends on them.
229
#[allow(clippy::too_many_arguments)]
230
pub async fn start_run_scoped(
231
    app: &Shared,
232
    row: &db::repos::RepoRow,
233
    bookmark: &str,
234
    state: &str,
235
    heads: &[ChangeHash],
236
    actor: &str,
237
    trigger: &str,
238
    touched: Option<&[String]>,
239
) -> Result<Option<i64>, String> {
240
    start_run_with(app, row, bookmark, state, heads, actor, trigger, touched).await
241
}
242
1edac4b6 Andy McCall ↶ 243
/// Start a run of the pipeline in the state `heads` make (whose state
244
/// hash is `state`), recorded on `bookmark`: for a push, "Run pipeline"
245
/// (`trigger` = `manual`) and re-runs (`rerun:N`). `actor` is the account
246
/// whose access the run's subrepo reads are checked against. Returns the
247
/// run's number, or `None` when the state has no pipeline. The caller
248
/// checks that Flow is enabled.
249
pub async fn start_run(
250
    app: &Shared,
251
    row: &db::repos::RepoRow,
252
    bookmark: &str,
253
    state: &str,
254
    heads: &[ChangeHash],
255
    actor: &str,
256
    trigger: &str,
257
) -> Result<Option<i64>, String> {
d7ce41ac Andy McCall ↶ 258
    start_run_with(app, row, bookmark, state, heads, actor, trigger, None).await
259
}
260
261
#[allow(clippy::too_many_arguments)]
262
async fn start_run_with(
263
    app: &Shared,
264
    row: &db::repos::RepoRow,
265
    bookmark: &str,
266
    state: &str,
267
    heads: &[ChangeHash],
268
    actor: &str,
269
    trigger: &str,
270
    touched: Option<&[String]>,
271
) -> Result<Option<i64>, String> {
1edac4b6 Andy McCall ↶ 272
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
42fa9554 Andy McCall ↶ 273
    let h = heads.to_vec();
274
    let text = app
275
        .repos
276
        .read(&repo, move |r, cache| {
277
            let loaded = cache.load(r, &h)?;
278
            Ok(match loaded.snapshot.files.get(PIPELINE_PATH).map(|f| &f.content) {
279
                Some(oxy_core::state::FileContent::Text { bytes, .. }) => Some(String::from_utf8_lossy(bytes).into_owned()),
280
                _ => None,
281
            })
282
        })
1edac4b6 Andy McCall ↶ 283
        .await
284
        .map_err(|e| format!("reading the pipeline: {e}"))?;
285
    let Some(text) = text else { return Ok(None) };
42fa9554 Andy McCall ↶ 286
    let mut hex: Vec<String> = heads.iter().map(|h| h.to_hex()).collect();
287
    hex.sort();
288
    let heads_s = hex.join("+");
1edac4b6 Andy McCall ↶ 289
    let base = db::flow::NewRun { repo_id: row.id, bookmark, state, heads: &heads_s, pipeline_hash: "", actor, trigger, error: None };
290
    let id = match Pipeline::parse(&text) {
42fa9554 Andy McCall ↶ 291
        Err(problems) => {
292
            let msg = problems.to_string();
1edac4b6 Andy McCall ↶ 293
            db::flow::create_run(&app.db, db::flow::NewRun { error: Some(&msg), ..base }, &[]).await.map_err(|e| e.to_string())?
42fa9554 Andy McCall ↶ 294
        }
295
        Ok(p) => {
296
            let depths = p.depths();
d7ce41ac Andy McCall ↶ 297
            // Jobs out of the push's scope stay out, unless a kept job
298
            // needs them.
c889911b Andy McCall ↶ 299
            // The jobs this run's event starts (`on`), out of those its
300
            // push touched (`scope`); then what they depend on.
301
            let event = event_of(trigger, bookmark);
302
            let mut keep: std::collections::HashSet<String> = p
303
                .order()
304
                .into_iter()
305
                .filter(|j| p.on_of(j).matches(event) && in_scope(&j.scope, touched))
306
                .map(|j| j.name.clone())
307
                .collect();
d7ce41ac Andy McCall ↶ 308
            loop {
309
                let needed: Vec<String> = p
310
                    .order()
311
                    .into_iter()
312
                    .filter(|j| keep.contains(&j.name))
313
                    .flat_map(|j| j.depends_on.iter().cloned())
314
                    .filter(|d| !keep.contains(d))
315
                    .collect();
316
                if needed.is_empty() {
317
                    break;
318
                }
319
                keep.extend(needed);
320
            }
42fa9554 Andy McCall ↶ 321
            let jobs: Vec<fdb::NewJob> = p
322
                .order()
323
                .into_iter()
d7ce41ac Andy McCall ↶ 324
                .filter(|j| keep.contains(&j.name))
42fa9554 Andy McCall ↶ 325
                .map(|j| {
326
                    let sb = &p.sandboxes[&j.sandbox];
327
                    fdb::NewJob {
328
                        name: j.name.clone(),
329
                        depth: depths[j.name.as_str()] as i64,
330
                        depends: j.depends_on.clone(),
331
                        spec: serde_json::to_string(j).unwrap_or_default(),
332
                        sandbox: serde_json::to_string(sb).unwrap_or_default(),
333
                        runtime: sb.runtime.as_str().into(),
334
                    }
335
                })
336
                .collect();
c889911b Andy McCall ↶ 337
            if jobs.is_empty() {
338
                // Nothing this event starts (or this push touched).
d7ce41ac Andy McCall ↶ 339
                return Ok(None);
340
            }
42fa9554 Andy McCall ↶ 341
            let hash = hex::encode(p.hash());
1edac4b6 Andy McCall ↶ 342
            let id =
343
                db::flow::create_run(&app.db, db::flow::NewRun { pipeline_hash: &hash, ..base }, &jobs).await.map_err(|e| e.to_string())?;
344
            if let Ok(Some(run)) = fdb::run_by_id(&app.db, id).await {
42fa9554 Andy McCall ↶ 345
                for j in &jobs {
a0d4bbfb Andy McCall ↶ 346
                    report_check(app, row, state, run.number, &j.name, "queued", "waiting for a runner").await;
42fa9554 Andy McCall ↶ 347
                }
348
            }
349
            id
350
        }
351
    };
1edac4b6 Andy McCall ↶ 352
    let run = fdb::run_by_id(&app.db, id).await.map_err(|e| e.to_string())?.ok_or("the run vanished")?;
353
    Ok(Some(run.number))
42fa9554 Andy McCall ↶ 354
}
355
a0d4bbfb Andy McCall ↶ 356
/// Record a job's check; it links to the run's page by the run's number.
357
async fn report_check(app: &Shared, row: &db::repos::RepoRow, state: &str, run_number: i64, job: &str, status: &str, summary: &str) {
358
    let url = format!("{}/{}/{}/flow/{run_number}", app.config.base_url(), row.owner, row.name);
42fa9554 Andy McCall ↶ 359
    let name = format!("flow/{job}");
360
    let c = db::hooks::upsert_check(
361
        &app.db,
362
        row.id,
363
        db::hooks::CheckReport { state, name: &name, status, summary, url: &url, reporter: None, seal: "" },
364
    )
365
    .await;
366
    if let Ok(c) = c {
367
        let api = crate::api::check_of(&c);
368
        app.events.proposal(row.id, hub_api::ProposalEvent { number: 0, kind: "check".into(), check: Some(api.clone()) });
369
        crate::hooks::emit(app, row, "check", "flow", serde_json::json!({ "check": api })).await;
370
    }
371
}
372
373
/// The check status a job's status shows as.
374
fn check_status(job_status: &str) -> &'static str {
375
    match job_status {
376
        "queued" => "queued",
377
        "running" => "running",
378
        "passed" => "passed",
379
        "skipped" => "skipped",
745d225d Andy McCall ↶ 380
        "cancelled" => "cancelled",
42fa9554 Andy McCall ↶ 381
        _ => "failed",
382
    }
383
}
384
b6b7e4cf Andy McCall ↶ 385
/// The runner a bearer token belongs to: a registered runner, or one
386
/// configured with a hub.toml token (tracked from its first request). A
387
/// disabled runner is refused.
388
async fn runner_auth(app: &Shared, headers: &HeaderMap) -> Result<db::runners::Runner, ApiError> {
42fa9554 Andy McCall ↶ 389
    let Some(t) = oxy_server::bearer_token(headers) else {
390
        return Err(ApiError(StatusCode::UNAUTHORIZED, "a runner token is required".into()));
b6b7e4cf Andy McCall ↶ 391
    };
392
    let hash = blake3::hash(t.as_bytes()).to_hex().to_string();
393
    let runner = match db::runners::by_token(&app.db, &hash).await? {
394
        Some(r) => r,
395
        None if app.config.flow.runner_tokens.iter().any(|x| crate::auth::mac_equal(x, t)) => {
396
            db::runners::configured(&app.db, &hash, "runner").await?
397
        }
398
        None => return Err(ApiError(StatusCode::UNAUTHORIZED, "not a runner token".into())),
42fa9554 Andy McCall ↶ 399
    };
b6b7e4cf Andy McCall ↶ 400
    if runner.state == "disabled" {
401
        return Err(ApiError(StatusCode::FORBIDDEN, format!("runner {} is disabled on this hub", runner.name)));
42fa9554 Andy McCall ↶ 402
    }
b6b7e4cf Andy McCall ↶ 403
    Ok(runner)
42fa9554 Andy McCall ↶ 404
}
405
b6b7e4cf Andy McCall ↶ 406
/// `POST /api/v1/flow/runners/register`: a one-time registration token
407
/// from Admin → Runners buys a runner credential, binding the runner's
408
/// signing identity (hub spec §13.2).
409
pub async fn register(
410
    State(app): State<Shared>,
411
    Json(req): Json<flow_core::api::Register>,
412
) -> Result<Json<flow_core::api::Registered>, ApiError> {
413
    let bytes = hex::decode(req.identity.trim()).map_err(|_| ApiError(StatusCode::BAD_REQUEST, "identity: not hex".into()))?;
414
    let identity = oxy_core::identity::Identity::decode(&bytes).map_err(|e| ApiError(StatusCode::BAD_REQUEST, format!("identity: {e}")))?;
415
    let name: String = req.name.chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')).take(64).collect();
416
    if name.is_empty() {
417
        return Err(ApiError(StatusCode::BAD_REQUEST, "give the runner a name of letters, digits, - _ or .".into()));
418
    }
419
    let credential = format!("flowrunner_{}", crate::auth::random_hex(32));
420
    let hash = blake3::hash(credential.as_bytes()).to_hex().to_string();
421
    let reg = blake3::hash(req.token.trim().as_bytes()).to_hex().to_string();
422
    let runner =
423
        db::runners::register(&app.db, &reg, &name, &hash, &identity.hash().to_hex(), &identity.name, "").await?.ok_or_else(|| {
424
            ApiError(StatusCode::FORBIDDEN, "that registration token is unknown, used or expired; create one on Admin → Runners".into())
425
        })?;
426
    db::runners::seen(&app.db, runner.id, None, Some(&req.info)).await?;
427
    tracing::info!("flow: runner {} registered (key {})", runner.name, &runner.key[..12]);
428
    Ok(Json(flow_core::api::Registered { id: runner.id, name: runner.name, credential }))
429
}
430
42fa9554 Andy McCall ↶ 431
/// `POST /api/v1/flow/claim`.
432
pub async fn claim(State(app): State<Shared>, headers: HeaderMap, Json(req): Json<ClaimRequest>) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 433
    let me = runner_auth(&app, &headers).await?;
434
    let claimed: String = req.runner.chars().filter(|c| !c.is_control()).take(64).collect();
435
    db::runners::seen(&app.db, me.id, Some(&claimed), req.info.as_ref()).await?;
eef677c4 Andy McCall ↶ 436
    if let Some(holding) = &req.holding {
437
        end_jobs_not_held(&app, &me, holding).await?;
438
    }
b6b7e4cf Andy McCall ↶ 439
    // A draining runner finishes what it has and takes nothing new.
440
    if me.state == "draining" {
441
        return Ok(StatusCode::NO_CONTENT.into_response());
442
    }
443
    let runner = if me.source == "registered" { me.name.clone() } else { claimed };
42fa9554 Andy McCall ↶ 444
    let secret = format!("{JOB_TOKEN_PREFIX}{}", crate::auth::random_hex(32));
b0612e7d Andy McCall ↶ 445
    let room = fdb::Room {
446
        memory_mib: req.free_memory_mib,
447
        cpus: req.free_cpus,
448
        images: req.images.as_ref().map(|i| i.iter().map(|h| h.to_ascii_lowercase()).collect()),
449
    };
c4c87f48 Andy McCall ↶ 450
    // The labels it reports now, or those it registered with.
451
    let labels: Vec<String> = match &req.info {
452
        Some(i) => i.labels.clone(),
453
        None => me.label_list().into_iter().map(String::from).collect(),
454
    };
455
    let Some(job) =
456
        fdb::claim_within(&app.db, &req.runtimes, &runner, &crate::auth::token_hash(&secret), room, &me.scope(), &labels).await?
457
    else {
42fa9554 Andy McCall ↶ 458
        return Ok(StatusCode::NO_CONTENT.into_response());
459
    };
b6b7e4cf Andy McCall ↶ 460
    db::runners::set_job_runner(&app.db, job.id, me.id).await?;
42fa9554 Andy McCall ↶ 461
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
462
    let row =
463
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
a0d4bbfb Andy McCall ↶ 464
    report_check(&app, &row, &run.state, run.number, &job.name, "running", &format!("running on {runner}")).await;
42fa9554 Andy McCall ↶ 465
    let bad = |e: serde_json::Error| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}"));
5dead1f4 Andy McCall ↶ 466
    // The job's limit: its own timeout, else its sandbox's, else the site
467
    // default, never past the site maximum; recorded for the reaper.
468
    let limits = limits(&app).await;
469
    let spec: flow_core::pipeline::Job = serde_json::from_str(&job.spec).map_err(bad)?;
470
    let sandbox: flow_core::pipeline::Sandbox = serde_json::from_str(&job.sandbox).map_err(bad)?;
471
    let timeout_s = spec.timeout_s.or(sandbox.timeout_s).unwrap_or(limits.default_s).min(limits.max_s).max(60);
472
    fdb::set_job_timeout(&app.db, job.id, timeout_s).await?;
42fa9554 Andy McCall ↶ 473
    let claim = Claim {
474
        job_id: job.id,
475
        run_id: run.id,
476
        repo: format!("{}/{}", row.owner, row.name),
477
        clone_url: format!("{}/{}/{}", app.config.base_url(), row.owner, row.name),
478
        token: secret,
479
        heads: run.head_list().into_iter().map(String::from).collect(),
480
        pipeline_hash: run.pipeline_hash.clone(),
5dead1f4 Andy McCall ↶ 481
        job: spec,
482
        sandbox,
483
        timeout_s,
fd3db644 Andy McCall ↶ 484
        subrepos: vec![],
9a065bc0 Andy McCall ↶ 485
        state: run.state.clone(),
486
        bookmark: run.bookmark.clone(),
487
        run_number: run.number,
42fa9554 Andy McCall ↶ 488
    };
fd3db644 Andy McCall ↶ 489
    let mut claim = claim;
490
    let (inputs, ids) = subrepo_inputs(&app, &row, &run).await;
491
    if !ids.is_empty() {
492
        fdb::set_also_read(&app.db, job.id, &ids).await?;
493
    }
494
    claim.subrepos = inputs;
42fa9554 Andy McCall ↶ 495
    Ok(Json(claim).into_response())
fd3db644 Andy McCall ↶ 496
}
497
25091e5c Andy McCall ↶ 498
/// The Oxygen subrepo pins in `name`'s state `heads`: (path, repository id, pinned heads).
499
async fn pins_in(
500
    app: &Shared,
501
    name: crate::repos::RepoName,
502
    heads: Vec<oxy_core::hash::ChangeHash>,
503
) -> Vec<(String, [u8; 16], Vec<oxy_core::hash::ChangeHash>)> {
504
    app.repos
fd3db644 Andy McCall ↶ 505
        .read(&name, move |repo, cache| {
506
            let loaded = cache.load(repo, &heads)?;
507
            let mut out = Vec::new();
508
            for (path, f) in &loaded.snapshot.files {
509
                if f.kind != oxy_core::objects::NodeKind::Subrepo {
510
                    continue;
511
                }
512
                if let oxy_core::state::FileContent::Opaque { blobs } = &f.content {
513
                    if let Some(b) = blobs.first() {
514
                        if let Ok(oxy_core::subrepo::Pin::Oxy { repo: id, heads, .. }) = repo
515
                            .read_blob(b)
516
                            .map_err(crate::repos::RepoError::Oxy)
517
                            .and_then(|bytes| oxy_core::subrepo::Pin::decode(&bytes).map_err(|e| crate::repos::RepoError::Oxy(e.into())))
518
                        {
519
                            out.push((path.clone(), id, heads));
520
                        }
521
                    }
522
                }
523
            }
524
            Ok(out)
525
        })
526
        .await
25091e5c Andy McCall ↶ 527
        .unwrap_or_default()
528
}
529
530
/// How deep subrepos of subrepos are followed.
531
const MAX_SUBREPO_DEPTH: usize = 3;
532
533
/// The Oxygen subrepos the run's state pins, and those their states pin in
534
/// turn (parents first, paths joined), that live on this hub and that the
535
/// run's pusher may read: what the job's workspace needs beside its own
536
/// tree. A pin the hub cannot serve is left out, with what it pins; the job
537
/// then fails on the missing files with an ordinary error.
538
async fn subrepo_inputs(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>) {
539
    let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![]) };
540
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
fd3db644 Andy McCall ↶ 541
    let pusher = db::accounts::account_by_name(&app.db, &run.actor).await.ok().flatten();
25091e5c Andy McCall ↶ 542
    let rows = db::repos::list(&app.db).await.unwrap_or_default();
fd3db644 Andy McCall ↶ 543
    let mut inputs = Vec::new();
544
    let mut ids = Vec::new();
25091e5c Andy McCall ↶ 545
    // (path prefix, repository, heads, depth), parents before children.
546
    let mut queue = std::collections::VecDeque::from([(String::new(), name, heads, 0usize)]);
547
    while let Some((prefix, name, heads, depth)) = queue.pop_front() {
548
        if depth == MAX_SUBREPO_DEPTH {
549
            continue;
550
        }
551
        for (path, id, pinned) in pins_in(app, name, heads).await {
552
            let path = if prefix.is_empty() { path } else { format!("{prefix}/{path}") };
553
            for other in &rows {
554
                let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue };
555
                let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false);
556
                if !matches {
557
                    continue;
558
                }
559
                let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten();
560
                if crate::auth::roles::allows(role, crate::auth::Access::Read) {
561
                    inputs.push(flow_core::api::SubrepoInput {
562
                        path: path.clone(),
563
                        repo: format!("{}/{}", other.owner, other.name),
564
                        clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name),
565
                        heads: pinned.iter().map(|h| h.to_hex()).collect(),
566
                    });
567
                    if !ids.contains(&other.id) {
568
                        ids.push(other.id);
569
                    }
570
                    queue.push_back((path.clone(), other_name, pinned.clone(), depth + 1));
571
                }
572
                break;
fd3db644 Andy McCall ↶ 573
            }
574
        }
575
    }
576
    (inputs, ids)
42fa9554 Andy McCall ↶ 577
}
578
579
async fn running_job(app: &Shared, id: i64) -> Result<fdb::Job, ApiError> {
580
    match fdb::job(&app.db, id).await? {
581
        Some(j) if j.status == "running" => Ok(j),
582
        Some(_) => Err(ApiError(StatusCode::CONFLICT, "this job is not running".into())),
583
        None => Err(ApiError(StatusCode::NOT_FOUND, "no such job".into())),
584
    }
585
}
586
587
/// `POST /api/v1/flow/jobs/{id}/log`.
588
pub async fn log(
589
    State(app): State<Shared>,
590
    headers: HeaderMap,
591
    Path(id): Path<i64>,
592
    Json(mut batch): Json<LogBatch>,
593
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 594
    runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 595
    let job = running_job(&app, id).await?;
745d225d Andy McCall ↶ 596
    // An empty batch asks only whether the job still runs: a cancelled
597
    // one answers 409 and its runner stops it (oxygenflow#32).
598
    if batch.records.is_empty() {
599
        return Ok(StatusCode::NO_CONTENT);
600
    }
42fa9554 Andy McCall ↶ 601
    for r in &mut batch.records {
602
        if r.text.len() > MAX_RECORD_TEXT {
603
            let mut cut = MAX_RECORD_TEXT;
604
            while !r.text.is_char_boundary(cut) {
605
                cut -= 1;
606
            }
607
            r.text.truncate(cut);
608
        }
609
        if !matches!(r.stream.as_str(), "out" | "err" | "console" | "flow") {
610
            r.stream = "console".into();
611
        }
612
    }
613
    let kept = fdb::append_log(&app.db, job.id, &batch.records, MAX_RECORDS as i64).await?;
614
    if kept < batch.records.len() && job.log_count < MAX_RECORDS as i64 {
615
        tracing::info!("flow: job {} reached the log limit", job.id);
616
    }
617
    Ok(StatusCode::NO_CONTENT)
618
}
619
a3c8288b Andy McCall ↶ 620
/// Largest artifact the hub keeps; the runner refuses larger ones first.
621
pub const MAX_ARTIFACT_BYTES: usize = 512 << 20;
622
623
/// `POST /api/v1/flow/jobs/{id}/artifacts/{name}`: the bytes of an artifact
624
/// the job declares (OxygenFlow spec §5.2), kept as a blob of the run's
625
/// repository. Answers `{"blob"}`.
626
pub async fn artifact(
627
    State(app): State<Shared>,
628
    headers: HeaderMap,
629
    Path((id, name)): Path<(i64, String)>,
630
    body: axum::body::Bytes,
631
) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 632
    runner_auth(&app, &headers).await?;
a3c8288b Andy McCall ↶ 633
    let job = running_job(&app, id).await?;
634
    let spec: flow_core::Job =
635
        serde_json::from_str(&job.spec).map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}")))?;
636
    let Some(out) = spec.outputs.iter().find(|o| o.name == name) else {
637
        return Err(ApiError(StatusCode::BAD_REQUEST, format!("job {} declares no artifact {name:?}", job.name)));
638
    };
639
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
640
    let row =
641
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
642
    let repo = RepoName::new(&row.owner, &row.name).ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "bad repository name".into()))?;
643
    let content = blake3::hash(&body).to_hex().to_string();
644
    let size = body.len() as i64;
645
    let blob = app
646
        .repos
647
        .write(&repo, move |r, _| Ok(r.store_blob(&body)?))
648
        .await
649
        .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("keep the artifact: {e}")))?;
650
    let a = fdb::Artifact { job_id: job.id, name, path: out.path.clone(), blob: blob.to_hex(), size, content, created_ms: db::now_ms() };
651
    fdb::add_artifact(&app.db, &a).await?;
652
    Ok(Json(serde_json::json!({ "blob": a.blob })).into_response())
653
}
654
42fa9554 Andy McCall ↶ 655
/// `POST /api/v1/flow/jobs/{id}/finish`.
656
pub async fn finish(
657
    State(app): State<Shared>,
658
    headers: HeaderMap,
659
    Path(id): Path<i64>,
660
    Json(f): Json<Finish>,
661
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 662
    let me = runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 663
    let job = running_job(&app, id).await?;
664
    let status = match (f.exit_code, &f.error) {
665
        (Some(0), None) => "passed",
666
        (Some(_), _) => "failed",
667
        (None, _) => "error",
668
    };
669
    let (receipt, receipt_hash) = match &f.receipt {
670
        Some(r) if r.job_name == job.name && Some(r.exit_code as i32) == f.exit_code => {
a3c8288b Andy McCall ↶ 671
            // The receipt names exactly the artifacts uploaded, in name order.
672
            let kept: Vec<String> = fdb::artifacts(&app.db, job.id).await?.into_iter().map(|a| a.blob).collect();
673
            if r.outputs.iter().map(hex::encode).collect::<Vec<_>>() != kept {
674
                return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt names other outputs than the artifacts uploaded".into()));
675
            }
42fa9554 Andy McCall ↶ 676
            (serde_json::to_string(r).unwrap_or_default(), hex::encode(r.hash()))
677
        }
678
        Some(_) => return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt does not match the job".into())),
679
        None => (String::new(), String::new()),
680
    };
681
    let error = f.error.clone().unwrap_or_default();
682
    let changed = fdb::finish(
683
        &app.db,
684
        job.id,
685
        fdb::Outcome { status, exit_code: f.exit_code.map(i64::from), error: &error, receipt: &receipt, receipt_hash: &receipt_hash },
686
    )
687
    .await?;
688
    if let Some(run) = fdb::run_by_id(&app.db, job.run_id).await? {
689
        if let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? {
cc19697f Andy McCall ↶ 690
            if let (Some(r), Some(signed)) = (&f.receipt, &f.signed_receipt) {
b6b7e4cf Andy McCall ↶ 691
                let (attest, signer, note) = match keep_receipt(&app, &me, &row, &run, r, signed).await {
cc19697f Andy McCall ↶ 692
                    Ok((attest, signer)) => (attest, signer, String::new()),
693
                    Err(e) => {
694
                        tracing::warn!("flow: job {} receipt not kept: {e}", job.id);
695
                        (String::new(), String::new(), e)
696
                    }
697
                };
698
                fdb::set_receipt(&app.db, job.id, &attest, &signer, &note).await?;
699
            }
42fa9554 Andy McCall ↶ 700
        }
701
    }
070a7b64 Andy McCall ↶ 702
    report_changed(&app, job.run_id, changed).await?;
42fa9554 Andy McCall ↶ 703
    Ok(StatusCode::NO_CONTENT)
070a7b64 Andy McCall ↶ 704
}
705
706
/// Report the checks of jobs whose status changed.
707
async fn report_changed(app: &Shared, run_id: i64, changed: Vec<fdb::Job>) -> sqlx::Result<()> {
708
    let Some(run) = fdb::run_by_id(&app.db, run_id).await? else {
709
        return Ok(());
710
    };
711
    let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? else {
712
        return Ok(());
713
    };
714
    for j in changed {
715
        let summary = match j.status.as_str() {
716
            "passed" => format!("passed in {}", duration(&j)),
717
            "failed" => format!("exit {} after {}", j.exit_code.unwrap_or(-1), duration(&j)),
718
            "skipped" => "skipped: a dependency did not pass".into(),
745d225d Andy McCall ↶ 719
            "cancelled" => j.error.clone(),
070a7b64 Andy McCall ↶ 720
            _ => format!("could not run: {}", j.error),
721
        };
722
        report_check(app, &row, &run.state, run.number, &j.name, check_status(&j.status), &summary).await;
723
    }
724
    Ok(())
725
}
726
5dead1f4 Andy McCall ↶ 727
/// The site's job time limits (oxygenhub#89): set on Admin → Runners →
728
/// Settings, else from `hub.toml`.
729
pub struct Limits {
730
    /// For a job whose pipeline names none.
731
    pub default_s: u64,
732
    /// The most a pipeline may ask for.
733
    pub max_s: u64,
734
    /// Grace past a job's limit before the hub gives it up: a runner stops
735
    /// a job at its limit and reports it within moments, so one still
736
    /// running this long after is one the runner no longer reports (it was
737
    /// restarted, or lost the hub while the job ended, oxygenhub#71).
738
    pub grace_ms: i64,
739
}
740
741
pub const SETTING_DEFAULT: &str = "flow.job_timeout_minutes";
742
pub const SETTING_MAX: &str = "flow.job_timeout_max_minutes";
743
pub const SETTING_GRACE: &str = "flow.lost_grace_minutes";
744
745
pub async fn limits(app: &Shared) -> Limits {
746
    let minutes = |key: &'static str| async move {
747
        db::accounts::setting(&app.db, key).await.ok().flatten().and_then(|v| v.parse::<u64>().ok()).filter(|m| *m > 0)
748
    };
749
    let default_m = minutes(SETTING_DEFAULT).await.unwrap_or(app.config.flow.job_timeout_minutes.max(1));
750
    let max_m = minutes(SETTING_MAX).await.unwrap_or(default_m.max(6 * 60)).max(default_m);
751
    let grace_m = minutes(SETTING_GRACE).await.unwrap_or(5);
752
    Limits { default_s: default_m * 60, max_s: max_m * 60, grace_ms: grace_m as i64 * 60_000 }
753
}
070a7b64 Andy McCall ↶ 754
5dead1f4 Andy McCall ↶ 755
/// End every job still running past its time limit plus the grace:
070a7b64 Andy McCall ↶ 756
/// it is marked an error with the reason, the jobs that depended on it are
757
/// skipped and its run settled, as when a runner reports an error.
758
pub async fn reap_lost_jobs(app: &Shared) -> sqlx::Result<usize> {
5dead1f4 Andy McCall ↶ 759
    let limits = limits(app).await;
760
    let lost = fdb::running_past_limit(&app.db, db::now_ms(), limits.default_s, limits.grace_ms).await?;
070a7b64 Andy McCall ↶ 761
    let n = lost.len();
762
    for job in lost {
763
        let error = format!(
764
            "lost: still running {} minutes after it started, past the time limit; the runner{} stopped reporting it",
765
            (db::now_ms() - job.started_ms.unwrap_or(0)) / 60_000,
766
            if job.runner.is_empty() { String::new() } else { format!(" {}", job.runner) },
767
        );
768
        let changed =
769
            fdb::finish(&app.db, job.id, fdb::Outcome { status: "error", exit_code: None, error: &error, receipt: "", receipt_hash: "" })
770
                .await?;
771
        tracing::warn!("flow: job {} {error}", job.id);
772
        report_changed(app, job.run_id, changed).await?;
773
    }
774
    Ok(n)
eef677c4 Andy McCall ↶ 775
}
776
777
/// End the jobs running on `runner` that it does not hold (oxygenhub#71):
778
/// it restarted since claiming them, or never received the claim. A
779
/// runner claims one job at a time, so a job it holds is always listed.
780
async fn end_jobs_not_held(app: &Shared, runner: &db::runners::Runner, holding: &[i64]) -> sqlx::Result<()> {
781
    for job in fdb::running_on(&app.db, runner.id).await? {
782
        if holding.contains(&job.id) {
783
            continue;
784
        }
785
        let error = format!("lost: the runner {} no longer holds this job (it restarted, or never received it)", runner.name);
786
        let changed =
787
            fdb::finish(&app.db, job.id, fdb::Outcome { status: "error", exit_code: None, error: &error, receipt: "", receipt_hash: "" })
788
                .await?;
789
        tracing::warn!("flow: job {} {error}", job.id);
790
        report_changed(app, job.run_id, changed).await?;
791
    }
792
    Ok(())
070a7b64 Andy McCall ↶ 793
}
794
795
/// Look for lost jobs once a minute.
796
pub async fn reaper(app: Shared) {
797
    loop {
798
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
799
        if let Err(e) = reap_lost_jobs(&app).await {
800
            tracing::warn!("flow: looking for lost jobs: {e}");
801
        }
802
    }
cc19697f Andy McCall ↶ 803
}
804
805
/// Check a runner's signed receipt for a job of `run` and keep it in the
806
/// repository (Oxygen spec §14.6): the attestation's hex and the runner's
807
/// name, or why it was not kept. It must be the job's receipt, for the
808
/// run's state and repository, signed by the key the runner's token first
809
/// signed with.
810
pub async fn keep_receipt(
811
    app: &Shared,
b6b7e4cf Andy McCall ↶ 812
    runner: &db::runners::Runner,
cc19697f Andy McCall ↶ 813
    row: &db::repos::RepoRow,
814
    run: &fdb::Run,
815
    receipt: &flow_core::Receipt,
816
    signed_hex: &str,
817
) -> Result<(String, String), String> {
818
    use oxy_core::receipt::SignedReceipt;
819
    let bytes = hex::decode(signed_hex).map_err(|_| "the signed receipt is not hex".to_string())?;
820
    let sr = SignedReceipt::decode(&bytes).map_err(|e| format!("not a signed receipt: {e}"))?;
821
    sr.verify().map_err(|e| format!("it does not verify: {e}"))?;
822
    if sr.receipt.hash().0 != receipt.hash() {
823
        return Err("it signs another receipt than the job's".into());
824
    }
825
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
826
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(heads).encode());
827
    if sr.state() != state {
828
        return Err("it is for another state than the run's".into());
829
    }
830
    let key = sr.runner.hash().to_hex();
b6b7e4cf Andy McCall ↶ 831
    let bound = db::runners::bind_key(&app.db, runner.id, &key, &sr.runner.name).await.map_err(|e| e.to_string())?;
cc19697f Andy McCall ↶ 832
    if bound != key {
833
        return Err(format!("signed by {}, not the key this runner signs with ({})", &key[..12], &bound[..12.min(bound.len())]));
834
    }
835
    let name = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
836
    let signer = sr.runner.name.clone();
837
    let attest = app.repos.write(&name, move |repo, _| Ok(repo.add_receipt(&sr)?)).await.map_err(|e| e.to_string())?;
838
    Ok((attest.to_hex(), signer))
42fa9554 Andy McCall ↶ 839
}
840
841
/// A job's wall time, as the pages show it.
842
pub fn duration(j: &fdb::Job) -> String {
843
    match (j.started_ms, j.finished_ms) {
844
        (Some(a), Some(b)) => human_ms(b - a),
845
        (Some(a), None) => human_ms(db::now_ms() - a),
846
        _ => "–".into(),
847
    }
848
}
849
850
pub fn human_ms(ms: i64) -> String {
851
    let ms = ms.max(0);
852
    if ms < 1000 {
853
        format!("{ms} ms")
854
    } else if ms < 60_000 {
855
        format!("{:.1} s", ms as f64 / 1000.0)
856
    } else {
857
        format!("{} min {} s", ms / 60_000, (ms % 60_000) / 1000)
858
    }
859
}
bf3d0754 Andy McCall ↶ 860
861
/// Where a live run stands among the runners' work: what its viewers may
862
/// know of it. Jobs of other repositories are named only when those are
863
/// public; the rest are counted.
864
#[derive(Clone, Debug, Default, PartialEq, Eq)]
865
pub struct QueueInfo {
866
    /// 1-based place of the run's first queued job among all queued jobs.
867
    pub position: Option<usize>,
868
    /// Jobs running now, and queued jobs ahead of the run's first one.
869
    pub running: Vec<String>,
870
    pub ahead: Vec<String>,
871
    /// Of those, jobs in repositories the page may not name.
872
    pub hidden: usize,
873
}
874
875
impl QueueInfo {
876
    /// "3rd in line" and the like.
877
    pub fn place(&self) -> Option<String> {
878
        self.position.map(|n| {
879
            let suffix = match (n % 10, n % 100) {
880
                (1, 11) | (2, 12) | (3, 13) => "th",
881
                (1, _) => "st",
882
                (2, _) => "nd",
883
                (3, _) => "rd",
884
                _ => "th",
885
            };
886
            format!("{n}{suffix} in line")
887
        })
888
    }
889
}
890
891
/// The queue as `run` (of repository `repo_id`) sees it, from `queue`.
892
pub fn queue_info(queue: &[fdb::QueueEntry], repo_id: i64, run_id: i64) -> QueueInfo {
893
    let mut info = QueueInfo::default();
894
    let first = queue.iter().filter(|e| e.status == "queued").position(|e| e.run_id == run_id);
895
    info.position = first.map(|i| i + 1);
896
    let label = |e: &fdb::QueueEntry| format!("{}/{} #{} {}", e.owner, e.name, e.run_number, e.job);
897
    let mut queued_seen = 0;
898
    for e in queue {
899
        if e.run_id == run_id {
900
            continue;
901
        }
902
        let ahead = match e.status.as_str() {
903
            "running" => true,
904
            _ => {
905
                queued_seen += 1;
906
                first.is_some_and(|f| queued_seen <= f)
907
            }
908
        };
909
        if !ahead {
910
            continue;
911
        }
912
        if e.repo_id != repo_id && e.visibility != "public" {
913
            info.hidden += 1;
914
        } else if e.status == "running" {
915
            info.running.push(label(e));
916
        } else {
917
            info.ahead.push(label(e));
918
        }
919
    }
920
    info
921
}
922
923
#[cfg(test)]
924
mod queue_tests {
925
    use super::*;
926
927
    fn e(job_id: i64, run_id: i64, repo_id: i64, vis: &str, job: &str, status: &str) -> fdb::QueueEntry {
928
        fdb::QueueEntry {
929
            job_id,
930
            run_id,
931
            repo_id,
932
            owner: "o".into(),
933
            name: format!("r{repo_id}"),
934
            visibility: vis.into(),
935
            run_number: run_id,
936
            job: job.into(),
937
            status: status.into(),
b66e70b7 Andy McCall ↶ 938
            runner: String::new(),
c4c87f48 Andy McCall ↶ 939
            spec: String::new(),
940
            sandbox: String::new(),
bf3d0754 Andy McCall ↶ 941
        }
942
    }
943
944
    #[test]
945
    fn a_run_learns_its_place_and_what_is_ahead_of_it() {
946
        // Running first, then queued in claim order (as `fdb::queue` returns them).
947
        let q = vec![
948
            e(1, 10, 1, "public", "build", "running"),
949
            e(2, 11, 2, "private", "test", "running"),
950
            e(3, 10, 1, "public", "test", "queued"),
951
            e(4, 12, 3, "private", "lint", "queued"),
952
            e(5, 13, 4, "public", "build", "queued"),
953
            e(6, 13, 4, "public", "test", "queued"),
954
            e(7, 14, 5, "public", "late", "queued"),
955
        ];
956
        let info = queue_info(&q, 4, 13);
957
        assert_eq!(info.position, Some(3));
958
        assert_eq!(info.place().as_deref(), Some("3rd in line"));
959
        assert_eq!(info.running, ["o/r1 #10 build"]);
960
        assert_eq!(info.ahead, ["o/r1 #10 test"], "queued ahead, not behind");
961
        assert_eq!(info.hidden, 2, "private repositories are counted, not named");
962
        // A private repository's own jobs are named to its viewers.
963
        assert_eq!(queue_info(&q, 3, 12).ahead, ["o/r1 #10 test"]);
964
        assert_eq!(queue_info(&q, 3, 12).position, Some(2));
965
        // A run with nothing queued has no place.
966
        assert_eq!(queue_info(&q, 1, 99).position, None);
967
        let ordinal = |n| QueueInfo { position: Some(n), ..Default::default() }.place().unwrap();
968
        assert_eq!(
969
            [ordinal(1), ordinal(2), ordinal(11), ordinal(22), ordinal(113)],
970
            ["1st in line", "2nd in line", "11th in line", "22nd in line", "113th in line"]
971
        );
972
    }
973
}