OXYGENHUB
# 2a83f5303458 oxygenhub / crates / hub / src / flow.rs
45.3 KiB · Rust · last changed Variables for pipelines at the site, organisation and repository 1 month ago source blame history raw
1
//! OxygenFlow on the hub (hub spec §13.2): runs started by pushes, the
2
//! runner API, job tokens and the checks that mirror jobs. Pipelines are
3
//! parsed by `flow-core`, the same code `flow check` runs.
4
5
use crate::api::ApiError;
6
use crate::db::{self, flow as fdb};
7
use crate::repos::RepoName;
8
use crate::Shared;
9
use axum::extract::{Path, State};
10
use axum::http::{HeaderMap, StatusCode};
11
use axum::response::{IntoResponse, Response};
12
use axum::Json;
13
use flow_core::api::{Claim, ClaimRequest, Finish, LogBatch, MAX_RECORDS, MAX_RECORD_TEXT};
14
use flow_core::{Pipeline, PIPELINE_PATH};
15
use oxy_core::hash::ChangeHash;
16
17
/// Prefix of job tokens, so the wire protocol can tell them from personal tokens.
18
pub const JOB_TOKEN_PREFIX: &str = "flowjob_";
19
/// A job token is refused this long after its job started, whatever its state.
20
pub const JOB_TOKEN_MAX_AGE_MS: i64 = 2 * 3600 * 1000;
21
22
/// Whether pushes to this repository start runs: its own setting when a
23
/// site administrator made one, else hub.toml's `[flow] repos`.
24
pub fn enabled(app: &Shared, row: &db::repos::RepoRow) -> bool {
25
    match row.flow.as_str() {
26
        "on" => true,
27
        "off" => false,
28
        _ => app.config.flow.allows(&row.owner, &row.name),
29
    }
30
}
31
32
/// A bookmark moved by a push: start a run when the repository is allowed
33
/// and the new state has a pipeline.
34
pub async fn on_push(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, state: &str, heads: &[ChangeHash], actor: &str) {
35
    if heads.is_empty() || !enabled(app, row) {
36
        return;
37
    }
38
    // A pushed tag runs the jobs whose `on` names it (oxygenflow#24).
39
    let started = match bookmark.strip_prefix("tags/") {
40
        Some(tag) => start_run_with(app, row, bookmark, state, heads, actor, &format!("tag:{tag}"), None).await,
41
        None => {
42
            let touched = touched_since_last_run(app, row, bookmark, heads).await;
43
            let started = start_run_scoped(app, row, bookmark, state, heads, actor, "push", touched.as_deref()).await;
44
            if let Ok(Some(n)) = started {
45
                supersede(app, row, bookmark, n).await;
46
            }
47
            started
48
        }
49
    };
50
    if let Err(e) = started {
51
        tracing::warn!("flow: starting a run of {}/{}: {e}", row.owner, row.name);
52
    }
53
}
54
55
/// "Run pipeline" (hub spec §13.2): run a bookmark's current state without
56
/// moving it. Returns the new run's number.
57
pub async fn run_bookmark(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, actor: &str) -> Result<i64, String> {
58
    if !enabled(app, row) {
59
        return Err("OxygenFlow is not enabled for this repository".into());
60
    }
61
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
62
    let b = bookmark.to_string();
63
    let found = app
64
        .repos
65
        .read(&repo, move |r, _| {
66
            let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) };
67
            Ok(Some((st, r.store().get_state(&st)?.heads)))
68
        })
69
        .await
70
        .map_err(|e| e.to_string())?;
71
    let Some((state, heads)) = found else { return Err(format!("no bookmark {bookmark}")) };
72
    if heads.is_empty() {
73
        return Err(format!("{bookmark} has no changes yet"));
74
    }
75
    start_run(app, row, bookmark, &state.to_hex(), &heads, actor, "manual")
76
        .await?
77
        .ok_or_else(|| format!("{bookmark} has no {PIPELINE_PATH}"))
78
}
79
80
/// "Run pipeline" on one committed change: the state whose only head is
81
/// `change`, whatever bookmarks hold it. Returns the new run's number.
82
pub async fn run_change(app: &Shared, row: &db::repos::RepoRow, change: &str, actor: &str) -> Result<i64, String> {
83
    if !enabled(app, row) {
84
        return Err("OxygenFlow is not enabled for this repository".into());
85
    }
86
    let h = ChangeHash::from_hex(change.trim().trim_start_matches('#')).ok_or("give the full hash of a change")?;
87
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
88
    let known = app.repos.read(&repo, move |r, _| Ok(r.store().get_change_meta(&h).is_ok())).await.map_err(|e| e.to_string())?;
89
    if !known {
90
        return Err(format!("no change {} in this repository", &change.trim()[..12.min(change.trim().len())]));
91
    }
92
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(vec![h]).encode());
93
    start_run(app, row, "", &state.to_hex(), &[h], actor, "manual")
94
        .await?
95
        .ok_or_else(|| format!("that change's state has no {PIPELINE_PATH}"))
96
}
97
98
/// Re-run a finished run: the same state, bookmark and pipeline, as a new
99
/// run whose checks replace the old ones. Returns the new run's number.
100
pub async fn rerun(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<i64, String> {
101
    if !enabled(app, row) {
102
        return Err("OxygenFlow is not enabled for this repository".into());
103
    }
104
    if run.live() {
105
        return Err(format!("run #{} is still {}", run.number, run.status));
106
    }
107
    let heads: Vec<ChangeHash> = run.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
108
    if heads.is_empty() {
109
        return Err("the run names no changes".into());
110
    }
111
    // A rerun repeats its run's event: the jobs it had are the jobs it gets.
112
    let trigger = format!("rerun:{}:{}", run.number, original_event(run));
113
    start_run(app, row, &run.bookmark, &run.state, &heads, actor, &trigger)
114
        .await?
115
        .ok_or_else(|| format!("that state has no {PIPELINE_PATH}"))
116
}
117
118
/// Cancel a run still going (oxygenflow#32), its checks with it.
119
pub async fn cancel(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<(), String> {
120
    if !run.live() {
121
        return Err(format!("run #{} is already {}", run.number, run.status));
122
    }
123
    cancel_with(app, run, &format!("cancelled by {actor}")).await.map_err(|e| {
124
        tracing::warn!("flow: cancelling run #{} of {}/{}: {e}", run.number, row.owner, row.name);
125
        "the run could not be cancelled".to_string()
126
    })
127
}
128
129
async fn cancel_with(app: &Shared, run: &fdb::Run, why: &str) -> sqlx::Result<()> {
130
    let changed = fdb::cancel_run(&app.db, run.id, why).await?;
131
    report_changed(app, run.id, changed).await
132
}
133
134
/// A push makes the push runs still going on its bookmark moot: they are
135
/// cancelled, and the new run's scope reaches back past them.
136
async fn supersede(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, number: i64) {
137
    let Ok(older) = fdb::live_push_runs_before(&app.db, row.id, bookmark, number).await else { return };
138
    for run in older {
139
        if let Err(e) = cancel_with(app, &run, &format!("superseded by run #{number}")).await {
140
            tracing::warn!("flow: superseding run #{} of {}/{}: {e}", run.number, row.owner, row.name);
141
        }
142
    }
143
}
144
145
/// The event a run's trigger stands for: `push`, `tag:NAME`, `manual`,
146
/// `schedule`, and `rerun:N:EVENT` repeating EVENT.
147
fn event_of<'a>(trigger: &'a str, bookmark: &'a str) -> flow_core::pipeline::Event<'a> {
148
    use flow_core::pipeline::Event;
149
    let t = match trigger.strip_prefix("rerun:") {
150
        Some(rest) => rest.split_once(':').map(|(_, e)| e).unwrap_or("manual"),
151
        None => trigger,
152
    };
153
    if let Some(tag) = t.strip_prefix("tag:") {
154
        Event::Tag(tag)
155
    } else if t == "push" {
156
        Event::Push(bookmark)
157
    } else if t == "schedule" {
158
        Event::Schedule
159
    } else {
160
        Event::Manual
161
    }
162
}
163
164
/// The event to repeat when rerunning `run`.
165
fn original_event(run: &fdb::Run) -> String {
166
    let t = run.trigger.as_str();
167
    match t.strip_prefix("rerun:") {
168
        Some(rest) => rest.split_once(':').map(|(_, e)| e.to_string()).unwrap_or_else(|| "manual".into()),
169
        None => t.to_string(),
170
    }
171
}
172
173
/// The paths the changes a push brought touch: the changes in `heads`'
174
/// state that the bookmark's previous run did not have, and the files their
175
/// edits and tree operations name, as the new state places them. `None`
176
/// (run every job) for the bookmark's first run, or when it cannot be told.
177
async fn touched_since_last_run(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, heads: &[ChangeHash]) -> Option<Vec<String>> {
178
    let last = fdb::latest_on(&app.db, row.id, bookmark).await.ok()??;
179
    let old: Vec<ChangeHash> = last.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
180
    if old.is_empty() {
181
        return None;
182
    }
183
    let repo = RepoName::new(&row.owner, &row.name)?;
184
    let new = heads.to_vec();
185
    app.repos
186
        .read(&repo, move |r, cache| {
187
            let fresh = r.negotiate(&old, &new)?;
188
            let loaded = cache.load(r, &new)?;
189
            let by_node: std::collections::HashMap<oxy_core::ids::NodeId, &String> =
190
                loaded.snapshot.files.iter().map(|(p, f)| (f.node, p)).collect();
191
            let mut paths = std::collections::BTreeSet::new();
192
            for h in fresh {
193
                let (c, _) = r.store().get_change_lazy(&h)?;
194
                let nodes = c.edits.iter().map(|e| e.node()).chain(c.tree_ops.iter().map(|o| o.node()));
195
                for n in nodes {
196
                    if let Some(p) = by_node.get(&n) {
197
                        paths.insert((*p).clone());
198
                    }
199
                }
200
            }
201
            Ok(paths.into_iter().collect::<Vec<String>>())
202
        })
203
        .await
204
        .ok()
205
}
206
207
/// Whether a job runs for a push that touched `touched`: always without a
208
/// scope or without knowing what was touched, else when a path matches.
209
fn in_scope(scope: &[String], touched: Option<&[String]>) -> bool {
210
    let Some(touched) = touched else { return true };
211
    if scope.is_empty() {
212
        return true;
213
    }
214
    let mut b = globset::GlobSetBuilder::new();
215
    for g in scope {
216
        match globset::Glob::new(g) {
217
            Ok(g) => {
218
                b.add(g);
219
            }
220
            Err(_) => return true,
221
        }
222
    }
223
    let Ok(set) = b.build() else { return true };
224
    touched.iter().any(|p| set.is_match(p))
225
}
226
227
/// [`start_run`] for a push: jobs whose `scope` the push did not touch are
228
/// left out of the run, unless a job that runs depends on them.
229
#[allow(clippy::too_many_arguments)]
230
pub async fn start_run_scoped(
231
    app: &Shared,
232
    row: &db::repos::RepoRow,
233
    bookmark: &str,
234
    state: &str,
235
    heads: &[ChangeHash],
236
    actor: &str,
237
    trigger: &str,
238
    touched: Option<&[String]>,
239
) -> Result<Option<i64>, String> {
240
    start_run_with(app, row, bookmark, state, heads, actor, trigger, touched).await
241
}
242
243
/// Start a run of the pipeline in the state `heads` make (whose state
244
/// hash is `state`), recorded on `bookmark`: for a push, "Run pipeline"
245
/// (`trigger` = `manual`) and re-runs (`rerun:N`). `actor` is the account
246
/// whose access the run's subrepo reads are checked against. Returns the
247
/// run's number, or `None` when the state has no pipeline. The caller
248
/// checks that Flow is enabled.
249
pub async fn start_run(
250
    app: &Shared,
251
    row: &db::repos::RepoRow,
252
    bookmark: &str,
253
    state: &str,
254
    heads: &[ChangeHash],
255
    actor: &str,
256
    trigger: &str,
257
) -> Result<Option<i64>, String> {
258
    start_run_with(app, row, bookmark, state, heads, actor, trigger, None).await
259
}
260
261
#[allow(clippy::too_many_arguments)]
262
async fn start_run_with(
263
    app: &Shared,
264
    row: &db::repos::RepoRow,
265
    bookmark: &str,
266
    state: &str,
267
    heads: &[ChangeHash],
268
    actor: &str,
269
    trigger: &str,
270
    touched: Option<&[String]>,
271
) -> Result<Option<i64>, String> {
272
    start_run_at(app, row, bookmark, state, heads, actor, trigger, touched, None).await
273
}
274
275
/// [`start_run_with`], for a scheduled run due at `at` on the hub's clock.
276
#[allow(clippy::too_many_arguments)]
277
async fn start_run_at(
278
    app: &Shared,
279
    row: &db::repos::RepoRow,
280
    bookmark: &str,
281
    state: &str,
282
    heads: &[ChangeHash],
283
    actor: &str,
284
    trigger: &str,
285
    touched: Option<&[String]>,
286
    at: Option<flow_core::pipeline::When>,
287
) -> Result<Option<i64>, String> {
288
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
289
    let h = heads.to_vec();
290
    let text = app
291
        .repos
292
        .read(&repo, move |r, cache| {
293
            let loaded = cache.load(r, &h)?;
294
            Ok(match loaded.snapshot.files.get(PIPELINE_PATH).map(|f| &f.content) {
295
                Some(oxy_core::state::FileContent::Text { bytes, .. }) => Some(String::from_utf8_lossy(bytes).into_owned()),
296
                _ => None,
297
            })
298
        })
299
        .await
300
        .map_err(|e| format!("reading the pipeline: {e}"))?;
301
    let Some(text) = text else { return Ok(None) };
302
    let mut hex: Vec<String> = heads.iter().map(|h| h.to_hex()).collect();
303
    hex.sort();
304
    let heads_s = hex.join("+");
305
    let base = db::flow::NewRun { repo_id: row.id, bookmark, state, heads: &heads_s, pipeline_hash: "", actor, trigger, error: None };
306
    let id = match Pipeline::parse(&text) {
307
        Err(problems) => {
308
            let msg = problems.to_string();
309
            db::flow::create_run(&app.db, db::flow::NewRun { error: Some(&msg), ..base }, &[]).await.map_err(|e| e.to_string())?
310
        }
311
        Ok(p) => {
312
            let depths = p.depths();
313
            // Jobs out of the push's scope stay out, unless a kept job
314
            // needs them.
315
            // The jobs this run's event starts (`on`), out of those its
316
            // push touched (`scope`); then what they depend on.
317
            let event = match at {
318
                Some(w) => flow_core::pipeline::Event::ScheduleAt(w),
319
                None => event_of(trigger, bookmark),
320
            };
321
            let mut keep: std::collections::HashSet<String> = p
322
                .order()
323
                .into_iter()
324
                .filter(|j| p.on_of(j).matches(event) && in_scope(&j.scope, touched))
325
                .map(|j| j.name.clone())
326
                .collect();
327
            loop {
328
                let needed: Vec<String> = p
329
                    .order()
330
                    .into_iter()
331
                    .filter(|j| keep.contains(&j.name))
332
                    .flat_map(|j| j.depends_on.iter().cloned())
333
                    .filter(|d| !keep.contains(d))
334
                    .collect();
335
                if needed.is_empty() {
336
                    break;
337
                }
338
                keep.extend(needed);
339
            }
340
            let jobs: Vec<fdb::NewJob> = p
341
                .order()
342
                .into_iter()
343
                .filter(|j| keep.contains(&j.name))
344
                .map(|j| {
345
                    let sb = &p.sandboxes[&j.sandbox];
346
                    fdb::NewJob {
347
                        name: j.name.clone(),
348
                        depth: depths[j.name.as_str()] as i64,
349
                        depends: j.depends_on.clone(),
350
                        spec: serde_json::to_string(j).unwrap_or_default(),
351
                        sandbox: serde_json::to_string(sb).unwrap_or_default(),
352
                        runtime: sb.runtime.as_str().into(),
353
                    }
354
                })
355
                .collect();
356
            if jobs.is_empty() {
357
                // Nothing this event starts (or this push touched).
358
                return Ok(None);
359
            }
360
            let hash = hex::encode(p.hash());
361
            let id =
362
                db::flow::create_run(&app.db, db::flow::NewRun { pipeline_hash: &hash, ..base }, &jobs).await.map_err(|e| e.to_string())?;
363
            if let Ok(Some(run)) = fdb::run_by_id(&app.db, id).await {
364
                for j in &jobs {
365
                    report_check(app, row, state, run.number, &j.name, "queued", "waiting for a runner").await;
366
                }
367
            }
368
            id
369
        }
370
    };
371
    let run = fdb::run_by_id(&app.db, id).await.map_err(|e| e.to_string())?.ok_or("the run vanished")?;
372
    Ok(Some(run.number))
373
}
374
375
/// Record a job's check; it links to the run's page by the run's number.
376
async fn report_check(app: &Shared, row: &db::repos::RepoRow, state: &str, run_number: i64, job: &str, status: &str, summary: &str) {
377
    let url = format!("{}/{}/{}/flow/{run_number}", app.config.base_url(), row.owner, row.name);
378
    let name = format!("flow/{job}");
379
    let c = db::hooks::upsert_check(
380
        &app.db,
381
        row.id,
382
        db::hooks::CheckReport { state, name: &name, status, summary, url: &url, reporter: None, seal: "" },
383
    )
384
    .await;
385
    if let Ok(c) = c {
386
        let api = crate::api::check_of(&c);
387
        app.events.proposal(row.id, hub_api::ProposalEvent { number: 0, kind: "check".into(), check: Some(api.clone()) });
388
        crate::hooks::emit(app, row, "check", "flow", serde_json::json!({ "check": api })).await;
389
    }
390
}
391
392
/// The check status a job's status shows as.
393
fn check_status(job_status: &str) -> &'static str {
394
    match job_status {
395
        "queued" => "queued",
396
        "running" => "running",
397
        "passed" => "passed",
398
        "skipped" => "skipped",
399
        "cancelled" => "cancelled",
400
        _ => "failed",
401
    }
402
}
403
404
/// The runner a bearer token belongs to: a registered runner, or one
405
/// configured with a hub.toml token (tracked from its first request). A
406
/// disabled runner is refused.
407
async fn runner_auth(app: &Shared, headers: &HeaderMap) -> Result<db::runners::Runner, ApiError> {
408
    let Some(t) = oxy_server::bearer_token(headers) else {
409
        return Err(ApiError(StatusCode::UNAUTHORIZED, "a runner token is required".into()));
410
    };
411
    let hash = blake3::hash(t.as_bytes()).to_hex().to_string();
412
    let runner = match db::runners::by_token(&app.db, &hash).await? {
413
        Some(r) => r,
414
        None if app.config.flow.runner_tokens.iter().any(|x| crate::auth::mac_equal(x, t)) => {
415
            db::runners::configured(&app.db, &hash, "runner").await?
416
        }
417
        None => return Err(ApiError(StatusCode::UNAUTHORIZED, "not a runner token".into())),
418
    };
419
    if runner.state == "disabled" {
420
        return Err(ApiError(StatusCode::FORBIDDEN, format!("runner {} is disabled on this hub", runner.name)));
421
    }
422
    Ok(runner)
423
}
424
425
/// `POST /api/v1/flow/runners/register`: a one-time registration token
426
/// from Admin → Runners buys a runner credential, binding the runner's
427
/// signing identity (hub spec §13.2).
428
pub async fn register(
429
    State(app): State<Shared>,
430
    Json(req): Json<flow_core::api::Register>,
431
) -> Result<Json<flow_core::api::Registered>, ApiError> {
432
    let bytes = hex::decode(req.identity.trim()).map_err(|_| ApiError(StatusCode::BAD_REQUEST, "identity: not hex".into()))?;
433
    let identity = oxy_core::identity::Identity::decode(&bytes).map_err(|e| ApiError(StatusCode::BAD_REQUEST, format!("identity: {e}")))?;
434
    let name: String = req.name.chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')).take(64).collect();
435
    if name.is_empty() {
436
        return Err(ApiError(StatusCode::BAD_REQUEST, "give the runner a name of letters, digits, - _ or .".into()));
437
    }
438
    let credential = format!("flowrunner_{}", crate::auth::random_hex(32));
439
    let hash = blake3::hash(credential.as_bytes()).to_hex().to_string();
440
    let reg = blake3::hash(req.token.trim().as_bytes()).to_hex().to_string();
441
    let runner =
442
        db::runners::register(&app.db, &reg, &name, &hash, &identity.hash().to_hex(), &identity.name, "").await?.ok_or_else(|| {
443
            ApiError(StatusCode::FORBIDDEN, "that registration token is unknown, used or expired; create one on Admin → Runners".into())
444
        })?;
445
    db::runners::seen(&app.db, runner.id, None, Some(&req.info)).await?;
446
    tracing::info!("flow: runner {} registered (key {})", runner.name, &runner.key[..12]);
447
    Ok(Json(flow_core::api::Registered { id: runner.id, name: runner.name, credential }))
448
}
449
450
/// `POST /api/v1/flow/claim`.
451
pub async fn claim(State(app): State<Shared>, headers: HeaderMap, Json(req): Json<ClaimRequest>) -> Result<Response, ApiError> {
452
    let me = runner_auth(&app, &headers).await?;
453
    let claimed: String = req.runner.chars().filter(|c| !c.is_control()).take(64).collect();
454
    db::runners::seen(&app.db, me.id, Some(&claimed), req.info.as_ref()).await?;
455
    if let Some(holding) = &req.holding {
456
        end_jobs_not_held(&app, &me, holding).await?;
457
    }
458
    // A draining runner finishes what it has and takes nothing new.
459
    if me.state == "draining" {
460
        return Ok(StatusCode::NO_CONTENT.into_response());
461
    }
462
    let runner = if me.source == "registered" { me.name.clone() } else { claimed };
463
    let secret = format!("{JOB_TOKEN_PREFIX}{}", crate::auth::random_hex(32));
464
    let room = fdb::Room {
465
        memory_mib: req.free_memory_mib,
466
        cpus: req.free_cpus,
467
        images: req.images.as_ref().map(|i| i.iter().map(|h| h.to_ascii_lowercase()).collect()),
468
    };
469
    // The labels it reports now, or those it registered with.
470
    let labels: Vec<String> = match &req.info {
471
        Some(i) => i.labels.clone(),
472
        None => me.label_list().into_iter().map(String::from).collect(),
473
    };
474
    let Some(job) =
475
        fdb::claim_within(&app.db, &req.runtimes, &runner, &crate::auth::token_hash(&secret), room, &me.scope(), &labels).await?
476
    else {
477
        return Ok(StatusCode::NO_CONTENT.into_response());
478
    };
479
    db::runners::set_job_runner(&app.db, job.id, me.id).await?;
480
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
481
    let row =
482
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
483
    report_check(&app, &row, &run.state, run.number, &job.name, "running", &format!("running on {runner}")).await;
484
    let bad = |e: serde_json::Error| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}"));
485
    // The job's limit: its own timeout, else its sandbox's, else the site
486
    // default, never past the site maximum; recorded for the reaper.
487
    let limits = limits(&app).await;
488
    let spec: flow_core::pipeline::Job = serde_json::from_str(&job.spec).map_err(bad)?;
489
    let sandbox: flow_core::pipeline::Sandbox = serde_json::from_str(&job.sandbox).map_err(bad)?;
490
    let timeout_s = spec.timeout_s.or(sandbox.timeout_s).unwrap_or(limits.default_s).min(limits.max_s).max(60);
491
    fdb::set_job_timeout(&app.db, job.id, timeout_s).await?;
492
    let claim = Claim {
493
        job_id: job.id,
494
        run_id: run.id,
495
        repo: format!("{}/{}", row.owner, row.name),
496
        clone_url: format!("{}/{}/{}", app.config.base_url(), row.owner, row.name),
497
        token: secret,
498
        heads: run.head_list().into_iter().map(String::from).collect(),
499
        pipeline_hash: run.pipeline_hash.clone(),
500
        job: spec,
501
        sandbox,
502
        timeout_s,
503
        subrepos: vec![],
504
        state: run.state.clone(),
505
        bookmark: run.bookmark.clone(),
506
        run_number: run.number,
507
        // The site's, the owner's and the repository's variables (#102).
508
        env: crate::variables::env_for_repo(&app.db, &row).await?,
509
    };
510
    let mut claim = claim;
511
    let (inputs, ids) = subrepo_inputs(&app, &row, &run).await;
512
    if !ids.is_empty() {
513
        fdb::set_also_read(&app.db, job.id, &ids).await?;
514
    }
515
    claim.subrepos = inputs;
516
    Ok(Json(claim).into_response())
517
}
518
519
/// The Oxygen subrepo pins in `name`'s state `heads`: (path, repository id, pinned heads).
520
async fn pins_in(
521
    app: &Shared,
522
    name: crate::repos::RepoName,
523
    heads: Vec<oxy_core::hash::ChangeHash>,
524
) -> Vec<(String, [u8; 16], Vec<oxy_core::hash::ChangeHash>)> {
525
    app.repos
526
        .read(&name, move |repo, cache| {
527
            let loaded = cache.load(repo, &heads)?;
528
            let mut out = Vec::new();
529
            for (path, f) in &loaded.snapshot.files {
530
                if f.kind != oxy_core::objects::NodeKind::Subrepo {
531
                    continue;
532
                }
533
                if let oxy_core::state::FileContent::Opaque { blobs } = &f.content {
534
                    if let Some(b) = blobs.first() {
535
                        if let Ok(oxy_core::subrepo::Pin::Oxy { repo: id, heads, .. }) = repo
536
                            .read_blob(b)
537
                            .map_err(crate::repos::RepoError::Oxy)
538
                            .and_then(|bytes| oxy_core::subrepo::Pin::decode(&bytes).map_err(|e| crate::repos::RepoError::Oxy(e.into())))
539
                        {
540
                            out.push((path.clone(), id, heads));
541
                        }
542
                    }
543
                }
544
            }
545
            Ok(out)
546
        })
547
        .await
548
        .unwrap_or_default()
549
}
550
551
/// How deep subrepos of subrepos are followed.
552
const MAX_SUBREPO_DEPTH: usize = 3;
553
554
/// The Oxygen subrepos the run's state pins, and those their states pin in
555
/// turn (parents first, paths joined), that live on this hub and that the
556
/// run's pusher may read: what the job's workspace needs beside its own
557
/// tree. A pin the hub cannot serve is left out, with what it pins; the job
558
/// then fails on the missing files with an ordinary error.
559
async fn subrepo_inputs(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>) {
560
    let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![]) };
561
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
562
    let pusher = db::accounts::account_by_name(&app.db, &run.actor).await.ok().flatten();
563
    let rows = db::repos::list(&app.db).await.unwrap_or_default();
564
    let mut inputs = Vec::new();
565
    let mut ids = Vec::new();
566
    // (path prefix, repository, heads, depth), parents before children.
567
    let mut queue = std::collections::VecDeque::from([(String::new(), name, heads, 0usize)]);
568
    while let Some((prefix, name, heads, depth)) = queue.pop_front() {
569
        if depth == MAX_SUBREPO_DEPTH {
570
            continue;
571
        }
572
        for (path, id, pinned) in pins_in(app, name, heads).await {
573
            let path = if prefix.is_empty() { path } else { format!("{prefix}/{path}") };
574
            for other in &rows {
575
                let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue };
576
                let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false);
577
                if !matches {
578
                    continue;
579
                }
580
                let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten();
581
                if crate::auth::roles::allows(role, crate::auth::Access::Read) {
582
                    inputs.push(flow_core::api::SubrepoInput {
583
                        path: path.clone(),
584
                        repo: format!("{}/{}", other.owner, other.name),
585
                        clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name),
586
                        heads: pinned.iter().map(|h| h.to_hex()).collect(),
587
                    });
588
                    if !ids.contains(&other.id) {
589
                        ids.push(other.id);
590
                    }
591
                    queue.push_back((path.clone(), other_name, pinned.clone(), depth + 1));
592
                }
593
                break;
594
            }
595
        }
596
    }
597
    (inputs, ids)
598
}
599
600
async fn running_job(app: &Shared, id: i64) -> Result<fdb::Job, ApiError> {
601
    match fdb::job(&app.db, id).await? {
602
        Some(j) if j.status == "running" => Ok(j),
603
        Some(_) => Err(ApiError(StatusCode::CONFLICT, "this job is not running".into())),
604
        None => Err(ApiError(StatusCode::NOT_FOUND, "no such job".into())),
605
    }
606
}
607
608
/// `POST /api/v1/flow/jobs/{id}/log`.
609
pub async fn log(
610
    State(app): State<Shared>,
611
    headers: HeaderMap,
612
    Path(id): Path<i64>,
613
    Json(mut batch): Json<LogBatch>,
614
) -> Result<StatusCode, ApiError> {
615
    runner_auth(&app, &headers).await?;
616
    let job = running_job(&app, id).await?;
617
    // An empty batch asks only whether the job still runs: a cancelled
618
    // one answers 409 and its runner stops it (oxygenflow#32).
619
    if batch.records.is_empty() {
620
        return Ok(StatusCode::NO_CONTENT);
621
    }
622
    for r in &mut batch.records {
623
        if r.text.len() > MAX_RECORD_TEXT {
624
            let mut cut = MAX_RECORD_TEXT;
625
            while !r.text.is_char_boundary(cut) {
626
                cut -= 1;
627
            }
628
            r.text.truncate(cut);
629
        }
630
        if !matches!(r.stream.as_str(), "out" | "err" | "console" | "flow") {
631
            r.stream = "console".into();
632
        }
633
    }
634
    let kept = fdb::append_log(&app.db, job.id, &batch.records, MAX_RECORDS as i64).await?;
635
    if kept < batch.records.len() && job.log_count < MAX_RECORDS as i64 {
636
        tracing::info!("flow: job {} reached the log limit", job.id);
637
    }
638
    Ok(StatusCode::NO_CONTENT)
639
}
640
641
/// Largest artifact the hub keeps; the runner refuses larger ones first.
642
pub const MAX_ARTIFACT_BYTES: usize = 512 << 20;
643
644
/// `POST /api/v1/flow/jobs/{id}/artifacts/{name}`: the bytes of an artifact
645
/// the job declares (OxygenFlow spec §5.2), kept as a blob of the run's
646
/// repository. Answers `{"blob"}`.
647
pub async fn artifact(
648
    State(app): State<Shared>,
649
    headers: HeaderMap,
650
    Path((id, name)): Path<(i64, String)>,
651
    body: axum::body::Bytes,
652
) -> Result<Response, ApiError> {
653
    runner_auth(&app, &headers).await?;
654
    let job = running_job(&app, id).await?;
655
    let spec: flow_core::Job =
656
        serde_json::from_str(&job.spec).map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}")))?;
657
    let Some(out) = spec.outputs.iter().find(|o| o.name == name) else {
658
        return Err(ApiError(StatusCode::BAD_REQUEST, format!("job {} declares no artifact {name:?}", job.name)));
659
    };
660
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
661
    let row =
662
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
663
    let repo = RepoName::new(&row.owner, &row.name).ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "bad repository name".into()))?;
664
    let content = blake3::hash(&body).to_hex().to_string();
665
    let size = body.len() as i64;
666
    let blob = app
667
        .repos
668
        .write(&repo, move |r, _| Ok(r.store_blob(&body)?))
669
        .await
670
        .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("keep the artifact: {e}")))?;
671
    let a = fdb::Artifact { job_id: job.id, name, path: out.path.clone(), blob: blob.to_hex(), size, content, created_ms: db::now_ms() };
672
    fdb::add_artifact(&app.db, &a).await?;
673
    Ok(Json(serde_json::json!({ "blob": a.blob })).into_response())
674
}
675
676
/// `POST /api/v1/flow/jobs/{id}/finish`.
677
pub async fn finish(
678
    State(app): State<Shared>,
679
    headers: HeaderMap,
680
    Path(id): Path<i64>,
681
    Json(f): Json<Finish>,
682
) -> Result<StatusCode, ApiError> {
683
    let me = runner_auth(&app, &headers).await?;
684
    let job = running_job(&app, id).await?;
685
    let status = match (f.exit_code, &f.error) {
686
        (Some(0), None) => "passed",
687
        (Some(_), _) => "failed",
688
        (None, _) => "error",
689
    };
690
    let (receipt, receipt_hash) = match &f.receipt {
691
        Some(r) if r.job_name == job.name && Some(r.exit_code as i32) == f.exit_code => {
692
            // The receipt names exactly the artifacts uploaded, in name order.
693
            let kept: Vec<String> = fdb::artifacts(&app.db, job.id).await?.into_iter().map(|a| a.blob).collect();
694
            if r.outputs.iter().map(hex::encode).collect::<Vec<_>>() != kept {
695
                return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt names other outputs than the artifacts uploaded".into()));
696
            }
697
            (serde_json::to_string(r).unwrap_or_default(), hex::encode(r.hash()))
698
        }
699
        Some(_) => return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt does not match the job".into())),
700
        None => (String::new(), String::new()),
701
    };
702
    let error = f.error.clone().unwrap_or_default();
703
    let changed = fdb::finish(
704
        &app.db,
705
        job.id,
706
        fdb::Outcome { status, exit_code: f.exit_code.map(i64::from), error: &error, receipt: &receipt, receipt_hash: &receipt_hash },
707
    )
708
    .await?;
709
    if let Some(run) = fdb::run_by_id(&app.db, job.run_id).await? {
710
        if let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? {
711
            if let (Some(r), Some(signed)) = (&f.receipt, &f.signed_receipt) {
712
                let (attest, signer, note) = match keep_receipt(&app, &me, &row, &run, r, signed).await {
713
                    Ok((attest, signer)) => (attest, signer, String::new()),
714
                    Err(e) => {
715
                        tracing::warn!("flow: job {} receipt not kept: {e}", job.id);
716
                        (String::new(), String::new(), e)
717
                    }
718
                };
719
                fdb::set_receipt(&app.db, job.id, &attest, &signer, &note).await?;
720
            }
721
        }
722
    }
723
    report_changed(&app, job.run_id, changed).await?;
724
    Ok(StatusCode::NO_CONTENT)
725
}
726
727
/// Report the checks of jobs whose status changed.
728
async fn report_changed(app: &Shared, run_id: i64, changed: Vec<fdb::Job>) -> sqlx::Result<()> {
729
    let Some(run) = fdb::run_by_id(&app.db, run_id).await? else {
730
        return Ok(());
731
    };
732
    let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? else {
733
        return Ok(());
734
    };
735
    for j in changed {
736
        let summary = match j.status.as_str() {
737
            "passed" => format!("passed in {}", duration(&j)),
738
            "failed" => format!("exit {} after {}", j.exit_code.unwrap_or(-1), duration(&j)),
739
            "skipped" => "skipped: a dependency did not pass".into(),
740
            "cancelled" => j.error.clone(),
741
            _ => format!("could not run: {}", j.error),
742
        };
743
        report_check(app, &row, &run.state, run.number, &j.name, check_status(&j.status), &summary).await;
744
    }
745
    Ok(())
746
}
747
748
/// The site's job time limits (oxygenhub#89): set on Admin → Runners →
749
/// Settings, else from `hub.toml`.
750
pub struct Limits {
751
    /// For a job whose pipeline names none.
752
    pub default_s: u64,
753
    /// The most a pipeline may ask for.
754
    pub max_s: u64,
755
    /// Grace past a job's limit before the hub gives it up: a runner stops
756
    /// a job at its limit and reports it within moments, so one still
757
    /// running this long after is one the runner no longer reports (it was
758
    /// restarted, or lost the hub while the job ended, oxygenhub#71).
759
    pub grace_ms: i64,
760
}
761
762
pub const SETTING_DEFAULT: &str = "flow.job_timeout_minutes";
763
pub const SETTING_MAX: &str = "flow.job_timeout_max_minutes";
764
pub const SETTING_GRACE: &str = "flow.lost_grace_minutes";
765
766
pub async fn limits(app: &Shared) -> Limits {
767
    let minutes = |key: &'static str| async move {
768
        db::accounts::setting(&app.db, key).await.ok().flatten().and_then(|v| v.parse::<u64>().ok()).filter(|m| *m > 0)
769
    };
770
    let default_m = minutes(SETTING_DEFAULT).await.unwrap_or(app.config.flow.job_timeout_minutes.max(1));
771
    let max_m = minutes(SETTING_MAX).await.unwrap_or(default_m.max(6 * 60)).max(default_m);
772
    let grace_m = minutes(SETTING_GRACE).await.unwrap_or(5);
773
    Limits { default_s: default_m * 60, max_s: max_m * 60, grace_ms: grace_m as i64 * 60_000 }
774
}
775
776
/// End every job still running past its time limit plus the grace:
777
/// it is marked an error with the reason, the jobs that depended on it are
778
/// skipped and its run settled, as when a runner reports an error.
779
pub async fn reap_lost_jobs(app: &Shared) -> sqlx::Result<usize> {
780
    let limits = limits(app).await;
781
    let lost = fdb::running_past_limit(&app.db, db::now_ms(), limits.default_s, limits.grace_ms).await?;
782
    let n = lost.len();
783
    for job in lost {
784
        let error = format!(
785
            "lost: still running {} minutes after it started, past the time limit; the runner{} stopped reporting it",
786
            (db::now_ms() - job.started_ms.unwrap_or(0)) / 60_000,
787
            if job.runner.is_empty() { String::new() } else { format!(" {}", job.runner) },
788
        );
789
        let changed =
790
            fdb::finish(&app.db, job.id, fdb::Outcome { status: "error", exit_code: None, error: &error, receipt: "", receipt_hash: "" })
791
                .await?;
792
        tracing::warn!("flow: job {} {error}", job.id);
793
        report_changed(app, job.run_id, changed).await?;
794
    }
795
    Ok(n)
796
}
797
798
/// End the jobs running on `runner` that it does not hold (oxygenhub#71):
799
/// it restarted since claiming them, or never received the claim. A
800
/// runner claims one job at a time, so a job it holds is always listed.
801
async fn end_jobs_not_held(app: &Shared, runner: &db::runners::Runner, holding: &[i64]) -> sqlx::Result<()> {
802
    for job in fdb::running_on(&app.db, runner.id).await? {
803
        if holding.contains(&job.id) {
804
            continue;
805
        }
806
        let error = format!("lost: the runner {} no longer holds this job (it restarted, or never received it)", runner.name);
807
        let changed =
808
            fdb::finish(&app.db, job.id, fdb::Outcome { status: "error", exit_code: None, error: &error, receipt: "", receipt_hash: "" })
809
                .await?;
810
        tracing::warn!("flow: job {} {error}", job.id);
811
        report_changed(app, job.run_id, changed).await?;
812
    }
813
    Ok(())
814
}
815
816
/// Start the scheduled runs due at `when` (oxygenflow#24): on each
817
/// repository that runs Flow, the jobs of its default bookmark's pipeline
818
/// with a `schedule` line due then. Returns how many runs started.
819
pub async fn run_due(app: &Shared, when: flow_core::pipeline::When) -> usize {
820
    let Ok(rows) = db::repos::list(&app.db).await else { return 0 };
821
    let mut started = 0;
822
    for row in rows.into_iter().filter(|r| enabled(app, r)) {
823
        let Some(repo) = RepoName::new(&row.owner, &row.name) else { continue };
824
        let b = row.default_bookmark.clone();
825
        let found = app
826
            .repos
827
            .read(&repo, move |r, _| {
828
                let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) };
829
                Ok(Some((st, r.store().get_state(&st)?.heads)))
830
            })
831
            .await;
832
        let Ok(Some((state, heads))) = found else { continue };
833
        if heads.is_empty() {
834
            continue;
835
        }
836
        match start_run_at(app, &row, &row.default_bookmark, &state.to_hex(), &heads, &row.owner, "schedule", None, Some(when)).await {
837
            Ok(Some(_)) => started += 1,
838
            Ok(None) => {}
839
            Err(e) => tracing::warn!("flow: scheduled run of {}/{}: {e}", row.owner, row.name),
840
        }
841
    }
842
    started
843
}
844
845
/// The hub's clock as a [`When`](flow_core::pipeline::When).
846
pub fn when_now() -> flow_core::pipeline::When {
847
    let now = time::OffsetDateTime::now_utc();
848
    flow_core::pipeline::When {
849
        minute: now.minute() as u32,
850
        hour: now.hour() as u32,
851
        day: now.day() as u32,
852
        month: u8::from(now.month()) as u32,
853
        weekday: now.weekday().number_days_from_sunday() as u32,
854
    }
855
}
856
857
/// Start the scheduled runs due each minute, once a minute.
858
pub async fn scheduler(app: Shared) {
859
    let mut last = None;
860
    loop {
861
        tokio::time::sleep(std::time::Duration::from_secs(15)).await;
862
        let minute = time::OffsetDateTime::now_utc().unix_timestamp() / 60;
863
        if last == Some(minute) {
864
            continue;
865
        }
866
        last = Some(minute);
867
        let n = run_due(&app, when_now()).await;
868
        if n > 0 {
869
            tracing::info!("flow: {n} scheduled run(s) started");
870
        }
871
    }
872
}
873
874
/// Look for lost jobs once a minute.
875
pub async fn reaper(app: Shared) {
876
    loop {
877
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
878
        if let Err(e) = reap_lost_jobs(&app).await {
879
            tracing::warn!("flow: looking for lost jobs: {e}");
880
        }
881
    }
882
}
883
884
/// Check a runner's signed receipt for a job of `run` and keep it in the
885
/// repository (Oxygen spec §14.6): the attestation's hex and the runner's
886
/// name, or why it was not kept. It must be the job's receipt, for the
887
/// run's state and repository, signed by the key the runner's token first
888
/// signed with.
889
pub async fn keep_receipt(
890
    app: &Shared,
891
    runner: &db::runners::Runner,
892
    row: &db::repos::RepoRow,
893
    run: &fdb::Run,
894
    receipt: &flow_core::Receipt,
895
    signed_hex: &str,
896
) -> Result<(String, String), String> {
897
    use oxy_core::receipt::SignedReceipt;
898
    let bytes = hex::decode(signed_hex).map_err(|_| "the signed receipt is not hex".to_string())?;
899
    let sr = SignedReceipt::decode(&bytes).map_err(|e| format!("not a signed receipt: {e}"))?;
900
    sr.verify().map_err(|e| format!("it does not verify: {e}"))?;
901
    if sr.receipt.hash().0 != receipt.hash() {
902
        return Err("it signs another receipt than the job's".into());
903
    }
904
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
905
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(heads).encode());
906
    if sr.state() != state {
907
        return Err("it is for another state than the run's".into());
908
    }
909
    let key = sr.runner.hash().to_hex();
910
    let bound = db::runners::bind_key(&app.db, runner.id, &key, &sr.runner.name).await.map_err(|e| e.to_string())?;
911
    if bound != key {
912
        return Err(format!("signed by {}, not the key this runner signs with ({})", &key[..12], &bound[..12.min(bound.len())]));
913
    }
914
    let name = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
915
    let signer = sr.runner.name.clone();
916
    let attest = app.repos.write(&name, move |repo, _| Ok(repo.add_receipt(&sr)?)).await.map_err(|e| e.to_string())?;
917
    Ok((attest.to_hex(), signer))
918
}
919
920
/// A job's wall time, as the pages show it.
921
pub fn duration(j: &fdb::Job) -> String {
922
    match (j.started_ms, j.finished_ms) {
923
        (Some(a), Some(b)) => human_ms(b - a),
924
        (Some(a), None) => human_ms(db::now_ms() - a),
925
        _ => "–".into(),
926
    }
927
}
928
929
pub fn human_ms(ms: i64) -> String {
930
    let ms = ms.max(0);
931
    if ms < 1000 {
932
        format!("{ms} ms")
933
    } else if ms < 60_000 {
934
        format!("{:.1} s", ms as f64 / 1000.0)
935
    } else {
936
        format!("{} min {} s", ms / 60_000, (ms % 60_000) / 1000)
937
    }
938
}
939
940
/// Where a live run stands among the runners' work: what its viewers may
941
/// know of it. Jobs of other repositories are named only when those are
942
/// public; the rest are counted.
943
#[derive(Clone, Debug, Default, PartialEq, Eq)]
944
pub struct QueueInfo {
945
    /// 1-based place of the run's first queued job among all queued jobs.
946
    pub position: Option<usize>,
947
    /// Jobs running now, and queued jobs ahead of the run's first one.
948
    pub running: Vec<String>,
949
    pub ahead: Vec<String>,
950
    /// Of those, jobs in repositories the page may not name.
951
    pub hidden: usize,
952
}
953
954
impl QueueInfo {
955
    /// "3rd in line" and the like.
956
    pub fn place(&self) -> Option<String> {
957
        self.position.map(|n| {
958
            let suffix = match (n % 10, n % 100) {
959
                (1, 11) | (2, 12) | (3, 13) => "th",
960
                (1, _) => "st",
961
                (2, _) => "nd",
962
                (3, _) => "rd",
963
                _ => "th",
964
            };
965
            format!("{n}{suffix} in line")
966
        })
967
    }
968
}
969
970
/// The queue as `run` (of repository `repo_id`) sees it, from `queue`.
971
pub fn queue_info(queue: &[fdb::QueueEntry], repo_id: i64, run_id: i64) -> QueueInfo {
972
    let mut info = QueueInfo::default();
973
    let first = queue.iter().filter(|e| e.status == "queued").position(|e| e.run_id == run_id);
974
    info.position = first.map(|i| i + 1);
975
    let label = |e: &fdb::QueueEntry| format!("{}/{} #{} {}", e.owner, e.name, e.run_number, e.job);
976
    let mut queued_seen = 0;
977
    for e in queue {
978
        if e.run_id == run_id {
979
            continue;
980
        }
981
        let ahead = match e.status.as_str() {
982
            "running" => true,
983
            _ => {
984
                queued_seen += 1;
985
                first.is_some_and(|f| queued_seen <= f)
986
            }
987
        };
988
        if !ahead {
989
            continue;
990
        }
991
        if e.repo_id != repo_id && e.visibility != "public" {
992
            info.hidden += 1;
993
        } else if e.status == "running" {
994
            info.running.push(label(e));
995
        } else {
996
            info.ahead.push(label(e));
997
        }
998
    }
999
    info
1000
}
1001
1002
#[cfg(test)]
1003
mod queue_tests {
1004
    use super::*;
1005
1006
    fn e(job_id: i64, run_id: i64, repo_id: i64, vis: &str, job: &str, status: &str) -> fdb::QueueEntry {
1007
        fdb::QueueEntry {
1008
            job_id,
1009
            run_id,
1010
            repo_id,
1011
            owner: "o".into(),
1012
            name: format!("r{repo_id}"),
1013
            visibility: vis.into(),
1014
            run_number: run_id,
1015
            job: job.into(),
1016
            status: status.into(),
1017
            runner: String::new(),
1018
            spec: String::new(),
1019
            sandbox: String::new(),
1020
        }
1021
    }
1022
1023
    #[test]
1024
    fn a_run_learns_its_place_and_what_is_ahead_of_it() {
1025
        // Running first, then queued in claim order (as `fdb::queue` returns them).
1026
        let q = vec![
1027
            e(1, 10, 1, "public", "build", "running"),
1028
            e(2, 11, 2, "private", "test", "running"),
1029
            e(3, 10, 1, "public", "test", "queued"),
1030
            e(4, 12, 3, "private", "lint", "queued"),
1031
            e(5, 13, 4, "public", "build", "queued"),
1032
            e(6, 13, 4, "public", "test", "queued"),
1033
            e(7, 14, 5, "public", "late", "queued"),
1034
        ];
1035
        let info = queue_info(&q, 4, 13);
1036
        assert_eq!(info.position, Some(3));
1037
        assert_eq!(info.place().as_deref(), Some("3rd in line"));
1038
        assert_eq!(info.running, ["o/r1 #10 build"]);
1039
        assert_eq!(info.ahead, ["o/r1 #10 test"], "queued ahead, not behind");
1040
        assert_eq!(info.hidden, 2, "private repositories are counted, not named");
1041
        // A private repository's own jobs are named to its viewers.
1042
        assert_eq!(queue_info(&q, 3, 12).ahead, ["o/r1 #10 test"]);
1043
        assert_eq!(queue_info(&q, 3, 12).position, Some(2));
1044
        // A run with nothing queued has no place.
1045
        assert_eq!(queue_info(&q, 1, 99).position, None);
1046
        let ordinal = |n| QueueInfo { position: Some(n), ..Default::default() }.place().unwrap();
1047
        assert_eq!(
1048
            [ordinal(1), ordinal(2), ordinal(11), ordinal(22), ordinal(113)],
1049
            ["1st in line", "2nd in line", "11th in line", "22nd in line", "113th in line"]
1050
        );
1051
    }
1052
}