OXYGENHUB
◆ v0.4.0 oxygenhub / crates / hub / src / flow.rs
46.5 KiB · Rust · last changed A runner busy with its only job stays online 1 month ago source blame history raw
42fa9554 Andy McCall ↶ 1
//! OxygenFlow on the hub (hub spec §13.2): runs started by pushes, the
2
//! runner API, job tokens and the checks that mirror jobs. Pipelines are
3
//! parsed by `flow-core`, the same code `flow check` runs.
4
5
use crate::api::ApiError;
6
use crate::db::{self, flow as fdb};
7
use crate::repos::RepoName;
8
use crate::Shared;
9
use axum::extract::{Path, State};
10
use axum::http::{HeaderMap, StatusCode};
11
use axum::response::{IntoResponse, Response};
12
use axum::Json;
13
use flow_core::api::{Claim, ClaimRequest, Finish, LogBatch, MAX_RECORDS, MAX_RECORD_TEXT};
14
use flow_core::{Pipeline, PIPELINE_PATH};
15
use oxy_core::hash::ChangeHash;
16
17
/// Prefix of job tokens, so the wire protocol can tell them from personal tokens.
18
pub const JOB_TOKEN_PREFIX: &str = "flowjob_";
19
/// A job token is refused this long after its job started, whatever its state.
20
pub const JOB_TOKEN_MAX_AGE_MS: i64 = 2 * 3600 * 1000;
21
9008bf6c Andy McCall ↶ 22
/// Whether pushes to this repository start runs: its own setting when a
23
/// site administrator made one, else hub.toml's `[flow] repos`.
24
pub fn enabled(app: &Shared, row: &db::repos::RepoRow) -> bool {
25
    match row.flow.as_str() {
26
        "on" => true,
27
        "off" => false,
28
        _ => app.config.flow.allows(&row.owner, &row.name),
29
    }
30
}
31
42fa9554 Andy McCall ↶ 32
/// A bookmark moved by a push: start a run when the repository is allowed
33
/// and the new state has a pipeline.
34
pub async fn on_push(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, state: &str, heads: &[ChangeHash], actor: &str) {
c889911b Andy McCall ↶ 35
    if heads.is_empty() || !enabled(app, row) {
42fa9554 Andy McCall ↶ 36
        return;
37
    }
c889911b Andy McCall ↶ 38
    // A pushed tag runs the jobs whose `on` names it (oxygenflow#24).
39
    let started = match bookmark.strip_prefix("tags/") {
40
        Some(tag) => start_run_with(app, row, bookmark, state, heads, actor, &format!("tag:{tag}"), None).await,
41
        None => {
42
            let touched = touched_since_last_run(app, row, bookmark, heads).await;
745d225d Andy McCall ↶ 43
            let started = start_run_scoped(app, row, bookmark, state, heads, actor, "push", touched.as_deref()).await;
44
            if let Ok(Some(n)) = started {
45
                supersede(app, row, bookmark, n).await;
46
            }
47
            started
c889911b Andy McCall ↶ 48
        }
49
    };
50
    if let Err(e) = started {
1edac4b6 Andy McCall ↶ 51
        tracing::warn!("flow: starting a run of {}/{}: {e}", row.owner, row.name);
52
    }
53
}
54
55
/// "Run pipeline" (hub spec §13.2): run a bookmark's current state without
56
/// moving it. Returns the new run's number.
57
pub async fn run_bookmark(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, actor: &str) -> Result<i64, String> {
58
    if !enabled(app, row) {
59
        return Err("OxygenFlow is not enabled for this repository".into());
60
    }
61
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
62
    let b = bookmark.to_string();
63
    let found = app
64
        .repos
65
        .read(&repo, move |r, _| {
66
            let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) };
67
            Ok(Some((st, r.store().get_state(&st)?.heads)))
68
        })
69
        .await
70
        .map_err(|e| e.to_string())?;
71
    let Some((state, heads)) = found else { return Err(format!("no bookmark {bookmark}")) };
72
    if heads.is_empty() {
73
        return Err(format!("{bookmark} has no changes yet"));
74
    }
75
    start_run(app, row, bookmark, &state.to_hex(), &heads, actor, "manual")
76
        .await?
77
        .ok_or_else(|| format!("{bookmark} has no {PIPELINE_PATH}"))
78
}
79
4f692533 Andy McCall ↶ 80
/// "Run pipeline" on one committed change: the state whose only head is
81
/// `change`, whatever bookmarks hold it. Returns the new run's number.
82
pub async fn run_change(app: &Shared, row: &db::repos::RepoRow, change: &str, actor: &str) -> Result<i64, String> {
83
    if !enabled(app, row) {
84
        return Err("OxygenFlow is not enabled for this repository".into());
85
    }
86
    let h = ChangeHash::from_hex(change.trim().trim_start_matches('#')).ok_or("give the full hash of a change")?;
87
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
df15b747 Andy McCall ↶ 88
    let known = app.repos.read(&repo, move |r, _| Ok(r.store().get_change_meta(&h).is_ok())).await.map_err(|e| e.to_string())?;
4f692533 Andy McCall ↶ 89
    if !known {
90
        return Err(format!("no change {} in this repository", &change.trim()[..12.min(change.trim().len())]));
91
    }
92
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(vec![h]).encode());
93
    start_run(app, row, "", &state.to_hex(), &[h], actor, "manual")
94
        .await?
95
        .ok_or_else(|| format!("that change's state has no {PIPELINE_PATH}"))
96
}
97
1edac4b6 Andy McCall ↶ 98
/// Re-run a finished run: the same state, bookmark and pipeline, as a new
99
/// run whose checks replace the old ones. Returns the new run's number.
100
pub async fn rerun(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<i64, String> {
101
    if !enabled(app, row) {
102
        return Err("OxygenFlow is not enabled for this repository".into());
103
    }
104
    if run.live() {
105
        return Err(format!("run #{} is still {}", run.number, run.status));
106
    }
107
    let heads: Vec<ChangeHash> = run.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
108
    if heads.is_empty() {
109
        return Err("the run names no changes".into());
110
    }
c889911b Andy McCall ↶ 111
    // A rerun repeats its run's event: the jobs it had are the jobs it gets.
112
    let trigger = format!("rerun:{}:{}", run.number, original_event(run));
113
    start_run(app, row, &run.bookmark, &run.state, &heads, actor, &trigger)
1edac4b6 Andy McCall ↶ 114
        .await?
115
        .ok_or_else(|| format!("that state has no {PIPELINE_PATH}"))
745d225d Andy McCall ↶ 116
}
117
118
/// Cancel a run still going (oxygenflow#32), its checks with it.
119
pub async fn cancel(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<(), String> {
120
    if !run.live() {
121
        return Err(format!("run #{} is already {}", run.number, run.status));
122
    }
123
    cancel_with(app, run, &format!("cancelled by {actor}")).await.map_err(|e| {
124
        tracing::warn!("flow: cancelling run #{} of {}/{}: {e}", run.number, row.owner, row.name);
125
        "the run could not be cancelled".to_string()
126
    })
127
}
128
129
async fn cancel_with(app: &Shared, run: &fdb::Run, why: &str) -> sqlx::Result<()> {
130
    let changed = fdb::cancel_run(&app.db, run.id, why).await?;
131
    report_changed(app, run.id, changed).await
c889911b Andy McCall ↶ 132
}
133
745d225d Andy McCall ↶ 134
/// A push makes the push runs still going on its bookmark moot: they are
135
/// cancelled, and the new run's scope reaches back past them.
136
async fn supersede(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, number: i64) {
137
    let Ok(older) = fdb::live_push_runs_before(&app.db, row.id, bookmark, number).await else { return };
138
    for run in older {
139
        if let Err(e) = cancel_with(app, &run, &format!("superseded by run #{number}")).await {
140
            tracing::warn!("flow: superseding run #{} of {}/{}: {e}", run.number, row.owner, row.name);
141
        }
142
    }
143
}
144
c889911b Andy McCall ↶ 145
/// The event a run's trigger stands for: `push`, `tag:NAME`, `manual`,
146
/// `schedule`, and `rerun:N:EVENT` repeating EVENT.
147
fn event_of<'a>(trigger: &'a str, bookmark: &'a str) -> flow_core::pipeline::Event<'a> {
148
    use flow_core::pipeline::Event;
149
    let t = match trigger.strip_prefix("rerun:") {
150
        Some(rest) => rest.split_once(':').map(|(_, e)| e).unwrap_or("manual"),
151
        None => trigger,
152
    };
153
    if let Some(tag) = t.strip_prefix("tag:") {
154
        Event::Tag(tag)
155
    } else if t == "push" {
156
        Event::Push(bookmark)
157
    } else if t == "schedule" {
158
        Event::Schedule
159
    } else {
160
        Event::Manual
161
    }
1edac4b6 Andy McCall ↶ 162
}
163
c889911b Andy McCall ↶ 164
/// The event to repeat when rerunning `run`.
165
fn original_event(run: &fdb::Run) -> String {
166
    let t = run.trigger.as_str();
167
    match t.strip_prefix("rerun:") {
168
        Some(rest) => rest.split_once(':').map(|(_, e)| e.to_string()).unwrap_or_else(|| "manual".into()),
169
        None => t.to_string(),
170
    }
171
}
172
d7ce41ac Andy McCall ↶ 173
/// The paths the changes a push brought touch: the changes in `heads`'
174
/// state that the bookmark's previous run did not have, and the files their
175
/// edits and tree operations name, as the new state places them. `None`
176
/// (run every job) for the bookmark's first run, or when it cannot be told.
177
async fn touched_since_last_run(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, heads: &[ChangeHash]) -> Option<Vec<String>> {
178
    let last = fdb::latest_on(&app.db, row.id, bookmark).await.ok()??;
179
    let old: Vec<ChangeHash> = last.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
180
    if old.is_empty() {
181
        return None;
182
    }
183
    let repo = RepoName::new(&row.owner, &row.name)?;
184
    let new = heads.to_vec();
185
    app.repos
186
        .read(&repo, move |r, cache| {
187
            let fresh = r.negotiate(&old, &new)?;
188
            let loaded = cache.load(r, &new)?;
189
            let by_node: std::collections::HashMap<oxy_core::ids::NodeId, &String> =
190
                loaded.snapshot.files.iter().map(|(p, f)| (f.node, p)).collect();
191
            let mut paths = std::collections::BTreeSet::new();
192
            for h in fresh {
193
                let (c, _) = r.store().get_change_lazy(&h)?;
194
                let nodes = c.edits.iter().map(|e| e.node()).chain(c.tree_ops.iter().map(|o| o.node()));
195
                for n in nodes {
196
                    if let Some(p) = by_node.get(&n) {
197
                        paths.insert((*p).clone());
198
                    }
199
                }
200
            }
201
            Ok(paths.into_iter().collect::<Vec<String>>())
202
        })
203
        .await
204
        .ok()
205
}
206
207
/// Whether a job runs for a push that touched `touched`: always without a
208
/// scope or without knowing what was touched, else when a path matches.
209
fn in_scope(scope: &[String], touched: Option<&[String]>) -> bool {
210
    let Some(touched) = touched else { return true };
211
    if scope.is_empty() {
212
        return true;
213
    }
214
    let mut b = globset::GlobSetBuilder::new();
215
    for g in scope {
216
        match globset::Glob::new(g) {
217
            Ok(g) => {
218
                b.add(g);
219
            }
220
            Err(_) => return true,
221
        }
222
    }
223
    let Ok(set) = b.build() else { return true };
224
    touched.iter().any(|p| set.is_match(p))
225
}
226
227
/// [`start_run`] for a push: jobs whose `scope` the push did not touch are
228
/// left out of the run, unless a job that runs depends on them.
229
#[allow(clippy::too_many_arguments)]
230
pub async fn start_run_scoped(
231
    app: &Shared,
232
    row: &db::repos::RepoRow,
233
    bookmark: &str,
234
    state: &str,
235
    heads: &[ChangeHash],
236
    actor: &str,
237
    trigger: &str,
238
    touched: Option<&[String]>,
239
) -> Result<Option<i64>, String> {
240
    start_run_with(app, row, bookmark, state, heads, actor, trigger, touched).await
241
}
242
1edac4b6 Andy McCall ↶ 243
/// Start a run of the pipeline in the state `heads` make (whose state
244
/// hash is `state`), recorded on `bookmark`: for a push, "Run pipeline"
245
/// (`trigger` = `manual`) and re-runs (`rerun:N`). `actor` is the account
246
/// whose access the run's subrepo reads are checked against. Returns the
247
/// run's number, or `None` when the state has no pipeline. The caller
248
/// checks that Flow is enabled.
249
pub async fn start_run(
250
    app: &Shared,
251
    row: &db::repos::RepoRow,
252
    bookmark: &str,
253
    state: &str,
254
    heads: &[ChangeHash],
255
    actor: &str,
256
    trigger: &str,
257
) -> Result<Option<i64>, String> {
d7ce41ac Andy McCall ↶ 258
    start_run_with(app, row, bookmark, state, heads, actor, trigger, None).await
259
}
260
261
#[allow(clippy::too_many_arguments)]
262
async fn start_run_with(
263
    app: &Shared,
264
    row: &db::repos::RepoRow,
265
    bookmark: &str,
266
    state: &str,
267
    heads: &[ChangeHash],
268
    actor: &str,
269
    trigger: &str,
270
    touched: Option<&[String]>,
271
) -> Result<Option<i64>, String> {
348f8c4b Andy McCall ↶ 272
    start_run_at(app, row, bookmark, state, heads, actor, trigger, touched, None).await
273
}
274
275
/// [`start_run_with`], for a scheduled run due at `at` on the hub's clock.
276
#[allow(clippy::too_many_arguments)]
277
async fn start_run_at(
278
    app: &Shared,
279
    row: &db::repos::RepoRow,
280
    bookmark: &str,
281
    state: &str,
282
    heads: &[ChangeHash],
283
    actor: &str,
284
    trigger: &str,
285
    touched: Option<&[String]>,
286
    at: Option<flow_core::pipeline::When>,
287
) -> Result<Option<i64>, String> {
1edac4b6 Andy McCall ↶ 288
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
42fa9554 Andy McCall ↶ 289
    let h = heads.to_vec();
290
    let text = app
291
        .repos
292
        .read(&repo, move |r, cache| {
293
            let loaded = cache.load(r, &h)?;
294
            Ok(match loaded.snapshot.files.get(PIPELINE_PATH).map(|f| &f.content) {
295
                Some(oxy_core::state::FileContent::Text { bytes, .. }) => Some(String::from_utf8_lossy(bytes).into_owned()),
296
                _ => None,
297
            })
298
        })
1edac4b6 Andy McCall ↶ 299
        .await
300
        .map_err(|e| format!("reading the pipeline: {e}"))?;
301
    let Some(text) = text else { return Ok(None) };
42fa9554 Andy McCall ↶ 302
    let mut hex: Vec<String> = heads.iter().map(|h| h.to_hex()).collect();
303
    hex.sort();
304
    let heads_s = hex.join("+");
1edac4b6 Andy McCall ↶ 305
    let base = db::flow::NewRun { repo_id: row.id, bookmark, state, heads: &heads_s, pipeline_hash: "", actor, trigger, error: None };
306
    let id = match Pipeline::parse(&text) {
42fa9554 Andy McCall ↶ 307
        Err(problems) => {
308
            let msg = problems.to_string();
1edac4b6 Andy McCall ↶ 309
            db::flow::create_run(&app.db, db::flow::NewRun { error: Some(&msg), ..base }, &[]).await.map_err(|e| e.to_string())?
42fa9554 Andy McCall ↶ 310
        }
311
        Ok(p) => {
312
            let depths = p.depths();
d7ce41ac Andy McCall ↶ 313
            // Jobs out of the push's scope stay out, unless a kept job
314
            // needs them.
c889911b Andy McCall ↶ 315
            // The jobs this run's event starts (`on`), out of those its
316
            // push touched (`scope`); then what they depend on.
348f8c4b Andy McCall ↶ 317
            let event = match at {
318
                Some(w) => flow_core::pipeline::Event::ScheduleAt(w),
319
                None => event_of(trigger, bookmark),
320
            };
c889911b Andy McCall ↶ 321
            let mut keep: std::collections::HashSet<String> = p
322
                .order()
323
                .into_iter()
324
                .filter(|j| p.on_of(j).matches(event) && in_scope(&j.scope, touched))
325
                .map(|j| j.name.clone())
326
                .collect();
d7ce41ac Andy McCall ↶ 327
            loop {
328
                let needed: Vec<String> = p
329
                    .order()
330
                    .into_iter()
331
                    .filter(|j| keep.contains(&j.name))
332
                    .flat_map(|j| j.depends_on.iter().cloned())
333
                    .filter(|d| !keep.contains(d))
334
                    .collect();
335
                if needed.is_empty() {
336
                    break;
337
                }
338
                keep.extend(needed);
339
            }
42fa9554 Andy McCall ↶ 340
            let jobs: Vec<fdb::NewJob> = p
341
                .order()
342
                .into_iter()
d7ce41ac Andy McCall ↶ 343
                .filter(|j| keep.contains(&j.name))
42fa9554 Andy McCall ↶ 344
                .map(|j| {
345
                    let sb = &p.sandboxes[&j.sandbox];
346
                    fdb::NewJob {
347
                        name: j.name.clone(),
348
                        depth: depths[j.name.as_str()] as i64,
349
                        depends: j.depends_on.clone(),
350
                        spec: serde_json::to_string(j).unwrap_or_default(),
351
                        sandbox: serde_json::to_string(sb).unwrap_or_default(),
352
                        runtime: sb.runtime.as_str().into(),
353
                    }
354
                })
355
                .collect();
c889911b Andy McCall ↶ 356
            if jobs.is_empty() {
357
                // Nothing this event starts (or this push touched).
d7ce41ac Andy McCall ↶ 358
                return Ok(None);
359
            }
42fa9554 Andy McCall ↶ 360
            let hash = hex::encode(p.hash());
1edac4b6 Andy McCall ↶ 361
            let id =
362
                db::flow::create_run(&app.db, db::flow::NewRun { pipeline_hash: &hash, ..base }, &jobs).await.map_err(|e| e.to_string())?;
363
            if let Ok(Some(run)) = fdb::run_by_id(&app.db, id).await {
42fa9554 Andy McCall ↶ 364
                for j in &jobs {
a0d4bbfb Andy McCall ↶ 365
                    report_check(app, row, state, run.number, &j.name, "queued", "waiting for a runner").await;
42fa9554 Andy McCall ↶ 366
                }
367
            }
368
            id
369
        }
370
    };
1edac4b6 Andy McCall ↶ 371
    let run = fdb::run_by_id(&app.db, id).await.map_err(|e| e.to_string())?.ok_or("the run vanished")?;
372
    Ok(Some(run.number))
42fa9554 Andy McCall ↶ 373
}
374
a0d4bbfb Andy McCall ↶ 375
/// Record a job's check; it links to the run's page by the run's number.
376
async fn report_check(app: &Shared, row: &db::repos::RepoRow, state: &str, run_number: i64, job: &str, status: &str, summary: &str) {
377
    let url = format!("{}/{}/{}/flow/{run_number}", app.config.base_url(), row.owner, row.name);
42fa9554 Andy McCall ↶ 378
    let name = format!("flow/{job}");
379
    let c = db::hooks::upsert_check(
380
        &app.db,
381
        row.id,
382
        db::hooks::CheckReport { state, name: &name, status, summary, url: &url, reporter: None, seal: "" },
383
    )
384
    .await;
385
    if let Ok(c) = c {
386
        let api = crate::api::check_of(&c);
387
        app.events.proposal(row.id, hub_api::ProposalEvent { number: 0, kind: "check".into(), check: Some(api.clone()) });
388
        crate::hooks::emit(app, row, "check", "flow", serde_json::json!({ "check": api })).await;
389
    }
390
}
391
392
/// The check status a job's status shows as.
393
fn check_status(job_status: &str) -> &'static str {
394
    match job_status {
395
        "queued" => "queued",
396
        "running" => "running",
397
        "passed" => "passed",
398
        "skipped" => "skipped",
745d225d Andy McCall ↶ 399
        "cancelled" => "cancelled",
42fa9554 Andy McCall ↶ 400
        _ => "failed",
401
    }
402
}
403
b6b7e4cf Andy McCall ↶ 404
/// The runner a bearer token belongs to: a registered runner, or one
405
/// configured with a hub.toml token (tracked from its first request). A
406
/// disabled runner is refused.
407
async fn runner_auth(app: &Shared, headers: &HeaderMap) -> Result<db::runners::Runner, ApiError> {
42fa9554 Andy McCall ↶ 408
    let Some(t) = oxy_server::bearer_token(headers) else {
409
        return Err(ApiError(StatusCode::UNAUTHORIZED, "a runner token is required".into()));
b6b7e4cf Andy McCall ↶ 410
    };
411
    let hash = blake3::hash(t.as_bytes()).to_hex().to_string();
412
    let runner = match db::runners::by_token(&app.db, &hash).await? {
413
        Some(r) => r,
414
        None if app.config.flow.runner_tokens.iter().any(|x| crate::auth::mac_equal(x, t)) => {
415
            db::runners::configured(&app.db, &hash, "runner").await?
416
        }
417
        None => return Err(ApiError(StatusCode::UNAUTHORIZED, "not a runner token".into())),
42fa9554 Andy McCall ↶ 418
    };
b6b7e4cf Andy McCall ↶ 419
    if runner.state == "disabled" {
420
        return Err(ApiError(StatusCode::FORBIDDEN, format!("runner {} is disabled on this hub", runner.name)));
42fa9554 Andy McCall ↶ 421
    }
b6b7e4cf Andy McCall ↶ 422
    Ok(runner)
42fa9554 Andy McCall ↶ 423
}
424
b6b7e4cf Andy McCall ↶ 425
/// `POST /api/v1/flow/runners/register`: a one-time registration token
426
/// from Admin → Runners buys a runner credential, binding the runner's
427
/// signing identity (hub spec §13.2).
428
pub async fn register(
429
    State(app): State<Shared>,
430
    Json(req): Json<flow_core::api::Register>,
431
) -> Result<Json<flow_core::api::Registered>, ApiError> {
432
    let bytes = hex::decode(req.identity.trim()).map_err(|_| ApiError(StatusCode::BAD_REQUEST, "identity: not hex".into()))?;
433
    let identity = oxy_core::identity::Identity::decode(&bytes).map_err(|e| ApiError(StatusCode::BAD_REQUEST, format!("identity: {e}")))?;
434
    let name: String = req.name.chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')).take(64).collect();
435
    if name.is_empty() {
436
        return Err(ApiError(StatusCode::BAD_REQUEST, "give the runner a name of letters, digits, - _ or .".into()));
437
    }
438
    let credential = format!("flowrunner_{}", crate::auth::random_hex(32));
439
    let hash = blake3::hash(credential.as_bytes()).to_hex().to_string();
440
    let reg = blake3::hash(req.token.trim().as_bytes()).to_hex().to_string();
441
    let runner =
442
        db::runners::register(&app.db, &reg, &name, &hash, &identity.hash().to_hex(), &identity.name, "").await?.ok_or_else(|| {
443
            ApiError(StatusCode::FORBIDDEN, "that registration token is unknown, used or expired; create one on Admin → Runners".into())
444
        })?;
445
    db::runners::seen(&app.db, runner.id, None, Some(&req.info)).await?;
446
    tracing::info!("flow: runner {} registered (key {})", runner.name, &runner.key[..12]);
447
    Ok(Json(flow_core::api::Registered { id: runner.id, name: runner.name, credential }))
448
}
449
42fa9554 Andy McCall ↶ 450
/// `POST /api/v1/flow/claim`.
451
pub async fn claim(State(app): State<Shared>, headers: HeaderMap, Json(req): Json<ClaimRequest>) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 452
    let me = runner_auth(&app, &headers).await?;
453
    let claimed: String = req.runner.chars().filter(|c| !c.is_control()).take(64).collect();
454
    db::runners::seen(&app.db, me.id, Some(&claimed), req.info.as_ref()).await?;
eef677c4 Andy McCall ↶ 455
    if let Some(holding) = &req.holding {
456
        end_jobs_not_held(&app, &me, holding).await?;
457
    }
b6b7e4cf Andy McCall ↶ 458
    // A draining runner finishes what it has and takes nothing new.
459
    if me.state == "draining" {
460
        return Ok(StatusCode::NO_CONTENT.into_response());
461
    }
462
    let runner = if me.source == "registered" { me.name.clone() } else { claimed };
42fa9554 Andy McCall ↶ 463
    let secret = format!("{JOB_TOKEN_PREFIX}{}", crate::auth::random_hex(32));
b0612e7d Andy McCall ↶ 464
    let room = fdb::Room {
465
        memory_mib: req.free_memory_mib,
466
        cpus: req.free_cpus,
467
        images: req.images.as_ref().map(|i| i.iter().map(|h| h.to_ascii_lowercase()).collect()),
468
    };
c4c87f48 Andy McCall ↶ 469
    // The labels it reports now, or those it registered with.
470
    let labels: Vec<String> = match &req.info {
471
        Some(i) => i.labels.clone(),
472
        None => me.label_list().into_iter().map(String::from).collect(),
473
    };
474
    let Some(job) =
475
        fdb::claim_within(&app.db, &req.runtimes, &runner, &crate::auth::token_hash(&secret), room, &me.scope(), &labels).await?
476
    else {
42fa9554 Andy McCall ↶ 477
        return Ok(StatusCode::NO_CONTENT.into_response());
478
    };
b6b7e4cf Andy McCall ↶ 479
    db::runners::set_job_runner(&app.db, job.id, me.id).await?;
42fa9554 Andy McCall ↶ 480
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
481
    let row =
482
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
a0d4bbfb Andy McCall ↶ 483
    report_check(&app, &row, &run.state, run.number, &job.name, "running", &format!("running on {runner}")).await;
42fa9554 Andy McCall ↶ 484
    let bad = |e: serde_json::Error| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}"));
5dead1f4 Andy McCall ↶ 485
    // The job's limit: its own timeout, else its sandbox's, else the site
486
    // default, never past the site maximum; recorded for the reaper.
487
    let limits = limits(&app).await;
488
    let spec: flow_core::pipeline::Job = serde_json::from_str(&job.spec).map_err(bad)?;
489
    let sandbox: flow_core::pipeline::Sandbox = serde_json::from_str(&job.sandbox).map_err(bad)?;
490
    let timeout_s = spec.timeout_s.or(sandbox.timeout_s).unwrap_or(limits.default_s).min(limits.max_s).max(60);
491
    fdb::set_job_timeout(&app.db, job.id, timeout_s).await?;
42fa9554 Andy McCall ↶ 492
    let claim = Claim {
493
        job_id: job.id,
494
        run_id: run.id,
495
        repo: format!("{}/{}", row.owner, row.name),
496
        clone_url: format!("{}/{}/{}", app.config.base_url(), row.owner, row.name),
497
        token: secret,
498
        heads: run.head_list().into_iter().map(String::from).collect(),
499
        pipeline_hash: run.pipeline_hash.clone(),
5dead1f4 Andy McCall ↶ 500
        job: spec,
501
        sandbox,
502
        timeout_s,
fd3db644 Andy McCall ↶ 503
        subrepos: vec![],
9a065bc0 Andy McCall ↶ 504
        state: run.state.clone(),
505
        bookmark: run.bookmark.clone(),
506
        run_number: run.number,
42fa9554 Andy McCall ↶ 507
    };
fd3db644 Andy McCall ↶ 508
    let mut claim = claim;
afd0be38 Andy McCall ↶ 509
    let (inputs, ids, left_out) = subrepo_inputs(&app, &row, &run).await;
fd3db644 Andy McCall ↶ 510
    if !ids.is_empty() {
511
        fdb::set_also_read(&app.db, job.id, &ids).await?;
512
    }
afd0be38 Andy McCall ↶ 513
    // Say at the top of the job's log what it will not have (oxygenhub#56),
514
    // rather than let it fail later on missing files.
515
    if !left_out.is_empty() {
516
        let records: Vec<flow_core::api::Record> =
517
            left_out.into_iter().map(|text| flow_core::api::Record { t: 0, stream: "flow".into(), text }).collect();
518
        fdb::append_log(&app.db, job.id, &records, MAX_RECORDS as i64).await?;
519
    }
fd3db644 Andy McCall ↶ 520
    claim.subrepos = inputs;
42fa9554 Andy McCall ↶ 521
    Ok(Json(claim).into_response())
fd3db644 Andy McCall ↶ 522
}
523
25091e5c Andy McCall ↶ 524
/// The Oxygen subrepo pins in `name`'s state `heads`: (path, repository id, pinned heads).
525
async fn pins_in(
526
    app: &Shared,
527
    name: crate::repos::RepoName,
528
    heads: Vec<oxy_core::hash::ChangeHash>,
529
) -> Vec<(String, [u8; 16], Vec<oxy_core::hash::ChangeHash>)> {
530
    app.repos
fd3db644 Andy McCall ↶ 531
        .read(&name, move |repo, cache| {
532
            let loaded = cache.load(repo, &heads)?;
533
            let mut out = Vec::new();
534
            for (path, f) in &loaded.snapshot.files {
535
                if f.kind != oxy_core::objects::NodeKind::Subrepo {
536
                    continue;
537
                }
538
                if let oxy_core::state::FileContent::Opaque { blobs } = &f.content {
539
                    if let Some(b) = blobs.first() {
540
                        if let Ok(oxy_core::subrepo::Pin::Oxy { repo: id, heads, .. }) = repo
541
                            .read_blob(b)
542
                            .map_err(crate::repos::RepoError::Oxy)
543
                            .and_then(|bytes| oxy_core::subrepo::Pin::decode(&bytes).map_err(|e| crate::repos::RepoError::Oxy(e.into())))
544
                        {
545
                            out.push((path.clone(), id, heads));
546
                        }
547
                    }
548
                }
549
            }
550
            Ok(out)
551
        })
552
        .await
25091e5c Andy McCall ↶ 553
        .unwrap_or_default()
554
}
555
556
/// How deep subrepos of subrepos are followed.
557
const MAX_SUBREPO_DEPTH: usize = 3;
558
559
/// The Oxygen subrepos the run's state pins, and those their states pin in
560
/// turn (parents first, paths joined), that live on this hub and that the
561
/// run's pusher may read: what the job's workspace needs beside its own
afd0be38 Andy McCall ↶ 562
/// tree. A pin the hub cannot serve is left out, with what it pins, and
563
/// the third list says why, for the job's log.
564
async fn subrepo_inputs(
565
    app: &Shared,
566
    row: &db::repos::RepoRow,
567
    run: &fdb::Run,
568
) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>, Vec<String>) {
569
    let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![], vec![]) };
25091e5c Andy McCall ↶ 570
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
fd3db644 Andy McCall ↶ 571
    let pusher = db::accounts::account_by_name(&app.db, &run.actor).await.ok().flatten();
25091e5c Andy McCall ↶ 572
    let rows = db::repos::list(&app.db).await.unwrap_or_default();
fd3db644 Andy McCall ↶ 573
    let mut inputs = Vec::new();
574
    let mut ids = Vec::new();
afd0be38 Andy McCall ↶ 575
    let mut left_out = Vec::new();
576
    let actor = if run.actor.is_empty() { "the run's starter".to_string() } else { run.actor.clone() };
25091e5c Andy McCall ↶ 577
    // (path prefix, repository, heads, depth), parents before children.
578
    let mut queue = std::collections::VecDeque::from([(String::new(), name, heads, 0usize)]);
579
    while let Some((prefix, name, heads, depth)) = queue.pop_front() {
580
        if depth == MAX_SUBREPO_DEPTH {
581
            continue;
582
        }
583
        for (path, id, pinned) in pins_in(app, name, heads).await {
584
            let path = if prefix.is_empty() { path } else { format!("{prefix}/{path}") };
afd0be38 Andy McCall ↶ 585
            let mut hosted = false;
25091e5c Andy McCall ↶ 586
            for other in &rows {
587
                let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue };
588
                let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false);
589
                if !matches {
590
                    continue;
591
                }
afd0be38 Andy McCall ↶ 592
                hosted = true;
25091e5c Andy McCall ↶ 593
                let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten();
594
                if crate::auth::roles::allows(role, crate::auth::Access::Read) {
595
                    inputs.push(flow_core::api::SubrepoInput {
596
                        path: path.clone(),
597
                        repo: format!("{}/{}", other.owner, other.name),
598
                        clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name),
599
                        heads: pinned.iter().map(|h| h.to_hex()).collect(),
600
                    });
601
                    if !ids.contains(&other.id) {
602
                        ids.push(other.id);
603
                    }
604
                    queue.push_back((path.clone(), other_name, pinned.clone(), depth + 1));
afd0be38 Andy McCall ↶ 605
                } else {
606
                    left_out.push(format!(
607
                        "subrepo {path} ({}/{}) is left out: {actor} cannot read it, so the job has no files there",
608
                        other.owner, other.name
609
                    ));
25091e5c Andy McCall ↶ 610
                }
611
                break;
afd0be38 Andy McCall ↶ 612
            }
613
            if !hosted {
614
                left_out.push(format!("subrepo {path} is left out: the repository it pins is not on this hub"));
fd3db644 Andy McCall ↶ 615
            }
616
        }
617
    }
afd0be38 Andy McCall ↶ 618
    (inputs, ids, left_out)
42fa9554 Andy McCall ↶ 619
}
620
621
async fn running_job(app: &Shared, id: i64) -> Result<fdb::Job, ApiError> {
622
    match fdb::job(&app.db, id).await? {
623
        Some(j) if j.status == "running" => Ok(j),
624
        Some(_) => Err(ApiError(StatusCode::CONFLICT, "this job is not running".into())),
625
        None => Err(ApiError(StatusCode::NOT_FOUND, "no such job".into())),
626
    }
627
}
628
629
/// `POST /api/v1/flow/jobs/{id}/log`.
630
pub async fn log(
631
    State(app): State<Shared>,
632
    headers: HeaderMap,
633
    Path(id): Path<i64>,
634
    Json(mut batch): Json<LogBatch>,
635
) -> Result<StatusCode, ApiError> {
d94449e4 Andy McCall ↶ 636
    let me = runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 637
    let job = running_job(&app, id).await?;
d94449e4 Andy McCall ↶ 638
    // A runner busy with its only job claims nothing, so its log and
639
    // finish calls are what keep it online (oxygenhub#78).
640
    db::runners::seen(&app.db, me.id, None, None).await?;
745d225d Andy McCall ↶ 641
    // An empty batch asks only whether the job still runs: a cancelled
642
    // one answers 409 and its runner stops it (oxygenflow#32).
643
    if batch.records.is_empty() {
644
        return Ok(StatusCode::NO_CONTENT);
645
    }
42fa9554 Andy McCall ↶ 646
    for r in &mut batch.records {
647
        if r.text.len() > MAX_RECORD_TEXT {
648
            let mut cut = MAX_RECORD_TEXT;
649
            while !r.text.is_char_boundary(cut) {
650
                cut -= 1;
651
            }
652
            r.text.truncate(cut);
653
        }
654
        if !matches!(r.stream.as_str(), "out" | "err" | "console" | "flow") {
655
            r.stream = "console".into();
656
        }
657
    }
658
    let kept = fdb::append_log(&app.db, job.id, &batch.records, MAX_RECORDS as i64).await?;
659
    if kept < batch.records.len() && job.log_count < MAX_RECORDS as i64 {
660
        tracing::info!("flow: job {} reached the log limit", job.id);
661
    }
662
    Ok(StatusCode::NO_CONTENT)
663
}
664
a3c8288b Andy McCall ↶ 665
/// Largest artifact the hub keeps; the runner refuses larger ones first.
666
pub const MAX_ARTIFACT_BYTES: usize = 512 << 20;
667
668
/// `POST /api/v1/flow/jobs/{id}/artifacts/{name}`: the bytes of an artifact
669
/// the job declares (OxygenFlow spec §5.2), kept as a blob of the run's
670
/// repository. Answers `{"blob"}`.
671
pub async fn artifact(
672
    State(app): State<Shared>,
673
    headers: HeaderMap,
674
    Path((id, name)): Path<(i64, String)>,
675
    body: axum::body::Bytes,
676
) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 677
    runner_auth(&app, &headers).await?;
a3c8288b Andy McCall ↶ 678
    let job = running_job(&app, id).await?;
679
    let spec: flow_core::Job =
680
        serde_json::from_str(&job.spec).map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}")))?;
681
    let Some(out) = spec.outputs.iter().find(|o| o.name == name) else {
682
        return Err(ApiError(StatusCode::BAD_REQUEST, format!("job {} declares no artifact {name:?}", job.name)));
683
    };
684
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
685
    let row =
686
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
687
    let repo = RepoName::new(&row.owner, &row.name).ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "bad repository name".into()))?;
688
    let content = blake3::hash(&body).to_hex().to_string();
689
    let size = body.len() as i64;
690
    let blob = app
691
        .repos
692
        .write(&repo, move |r, _| Ok(r.store_blob(&body)?))
693
        .await
694
        .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("keep the artifact: {e}")))?;
695
    let a = fdb::Artifact { job_id: job.id, name, path: out.path.clone(), blob: blob.to_hex(), size, content, created_ms: db::now_ms() };
696
    fdb::add_artifact(&app.db, &a).await?;
697
    Ok(Json(serde_json::json!({ "blob": a.blob })).into_response())
698
}
699
42fa9554 Andy McCall ↶ 700
/// `POST /api/v1/flow/jobs/{id}/finish`.
701
pub async fn finish(
702
    State(app): State<Shared>,
703
    headers: HeaderMap,
704
    Path(id): Path<i64>,
705
    Json(f): Json<Finish>,
706
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 707
    let me = runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 708
    let job = running_job(&app, id).await?;
d94449e4 Andy McCall ↶ 709
    db::runners::seen(&app.db, me.id, None, None).await?;
42fa9554 Andy McCall ↶ 710
    let status = match (f.exit_code, &f.error) {
711
        (Some(0), None) => "passed",
712
        (Some(_), _) => "failed",
713
        (None, _) => "error",
714
    };
715
    let (receipt, receipt_hash) = match &f.receipt {
716
        Some(r) if r.job_name == job.name && Some(r.exit_code as i32) == f.exit_code => {
a3c8288b Andy McCall ↶ 717
            // The receipt names exactly the artifacts uploaded, in name order.
718
            let kept: Vec<String> = fdb::artifacts(&app.db, job.id).await?.into_iter().map(|a| a.blob).collect();
719
            if r.outputs.iter().map(hex::encode).collect::<Vec<_>>() != kept {
720
                return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt names other outputs than the artifacts uploaded".into()));
721
            }
42fa9554 Andy McCall ↶ 722
            (serde_json::to_string(r).unwrap_or_default(), hex::encode(r.hash()))
723
        }
724
        Some(_) => return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt does not match the job".into())),
725
        None => (String::new(), String::new()),
726
    };
727
    let error = f.error.clone().unwrap_or_default();
728
    let changed = fdb::finish(
729
        &app.db,
730
        job.id,
731
        fdb::Outcome { status, exit_code: f.exit_code.map(i64::from), error: &error, receipt: &receipt, receipt_hash: &receipt_hash },
732
    )
733
    .await?;
734
    if let Some(run) = fdb::run_by_id(&app.db, job.run_id).await? {
735
        if let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? {
cc19697f Andy McCall ↶ 736
            if let (Some(r), Some(signed)) = (&f.receipt, &f.signed_receipt) {
b6b7e4cf Andy McCall ↶ 737
                let (attest, signer, note) = match keep_receipt(&app, &me, &row, &run, r, signed).await {
cc19697f Andy McCall ↶ 738
                    Ok((attest, signer)) => (attest, signer, String::new()),
739
                    Err(e) => {
740
                        tracing::warn!("flow: job {} receipt not kept: {e}", job.id);
741
                        (String::new(), String::new(), e)
742
                    }
743
                };
744
                fdb::set_receipt(&app.db, job.id, &attest, &signer, &note).await?;
745
            }
42fa9554 Andy McCall ↶ 746
        }
747
    }
070a7b64 Andy McCall ↶ 748
    report_changed(&app, job.run_id, changed).await?;
42fa9554 Andy McCall ↶ 749
    Ok(StatusCode::NO_CONTENT)
070a7b64 Andy McCall ↶ 750
}
751
752
/// Report the checks of jobs whose status changed.
753
async fn report_changed(app: &Shared, run_id: i64, changed: Vec<fdb::Job>) -> sqlx::Result<()> {
754
    let Some(run) = fdb::run_by_id(&app.db, run_id).await? else {
755
        return Ok(());
756
    };
757
    let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? else {
758
        return Ok(());
759
    };
760
    for j in changed {
761
        let summary = match j.status.as_str() {
762
            "passed" => format!("passed in {}", duration(&j)),
763
            "failed" => format!("exit {} after {}", j.exit_code.unwrap_or(-1), duration(&j)),
764
            "skipped" => "skipped: a dependency did not pass".into(),
745d225d Andy McCall ↶ 765
            "cancelled" => j.error.clone(),
070a7b64 Andy McCall ↶ 766
            _ => format!("could not run: {}", j.error),
767
        };
768
        report_check(app, &row, &run.state, run.number, &j.name, check_status(&j.status), &summary).await;
769
    }
770
    Ok(())
771
}
772
5dead1f4 Andy McCall ↶ 773
/// The site's job time limits (oxygenhub#89): set on Admin → Runners →
774
/// Settings, else from `hub.toml`.
775
pub struct Limits {
776
    /// For a job whose pipeline names none.
777
    pub default_s: u64,
778
    /// The most a pipeline may ask for.
779
    pub max_s: u64,
780
    /// Grace past a job's limit before the hub gives it up: a runner stops
781
    /// a job at its limit and reports it within moments, so one still
782
    /// running this long after is one the runner no longer reports (it was
783
    /// restarted, or lost the hub while the job ended, oxygenhub#71).
784
    pub grace_ms: i64,
785
}
786
787
pub const SETTING_DEFAULT: &str = "flow.job_timeout_minutes";
788
pub const SETTING_MAX: &str = "flow.job_timeout_max_minutes";
789
pub const SETTING_GRACE: &str = "flow.lost_grace_minutes";
790
791
pub async fn limits(app: &Shared) -> Limits {
792
    let minutes = |key: &'static str| async move {
793
        db::accounts::setting(&app.db, key).await.ok().flatten().and_then(|v| v.parse::<u64>().ok()).filter(|m| *m > 0)
794
    };
795
    let default_m = minutes(SETTING_DEFAULT).await.unwrap_or(app.config.flow.job_timeout_minutes.max(1));
796
    let max_m = minutes(SETTING_MAX).await.unwrap_or(default_m.max(6 * 60)).max(default_m);
797
    let grace_m = minutes(SETTING_GRACE).await.unwrap_or(5);
798
    Limits { default_s: default_m * 60, max_s: max_m * 60, grace_ms: grace_m as i64 * 60_000 }
799
}
070a7b64 Andy McCall ↶ 800
5dead1f4 Andy McCall ↶ 801
/// End every job still running past its time limit plus the grace:
070a7b64 Andy McCall ↶ 802
/// it is marked an error with the reason, the jobs that depended on it are
803
/// skipped and its run settled, as when a runner reports an error.
804
pub async fn reap_lost_jobs(app: &Shared) -> sqlx::Result<usize> {
5dead1f4 Andy McCall ↶ 805
    let limits = limits(app).await;
806
    let lost = fdb::running_past_limit(&app.db, db::now_ms(), limits.default_s, limits.grace_ms).await?;
070a7b64 Andy McCall ↶ 807
    let n = lost.len();
808
    for job in lost {
809
        let error = format!(
810
            "lost: still running {} minutes after it started, past the time limit; the runner{} stopped reporting it",
811
            (db::now_ms() - job.started_ms.unwrap_or(0)) / 60_000,
812
            if job.runner.is_empty() { String::new() } else { format!(" {}", job.runner) },
813
        );
814
        let changed =
815
            fdb::finish(&app.db, job.id, fdb::Outcome { status: "error", exit_code: None, error: &error, receipt: "", receipt_hash: "" })
816
                .await?;
817
        tracing::warn!("flow: job {} {error}", job.id);
818
        report_changed(app, job.run_id, changed).await?;
819
    }
820
    Ok(n)
eef677c4 Andy McCall ↶ 821
}
822
823
/// End the jobs running on `runner` that it does not hold (oxygenhub#71):
824
/// it restarted since claiming them, or never received the claim. A
825
/// runner claims one job at a time, so a job it holds is always listed.
826
async fn end_jobs_not_held(app: &Shared, runner: &db::runners::Runner, holding: &[i64]) -> sqlx::Result<()> {
827
    for job in fdb::running_on(&app.db, runner.id).await? {
828
        if holding.contains(&job.id) {
829
            continue;
830
        }
831
        let error = format!("lost: the runner {} no longer holds this job (it restarted, or never received it)", runner.name);
832
        let changed =
833
            fdb::finish(&app.db, job.id, fdb::Outcome { status: "error", exit_code: None, error: &error, receipt: "", receipt_hash: "" })
834
                .await?;
835
        tracing::warn!("flow: job {} {error}", job.id);
836
        report_changed(app, job.run_id, changed).await?;
837
    }
838
    Ok(())
348f8c4b Andy McCall ↶ 839
}
840
841
/// Start the scheduled runs due at `when` (oxygenflow#24): on each
842
/// repository that runs Flow, the jobs of its default bookmark's pipeline
843
/// with a `schedule` line due then. Returns how many runs started.
844
pub async fn run_due(app: &Shared, when: flow_core::pipeline::When) -> usize {
845
    let Ok(rows) = db::repos::list(&app.db).await else { return 0 };
846
    let mut started = 0;
847
    for row in rows.into_iter().filter(|r| enabled(app, r)) {
848
        let Some(repo) = RepoName::new(&row.owner, &row.name) else { continue };
849
        let b = row.default_bookmark.clone();
850
        let found = app
851
            .repos
852
            .read(&repo, move |r, _| {
853
                let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) };
854
                Ok(Some((st, r.store().get_state(&st)?.heads)))
855
            })
856
            .await;
857
        let Ok(Some((state, heads))) = found else { continue };
858
        if heads.is_empty() {
859
            continue;
860
        }
861
        match start_run_at(app, &row, &row.default_bookmark, &state.to_hex(), &heads, &row.owner, "schedule", None, Some(when)).await {
862
            Ok(Some(_)) => started += 1,
863
            Ok(None) => {}
864
            Err(e) => tracing::warn!("flow: scheduled run of {}/{}: {e}", row.owner, row.name),
865
        }
866
    }
867
    started
868
}
869
870
/// The hub's clock as a [`When`](flow_core::pipeline::When).
871
pub fn when_now() -> flow_core::pipeline::When {
872
    let now = time::OffsetDateTime::now_utc();
873
    flow_core::pipeline::When {
874
        minute: now.minute() as u32,
875
        hour: now.hour() as u32,
876
        day: now.day() as u32,
877
        month: u8::from(now.month()) as u32,
878
        weekday: now.weekday().number_days_from_sunday() as u32,
879
    }
880
}
881
882
/// Start the scheduled runs due each minute, once a minute.
883
pub async fn scheduler(app: Shared) {
884
    let mut last = None;
885
    loop {
886
        tokio::time::sleep(std::time::Duration::from_secs(15)).await;
887
        let minute = time::OffsetDateTime::now_utc().unix_timestamp() / 60;
888
        if last == Some(minute) {
889
            continue;
890
        }
891
        last = Some(minute);
892
        let n = run_due(&app, when_now()).await;
893
        if n > 0 {
894
            tracing::info!("flow: {n} scheduled run(s) started");
895
        }
896
    }
070a7b64 Andy McCall ↶ 897
}
898
899
/// Look for lost jobs once a minute.
900
pub async fn reaper(app: Shared) {
901
    loop {
902
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
903
        if let Err(e) = reap_lost_jobs(&app).await {
904
            tracing::warn!("flow: looking for lost jobs: {e}");
905
        }
906
    }
cc19697f Andy McCall ↶ 907
}
908
909
/// Check a runner's signed receipt for a job of `run` and keep it in the
910
/// repository (Oxygen spec §14.6): the attestation's hex and the runner's
911
/// name, or why it was not kept. It must be the job's receipt, for the
912
/// run's state and repository, signed by the key the runner's token first
913
/// signed with.
914
pub async fn keep_receipt(
915
    app: &Shared,
b6b7e4cf Andy McCall ↶ 916
    runner: &db::runners::Runner,
cc19697f Andy McCall ↶ 917
    row: &db::repos::RepoRow,
918
    run: &fdb::Run,
919
    receipt: &flow_core::Receipt,
920
    signed_hex: &str,
921
) -> Result<(String, String), String> {
922
    use oxy_core::receipt::SignedReceipt;
923
    let bytes = hex::decode(signed_hex).map_err(|_| "the signed receipt is not hex".to_string())?;
924
    let sr = SignedReceipt::decode(&bytes).map_err(|e| format!("not a signed receipt: {e}"))?;
925
    sr.verify().map_err(|e| format!("it does not verify: {e}"))?;
926
    if sr.receipt.hash().0 != receipt.hash() {
927
        return Err("it signs another receipt than the job's".into());
928
    }
929
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
930
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(heads).encode());
931
    if sr.state() != state {
932
        return Err("it is for another state than the run's".into());
933
    }
934
    let key = sr.runner.hash().to_hex();
b6b7e4cf Andy McCall ↶ 935
    let bound = db::runners::bind_key(&app.db, runner.id, &key, &sr.runner.name).await.map_err(|e| e.to_string())?;
cc19697f Andy McCall ↶ 936
    if bound != key {
937
        return Err(format!("signed by {}, not the key this runner signs with ({})", &key[..12], &bound[..12.min(bound.len())]));
938
    }
939
    let name = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
940
    let signer = sr.runner.name.clone();
941
    let attest = app.repos.write(&name, move |repo, _| Ok(repo.add_receipt(&sr)?)).await.map_err(|e| e.to_string())?;
942
    Ok((attest.to_hex(), signer))
42fa9554 Andy McCall ↶ 943
}
944
945
/// A job's wall time, as the pages show it.
946
pub fn duration(j: &fdb::Job) -> String {
947
    match (j.started_ms, j.finished_ms) {
948
        (Some(a), Some(b)) => human_ms(b - a),
949
        (Some(a), None) => human_ms(db::now_ms() - a),
950
        _ => "–".into(),
951
    }
952
}
953
954
pub fn human_ms(ms: i64) -> String {
955
    let ms = ms.max(0);
956
    if ms < 1000 {
957
        format!("{ms} ms")
958
    } else if ms < 60_000 {
959
        format!("{:.1} s", ms as f64 / 1000.0)
960
    } else {
961
        format!("{} min {} s", ms / 60_000, (ms % 60_000) / 1000)
962
    }
963
}
bf3d0754 Andy McCall ↶ 964
965
/// Where a live run stands among the runners' work: what its viewers may
966
/// know of it. Jobs of other repositories are named only when those are
967
/// public; the rest are counted.
968
#[derive(Clone, Debug, Default, PartialEq, Eq)]
969
pub struct QueueInfo {
970
    /// 1-based place of the run's first queued job among all queued jobs.
971
    pub position: Option<usize>,
972
    /// Jobs running now, and queued jobs ahead of the run's first one.
973
    pub running: Vec<String>,
974
    pub ahead: Vec<String>,
975
    /// Of those, jobs in repositories the page may not name.
976
    pub hidden: usize,
977
}
978
979
impl QueueInfo {
980
    /// "3rd in line" and the like.
981
    pub fn place(&self) -> Option<String> {
982
        self.position.map(|n| {
983
            let suffix = match (n % 10, n % 100) {
984
                (1, 11) | (2, 12) | (3, 13) => "th",
985
                (1, _) => "st",
986
                (2, _) => "nd",
987
                (3, _) => "rd",
988
                _ => "th",
989
            };
990
            format!("{n}{suffix} in line")
991
        })
992
    }
993
}
994
995
/// The queue as `run` (of repository `repo_id`) sees it, from `queue`.
996
pub fn queue_info(queue: &[fdb::QueueEntry], repo_id: i64, run_id: i64) -> QueueInfo {
997
    let mut info = QueueInfo::default();
998
    let first = queue.iter().filter(|e| e.status == "queued").position(|e| e.run_id == run_id);
999
    info.position = first.map(|i| i + 1);
1000
    let label = |e: &fdb::QueueEntry| format!("{}/{} #{} {}", e.owner, e.name, e.run_number, e.job);
1001
    let mut queued_seen = 0;
1002
    for e in queue {
1003
        if e.run_id == run_id {
1004
            continue;
1005
        }
1006
        let ahead = match e.status.as_str() {
1007
            "running" => true,
1008
            _ => {
1009
                queued_seen += 1;
1010
                first.is_some_and(|f| queued_seen <= f)
1011
            }
1012
        };
1013
        if !ahead {
1014
            continue;
1015
        }
1016
        if e.repo_id != repo_id && e.visibility != "public" {
1017
            info.hidden += 1;
1018
        } else if e.status == "running" {
1019
            info.running.push(label(e));
1020
        } else {
1021
            info.ahead.push(label(e));
1022
        }
1023
    }
1024
    info
1025
}
1026
1027
#[cfg(test)]
1028
mod queue_tests {
1029
    use super::*;
1030
1031
    fn e(job_id: i64, run_id: i64, repo_id: i64, vis: &str, job: &str, status: &str) -> fdb::QueueEntry {
1032
        fdb::QueueEntry {
1033
            job_id,
1034
            run_id,
1035
            repo_id,
1036
            owner: "o".into(),
1037
            name: format!("r{repo_id}"),
1038
            visibility: vis.into(),
1039
            run_number: run_id,
1040
            job: job.into(),
1041
            status: status.into(),
b66e70b7 Andy McCall ↶ 1042
            runner: String::new(),
c4c87f48 Andy McCall ↶ 1043
            spec: String::new(),
1044
            sandbox: String::new(),
bf3d0754 Andy McCall ↶ 1045
        }
1046
    }
1047
1048
    #[test]
1049
    fn a_run_learns_its_place_and_what_is_ahead_of_it() {
1050
        // Running first, then queued in claim order (as `fdb::queue` returns them).
1051
        let q = vec![
1052
            e(1, 10, 1, "public", "build", "running"),
1053
            e(2, 11, 2, "private", "test", "running"),
1054
            e(3, 10, 1, "public", "test", "queued"),
1055
            e(4, 12, 3, "private", "lint", "queued"),
1056
            e(5, 13, 4, "public", "build", "queued"),
1057
            e(6, 13, 4, "public", "test", "queued"),
1058
            e(7, 14, 5, "public", "late", "queued"),
1059
        ];
1060
        let info = queue_info(&q, 4, 13);
1061
        assert_eq!(info.position, Some(3));
1062
        assert_eq!(info.place().as_deref(), Some("3rd in line"));
1063
        assert_eq!(info.running, ["o/r1 #10 build"]);
1064
        assert_eq!(info.ahead, ["o/r1 #10 test"], "queued ahead, not behind");
1065
        assert_eq!(info.hidden, 2, "private repositories are counted, not named");
1066
        // A private repository's own jobs are named to its viewers.
1067
        assert_eq!(queue_info(&q, 3, 12).ahead, ["o/r1 #10 test"]);
1068
        assert_eq!(queue_info(&q, 3, 12).position, Some(2));
1069
        // A run with nothing queued has no place.
1070
        assert_eq!(queue_info(&q, 1, 99).position, None);
1071
        let ordinal = |n| QueueInfo { position: Some(n), ..Default::default() }.place().unwrap();
1072
        assert_eq!(
1073
            [ordinal(1), ordinal(2), ordinal(11), ordinal(22), ordinal(113)],
1074
            ["1st in line", "2nd in line", "11th in line", "22nd in line", "113th in line"]
1075
        );
1076
    }
1077
}