OXYGENHUB
◆ v0.3.1 oxygenhub / crates / hub / src / flow.rs
46.2 KiB · Rust · last changed Cancel runs from oh, the run page, and newer pushes 1 month ago source blame history raw
42fa9554 Andy McCall ↶ 1
//! OxygenFlow on the hub (hub spec §13.2): runs started by pushes, the
2
//! runner API, job tokens and the checks that mirror jobs. Pipelines are
3
//! parsed by `flow-core`, the same code `flow check` runs.
4
5
use crate::api::ApiError;
6
use crate::db::{self, flow as fdb};
7
use crate::repos::RepoName;
8
use crate::Shared;
9
use axum::extract::{Path, State};
10
use axum::http::{HeaderMap, StatusCode};
11
use axum::response::{IntoResponse, Response};
12
use axum::Json;
13
use flow_core::api::{Claim, ClaimRequest, Finish, LogBatch, MAX_RECORDS, MAX_RECORD_TEXT};
14
use flow_core::{Pipeline, PIPELINE_PATH};
15
use oxy_core::hash::ChangeHash;
16
17
/// Prefix of job tokens, so the wire protocol can tell them from personal tokens.
18
pub const JOB_TOKEN_PREFIX: &str = "flowjob_";
19
/// A job token is refused this long after its job started, whatever its state.
20
pub const JOB_TOKEN_MAX_AGE_MS: i64 = 2 * 3600 * 1000;
21
9008bf6c Andy McCall ↶ 22
/// Whether pushes to this repository start runs: its own setting when a
23
/// site administrator made one, else hub.toml's `[flow] repos`.
24
pub fn enabled(app: &Shared, row: &db::repos::RepoRow) -> bool {
25
    match row.flow.as_str() {
26
        "on" => true,
27
        "off" => false,
28
        _ => app.config.flow.allows(&row.owner, &row.name),
29
    }
30
}
31
42fa9554 Andy McCall ↶ 32
/// A bookmark moved by a push: start a run when the repository is allowed
33
/// and the new state has a pipeline.
34
pub async fn on_push(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, state: &str, heads: &[ChangeHash], actor: &str) {
c889911b Andy McCall ↶ 35
    if heads.is_empty() || !enabled(app, row) {
42fa9554 Andy McCall ↶ 36
        return;
37
    }
c889911b Andy McCall ↶ 38
    // A pushed tag runs the jobs whose `on` names it (oxygenflow#24).
39
    let started = match bookmark.strip_prefix("tags/") {
40
        Some(tag) => start_run_with(app, row, bookmark, state, heads, actor, &format!("tag:{tag}"), None).await,
41
        None => {
42
            let touched = touched_since_last_run(app, row, bookmark, heads).await;
745d225d Andy McCall ↶ 43
            let started = start_run_scoped(app, row, bookmark, state, heads, actor, "push", touched.as_deref()).await;
44
            if let Ok(Some(n)) = started {
45
                supersede(app, row, bookmark, n).await;
46
            }
47
            started
c889911b Andy McCall ↶ 48
        }
49
    };
50
    if let Err(e) = started {
1edac4b6 Andy McCall ↶ 51
        tracing::warn!("flow: starting a run of {}/{}: {e}", row.owner, row.name);
52
    }
53
}
54
55
/// "Run pipeline" (hub spec §13.2): run a bookmark's current state without
56
/// moving it. Returns the new run's number.
57
pub async fn run_bookmark(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, actor: &str) -> Result<i64, String> {
58
    if !enabled(app, row) {
59
        return Err("OxygenFlow is not enabled for this repository".into());
60
    }
61
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
62
    let b = bookmark.to_string();
63
    let found = app
64
        .repos
65
        .read(&repo, move |r, _| {
66
            let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) };
67
            Ok(Some((st, r.store().get_state(&st)?.heads)))
68
        })
69
        .await
70
        .map_err(|e| e.to_string())?;
71
    let Some((state, heads)) = found else { return Err(format!("no bookmark {bookmark}")) };
72
    if heads.is_empty() {
73
        return Err(format!("{bookmark} has no changes yet"));
74
    }
75
    start_run(app, row, bookmark, &state.to_hex(), &heads, actor, "manual")
76
        .await?
77
        .ok_or_else(|| format!("{bookmark} has no {PIPELINE_PATH}"))
78
}
79
4f692533 Andy McCall ↶ 80
/// "Run pipeline" on one committed change: the state whose only head is
81
/// `change`, whatever bookmarks hold it. Returns the new run's number.
82
pub async fn run_change(app: &Shared, row: &db::repos::RepoRow, change: &str, actor: &str) -> Result<i64, String> {
83
    if !enabled(app, row) {
84
        return Err("OxygenFlow is not enabled for this repository".into());
85
    }
86
    let h = ChangeHash::from_hex(change.trim().trim_start_matches('#')).ok_or("give the full hash of a change")?;
87
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
df15b747 Andy McCall ↶ 88
    let known = app.repos.read(&repo, move |r, _| Ok(r.store().get_change_meta(&h).is_ok())).await.map_err(|e| e.to_string())?;
4f692533 Andy McCall ↶ 89
    if !known {
90
        return Err(format!("no change {} in this repository", &change.trim()[..12.min(change.trim().len())]));
91
    }
92
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(vec![h]).encode());
93
    start_run(app, row, "", &state.to_hex(), &[h], actor, "manual")
94
        .await?
95
        .ok_or_else(|| format!("that change's state has no {PIPELINE_PATH}"))
96
}
97
1edac4b6 Andy McCall ↶ 98
/// Re-run a finished run: the same state, bookmark and pipeline, as a new
99
/// run whose checks replace the old ones. Returns the new run's number.
100
pub async fn rerun(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<i64, String> {
101
    if !enabled(app, row) {
102
        return Err("OxygenFlow is not enabled for this repository".into());
103
    }
104
    if run.live() {
105
        return Err(format!("run #{} is still {}", run.number, run.status));
106
    }
107
    let heads: Vec<ChangeHash> = run.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
108
    if heads.is_empty() {
109
        return Err("the run names no changes".into());
110
    }
c889911b Andy McCall ↶ 111
    // A rerun repeats its run's event: the jobs it had are the jobs it gets.
112
    let trigger = format!("rerun:{}:{}", run.number, original_event(run));
113
    start_run(app, row, &run.bookmark, &run.state, &heads, actor, &trigger)
1edac4b6 Andy McCall ↶ 114
        .await?
115
        .ok_or_else(|| format!("that state has no {PIPELINE_PATH}"))
745d225d Andy McCall ↶ 116
}
117
118
/// Cancel a run still going (oxygenflow#32), its checks with it.
119
pub async fn cancel(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<(), String> {
120
    if !run.live() {
121
        return Err(format!("run #{} is already {}", run.number, run.status));
122
    }
123
    cancel_with(app, run, &format!("cancelled by {actor}")).await.map_err(|e| {
124
        tracing::warn!("flow: cancelling run #{} of {}/{}: {e}", run.number, row.owner, row.name);
125
        "the run could not be cancelled".to_string()
126
    })
127
}
128
129
async fn cancel_with(app: &Shared, run: &fdb::Run, why: &str) -> sqlx::Result<()> {
130
    let changed = fdb::cancel_run(&app.db, run.id, why).await?;
131
    report_changed(app, run.id, changed).await
c889911b Andy McCall ↶ 132
}
133
745d225d Andy McCall ↶ 134
/// A push makes the push runs still going on its bookmark moot: they are
135
/// cancelled, and the new run's scope reaches back past them.
136
async fn supersede(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, number: i64) {
137
    let Ok(older) = fdb::live_push_runs_before(&app.db, row.id, bookmark, number).await else { return };
138
    for run in older {
139
        if let Err(e) = cancel_with(app, &run, &format!("superseded by run #{number}")).await {
140
            tracing::warn!("flow: superseding run #{} of {}/{}: {e}", run.number, row.owner, row.name);
141
        }
142
    }
143
}
144
c889911b Andy McCall ↶ 145
/// The event a run's trigger stands for: `push`, `tag:NAME`, `manual`,
146
/// `schedule`, and `rerun:N:EVENT` repeating EVENT.
147
fn event_of<'a>(trigger: &'a str, bookmark: &'a str) -> flow_core::pipeline::Event<'a> {
148
    use flow_core::pipeline::Event;
149
    let t = match trigger.strip_prefix("rerun:") {
150
        Some(rest) => rest.split_once(':').map(|(_, e)| e).unwrap_or("manual"),
151
        None => trigger,
152
    };
153
    if let Some(tag) = t.strip_prefix("tag:") {
154
        Event::Tag(tag)
155
    } else if t == "push" {
156
        Event::Push(bookmark)
157
    } else if t == "schedule" {
158
        Event::Schedule
159
    } else {
160
        Event::Manual
161
    }
1edac4b6 Andy McCall ↶ 162
}
163
c889911b Andy McCall ↶ 164
/// The event to repeat when rerunning `run`.
165
fn original_event(run: &fdb::Run) -> String {
166
    let t = run.trigger.as_str();
167
    match t.strip_prefix("rerun:") {
168
        Some(rest) => rest.split_once(':').map(|(_, e)| e.to_string()).unwrap_or_else(|| "manual".into()),
169
        None => t.to_string(),
170
    }
171
}
172
d7ce41ac Andy McCall ↶ 173
/// The paths the changes a push brought touch: the changes in `heads`'
174
/// state that the bookmark's previous run did not have, and the files their
175
/// edits and tree operations name, as the new state places them. `None`
176
/// (run every job) for the bookmark's first run, or when it cannot be told.
177
async fn touched_since_last_run(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, heads: &[ChangeHash]) -> Option<Vec<String>> {
178
    let last = fdb::latest_on(&app.db, row.id, bookmark).await.ok()??;
179
    let old: Vec<ChangeHash> = last.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
180
    if old.is_empty() {
181
        return None;
182
    }
183
    let repo = RepoName::new(&row.owner, &row.name)?;
184
    let new = heads.to_vec();
185
    app.repos
186
        .read(&repo, move |r, cache| {
187
            let fresh = r.negotiate(&old, &new)?;
188
            let loaded = cache.load(r, &new)?;
189
            let by_node: std::collections::HashMap<oxy_core::ids::NodeId, &String> =
190
                loaded.snapshot.files.iter().map(|(p, f)| (f.node, p)).collect();
191
            let mut paths = std::collections::BTreeSet::new();
192
            for h in fresh {
193
                let (c, _) = r.store().get_change_lazy(&h)?;
194
                let nodes = c.edits.iter().map(|e| e.node()).chain(c.tree_ops.iter().map(|o| o.node()));
195
                for n in nodes {
196
                    if let Some(p) = by_node.get(&n) {
197
                        paths.insert((*p).clone());
198
                    }
199
                }
200
            }
201
            Ok(paths.into_iter().collect::<Vec<String>>())
202
        })
203
        .await
204
        .ok()
205
}
206
207
/// Whether a job runs for a push that touched `touched`: always without a
208
/// scope or without knowing what was touched, else when a path matches.
209
fn in_scope(scope: &[String], touched: Option<&[String]>) -> bool {
210
    let Some(touched) = touched else { return true };
211
    if scope.is_empty() {
212
        return true;
213
    }
214
    let mut b = globset::GlobSetBuilder::new();
215
    for g in scope {
216
        match globset::Glob::new(g) {
217
            Ok(g) => {
218
                b.add(g);
219
            }
220
            Err(_) => return true,
221
        }
222
    }
223
    let Ok(set) = b.build() else { return true };
224
    touched.iter().any(|p| set.is_match(p))
225
}
226
227
/// [`start_run`] for a push: jobs whose `scope` the push did not touch are
228
/// left out of the run, unless a job that runs depends on them.
229
#[allow(clippy::too_many_arguments)]
230
pub async fn start_run_scoped(
231
    app: &Shared,
232
    row: &db::repos::RepoRow,
233
    bookmark: &str,
234
    state: &str,
235
    heads: &[ChangeHash],
236
    actor: &str,
237
    trigger: &str,
238
    touched: Option<&[String]>,
239
) -> Result<Option<i64>, String> {
240
    start_run_with(app, row, bookmark, state, heads, actor, trigger, touched).await
241
}
242
1edac4b6 Andy McCall ↶ 243
/// Start a run of the pipeline in the state `heads` make (whose state
244
/// hash is `state`), recorded on `bookmark`: for a push, "Run pipeline"
245
/// (`trigger` = `manual`) and re-runs (`rerun:N`). `actor` is the account
246
/// whose access the run's subrepo reads are checked against. Returns the
247
/// run's number, or `None` when the state has no pipeline. The caller
248
/// checks that Flow is enabled.
249
pub async fn start_run(
250
    app: &Shared,
251
    row: &db::repos::RepoRow,
252
    bookmark: &str,
253
    state: &str,
254
    heads: &[ChangeHash],
255
    actor: &str,
256
    trigger: &str,
257
) -> Result<Option<i64>, String> {
d7ce41ac Andy McCall ↶ 258
    start_run_with(app, row, bookmark, state, heads, actor, trigger, None).await
259
}
260
261
#[allow(clippy::too_many_arguments)]
262
async fn start_run_with(
263
    app: &Shared,
264
    row: &db::repos::RepoRow,
265
    bookmark: &str,
266
    state: &str,
267
    heads: &[ChangeHash],
268
    actor: &str,
269
    trigger: &str,
270
    touched: Option<&[String]>,
271
) -> Result<Option<i64>, String> {
348f8c4b Andy McCall ↶ 272
    start_run_at(app, row, bookmark, state, heads, actor, trigger, touched, None).await
273
}
274
275
/// [`start_run_with`], for a scheduled run due at `at` on the hub's clock.
276
#[allow(clippy::too_many_arguments)]
277
async fn start_run_at(
278
    app: &Shared,
279
    row: &db::repos::RepoRow,
280
    bookmark: &str,
281
    state: &str,
282
    heads: &[ChangeHash],
283
    actor: &str,
284
    trigger: &str,
285
    touched: Option<&[String]>,
286
    at: Option<flow_core::pipeline::When>,
287
) -> Result<Option<i64>, String> {
1edac4b6 Andy McCall ↶ 288
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
42fa9554 Andy McCall ↶ 289
    let h = heads.to_vec();
290
    let text = app
291
        .repos
292
        .read(&repo, move |r, cache| {
293
            let loaded = cache.load(r, &h)?;
294
            Ok(match loaded.snapshot.files.get(PIPELINE_PATH).map(|f| &f.content) {
295
                Some(oxy_core::state::FileContent::Text { bytes, .. }) => Some(String::from_utf8_lossy(bytes).into_owned()),
296
                _ => None,
297
            })
298
        })
1edac4b6 Andy McCall ↶ 299
        .await
300
        .map_err(|e| format!("reading the pipeline: {e}"))?;
301
    let Some(text) = text else { return Ok(None) };
42fa9554 Andy McCall ↶ 302
    let mut hex: Vec<String> = heads.iter().map(|h| h.to_hex()).collect();
303
    hex.sort();
304
    let heads_s = hex.join("+");
1edac4b6 Andy McCall ↶ 305
    let base = db::flow::NewRun { repo_id: row.id, bookmark, state, heads: &heads_s, pipeline_hash: "", actor, trigger, error: None };
306
    let id = match Pipeline::parse(&text) {
42fa9554 Andy McCall ↶ 307
        Err(problems) => {
308
            let msg = problems.to_string();
1edac4b6 Andy McCall ↶ 309
            db::flow::create_run(&app.db, db::flow::NewRun { error: Some(&msg), ..base }, &[]).await.map_err(|e| e.to_string())?
42fa9554 Andy McCall ↶ 310
        }
311
        Ok(p) => {
312
            let depths = p.depths();
d7ce41ac Andy McCall ↶ 313
            // Jobs out of the push's scope stay out, unless a kept job
314
            // needs them.
c889911b Andy McCall ↶ 315
            // The jobs this run's event starts (`on`), out of those its
316
            // push touched (`scope`); then what they depend on.
348f8c4b Andy McCall ↶ 317
            let event = match at {
318
                Some(w) => flow_core::pipeline::Event::ScheduleAt(w),
319
                None => event_of(trigger, bookmark),
320
            };
c889911b Andy McCall ↶ 321
            let mut keep: std::collections::HashSet<String> = p
322
                .order()
323
                .into_iter()
324
                .filter(|j| p.on_of(j).matches(event) && in_scope(&j.scope, touched))
325
                .map(|j| j.name.clone())
326
                .collect();
d7ce41ac Andy McCall ↶ 327
            loop {
328
                let needed: Vec<String> = p
329
                    .order()
330
                    .into_iter()
331
                    .filter(|j| keep.contains(&j.name))
332
                    .flat_map(|j| j.depends_on.iter().cloned())
333
                    .filter(|d| !keep.contains(d))
334
                    .collect();
335
                if needed.is_empty() {
336
                    break;
337
                }
338
                keep.extend(needed);
339
            }
42fa9554 Andy McCall ↶ 340
            let jobs: Vec<fdb::NewJob> = p
341
                .order()
342
                .into_iter()
d7ce41ac Andy McCall ↶ 343
                .filter(|j| keep.contains(&j.name))
42fa9554 Andy McCall ↶ 344
                .map(|j| {
345
                    let sb = &p.sandboxes[&j.sandbox];
346
                    fdb::NewJob {
347
                        name: j.name.clone(),
348
                        depth: depths[j.name.as_str()] as i64,
349
                        depends: j.depends_on.clone(),
350
                        spec: serde_json::to_string(j).unwrap_or_default(),
351
                        sandbox: serde_json::to_string(sb).unwrap_or_default(),
352
                        runtime: sb.runtime.as_str().into(),
353
                    }
354
                })
355
                .collect();
c889911b Andy McCall ↶ 356
            if jobs.is_empty() {
357
                // Nothing this event starts (or this push touched).
d7ce41ac Andy McCall ↶ 358
                return Ok(None);
359
            }
42fa9554 Andy McCall ↶ 360
            let hash = hex::encode(p.hash());
1edac4b6 Andy McCall ↶ 361
            let id =
362
                db::flow::create_run(&app.db, db::flow::NewRun { pipeline_hash: &hash, ..base }, &jobs).await.map_err(|e| e.to_string())?;
363
            if let Ok(Some(run)) = fdb::run_by_id(&app.db, id).await {
42fa9554 Andy McCall ↶ 364
                for j in &jobs {
a0d4bbfb Andy McCall ↶ 365
                    report_check(app, row, state, run.number, &j.name, "queued", "waiting for a runner").await;
42fa9554 Andy McCall ↶ 366
                }
367
            }
368
            id
369
        }
370
    };
1edac4b6 Andy McCall ↶ 371
    let run = fdb::run_by_id(&app.db, id).await.map_err(|e| e.to_string())?.ok_or("the run vanished")?;
372
    Ok(Some(run.number))
42fa9554 Andy McCall ↶ 373
}
374
a0d4bbfb Andy McCall ↶ 375
/// Record a job's check; it links to the run's page by the run's number.
376
async fn report_check(app: &Shared, row: &db::repos::RepoRow, state: &str, run_number: i64, job: &str, status: &str, summary: &str) {
377
    let url = format!("{}/{}/{}/flow/{run_number}", app.config.base_url(), row.owner, row.name);
42fa9554 Andy McCall ↶ 378
    let name = format!("flow/{job}");
379
    let c = db::hooks::upsert_check(
380
        &app.db,
381
        row.id,
382
        db::hooks::CheckReport { state, name: &name, status, summary, url: &url, reporter: None, seal: "" },
383
    )
384
    .await;
385
    if let Ok(c) = c {
386
        let api = crate::api::check_of(&c);
387
        app.events.proposal(row.id, hub_api::ProposalEvent { number: 0, kind: "check".into(), check: Some(api.clone()) });
388
        crate::hooks::emit(app, row, "check", "flow", serde_json::json!({ "check": api })).await;
389
    }
390
}
391
392
/// The check status a job's status shows as.
393
fn check_status(job_status: &str) -> &'static str {
394
    match job_status {
395
        "queued" => "queued",
396
        "running" => "running",
397
        "passed" => "passed",
398
        "skipped" => "skipped",
745d225d Andy McCall ↶ 399
        "cancelled" => "cancelled",
42fa9554 Andy McCall ↶ 400
        _ => "failed",
401
    }
402
}
403
b6b7e4cf Andy McCall ↶ 404
/// The runner a bearer token belongs to: a registered runner, or one
405
/// configured with a hub.toml token (tracked from its first request). A
406
/// disabled runner is refused.
407
async fn runner_auth(app: &Shared, headers: &HeaderMap) -> Result<db::runners::Runner, ApiError> {
42fa9554 Andy McCall ↶ 408
    let Some(t) = oxy_server::bearer_token(headers) else {
409
        return Err(ApiError(StatusCode::UNAUTHORIZED, "a runner token is required".into()));
b6b7e4cf Andy McCall ↶ 410
    };
411
    let hash = blake3::hash(t.as_bytes()).to_hex().to_string();
412
    let runner = match db::runners::by_token(&app.db, &hash).await? {
413
        Some(r) => r,
414
        None if app.config.flow.runner_tokens.iter().any(|x| crate::auth::mac_equal(x, t)) => {
415
            db::runners::configured(&app.db, &hash, "runner").await?
416
        }
417
        None => return Err(ApiError(StatusCode::UNAUTHORIZED, "not a runner token".into())),
42fa9554 Andy McCall ↶ 418
    };
b6b7e4cf Andy McCall ↶ 419
    if runner.state == "disabled" {
420
        return Err(ApiError(StatusCode::FORBIDDEN, format!("runner {} is disabled on this hub", runner.name)));
42fa9554 Andy McCall ↶ 421
    }
b6b7e4cf Andy McCall ↶ 422
    Ok(runner)
42fa9554 Andy McCall ↶ 423
}
424
b6b7e4cf Andy McCall ↶ 425
/// `POST /api/v1/flow/runners/register`: a one-time registration token
426
/// from Admin → Runners buys a runner credential, binding the runner's
427
/// signing identity (hub spec §13.2).
428
pub async fn register(
429
    State(app): State<Shared>,
430
    Json(req): Json<flow_core::api::Register>,
431
) -> Result<Json<flow_core::api::Registered>, ApiError> {
432
    let bytes = hex::decode(req.identity.trim()).map_err(|_| ApiError(StatusCode::BAD_REQUEST, "identity: not hex".into()))?;
433
    let identity = oxy_core::identity::Identity::decode(&bytes).map_err(|e| ApiError(StatusCode::BAD_REQUEST, format!("identity: {e}")))?;
434
    let name: String = req.name.chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')).take(64).collect();
435
    if name.is_empty() {
436
        return Err(ApiError(StatusCode::BAD_REQUEST, "give the runner a name of letters, digits, - _ or .".into()));
437
    }
438
    let credential = format!("flowrunner_{}", crate::auth::random_hex(32));
439
    let hash = blake3::hash(credential.as_bytes()).to_hex().to_string();
440
    let reg = blake3::hash(req.token.trim().as_bytes()).to_hex().to_string();
441
    let runner =
442
        db::runners::register(&app.db, &reg, &name, &hash, &identity.hash().to_hex(), &identity.name, "").await?.ok_or_else(|| {
443
            ApiError(StatusCode::FORBIDDEN, "that registration token is unknown, used or expired; create one on Admin → Runners".into())
444
        })?;
445
    db::runners::seen(&app.db, runner.id, None, Some(&req.info)).await?;
446
    tracing::info!("flow: runner {} registered (key {})", runner.name, &runner.key[..12]);
447
    Ok(Json(flow_core::api::Registered { id: runner.id, name: runner.name, credential }))
448
}
449
42fa9554 Andy McCall ↶ 450
/// `POST /api/v1/flow/claim`.
451
pub async fn claim(State(app): State<Shared>, headers: HeaderMap, Json(req): Json<ClaimRequest>) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 452
    let me = runner_auth(&app, &headers).await?;
453
    let claimed: String = req.runner.chars().filter(|c| !c.is_control()).take(64).collect();
454
    db::runners::seen(&app.db, me.id, Some(&claimed), req.info.as_ref()).await?;
eef677c4 Andy McCall ↶ 455
    if let Some(holding) = &req.holding {
456
        end_jobs_not_held(&app, &me, holding).await?;
457
    }
b6b7e4cf Andy McCall ↶ 458
    // A draining runner finishes what it has and takes nothing new.
459
    if me.state == "draining" {
460
        return Ok(StatusCode::NO_CONTENT.into_response());
461
    }
462
    let runner = if me.source == "registered" { me.name.clone() } else { claimed };
42fa9554 Andy McCall ↶ 463
    let secret = format!("{JOB_TOKEN_PREFIX}{}", crate::auth::random_hex(32));
b0612e7d Andy McCall ↶ 464
    let room = fdb::Room {
465
        memory_mib: req.free_memory_mib,
466
        cpus: req.free_cpus,
467
        images: req.images.as_ref().map(|i| i.iter().map(|h| h.to_ascii_lowercase()).collect()),
468
    };
c4c87f48 Andy McCall ↶ 469
    // The labels it reports now, or those it registered with.
470
    let labels: Vec<String> = match &req.info {
471
        Some(i) => i.labels.clone(),
472
        None => me.label_list().into_iter().map(String::from).collect(),
473
    };
474
    let Some(job) =
475
        fdb::claim_within(&app.db, &req.runtimes, &runner, &crate::auth::token_hash(&secret), room, &me.scope(), &labels).await?
476
    else {
42fa9554 Andy McCall ↶ 477
        return Ok(StatusCode::NO_CONTENT.into_response());
478
    };
b6b7e4cf Andy McCall ↶ 479
    db::runners::set_job_runner(&app.db, job.id, me.id).await?;
42fa9554 Andy McCall ↶ 480
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
481
    let row =
482
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
a0d4bbfb Andy McCall ↶ 483
    report_check(&app, &row, &run.state, run.number, &job.name, "running", &format!("running on {runner}")).await;
42fa9554 Andy McCall ↶ 484
    let bad = |e: serde_json::Error| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}"));
5dead1f4 Andy McCall ↶ 485
    // The job's limit: its own timeout, else its sandbox's, else the site
486
    // default, never past the site maximum; recorded for the reaper.
487
    let limits = limits(&app).await;
488
    let spec: flow_core::pipeline::Job = serde_json::from_str(&job.spec).map_err(bad)?;
489
    let sandbox: flow_core::pipeline::Sandbox = serde_json::from_str(&job.sandbox).map_err(bad)?;
490
    let timeout_s = spec.timeout_s.or(sandbox.timeout_s).unwrap_or(limits.default_s).min(limits.max_s).max(60);
491
    fdb::set_job_timeout(&app.db, job.id, timeout_s).await?;
42fa9554 Andy McCall ↶ 492
    let claim = Claim {
493
        job_id: job.id,
494
        run_id: run.id,
495
        repo: format!("{}/{}", row.owner, row.name),
496
        clone_url: format!("{}/{}/{}", app.config.base_url(), row.owner, row.name),
497
        token: secret,
498
        heads: run.head_list().into_iter().map(String::from).collect(),
499
        pipeline_hash: run.pipeline_hash.clone(),
5dead1f4 Andy McCall ↶ 500
        job: spec,
501
        sandbox,
502
        timeout_s,
fd3db644 Andy McCall ↶ 503
        subrepos: vec![],
9a065bc0 Andy McCall ↶ 504
        state: run.state.clone(),
505
        bookmark: run.bookmark.clone(),
506
        run_number: run.number,
42fa9554 Andy McCall ↶ 507
    };
fd3db644 Andy McCall ↶ 508
    let mut claim = claim;
afd0be38 Andy McCall ↶ 509
    let (inputs, ids, left_out) = subrepo_inputs(&app, &row, &run).await;
fd3db644 Andy McCall ↶ 510
    if !ids.is_empty() {
511
        fdb::set_also_read(&app.db, job.id, &ids).await?;
512
    }
afd0be38 Andy McCall ↶ 513
    // Say at the top of the job's log what it will not have (oxygenhub#56),
514
    // rather than let it fail later on missing files.
515
    if !left_out.is_empty() {
516
        let records: Vec<flow_core::api::Record> =
517
            left_out.into_iter().map(|text| flow_core::api::Record { t: 0, stream: "flow".into(), text }).collect();
518
        fdb::append_log(&app.db, job.id, &records, MAX_RECORDS as i64).await?;
519
    }
fd3db644 Andy McCall ↶ 520
    claim.subrepos = inputs;
42fa9554 Andy McCall ↶ 521
    Ok(Json(claim).into_response())
fd3db644 Andy McCall ↶ 522
}
523
25091e5c Andy McCall ↶ 524
/// The Oxygen subrepo pins in `name`'s state `heads`: (path, repository id, pinned heads).
525
async fn pins_in(
526
    app: &Shared,
527
    name: crate::repos::RepoName,
528
    heads: Vec<oxy_core::hash::ChangeHash>,
529
) -> Vec<(String, [u8; 16], Vec<oxy_core::hash::ChangeHash>)> {
530
    app.repos
fd3db644 Andy McCall ↶ 531
        .read(&name, move |repo, cache| {
532
            let loaded = cache.load(repo, &heads)?;
533
            let mut out = Vec::new();
534
            for (path, f) in &loaded.snapshot.files {
535
                if f.kind != oxy_core::objects::NodeKind::Subrepo {
536
                    continue;
537
                }
538
                if let oxy_core::state::FileContent::Opaque { blobs } = &f.content {
539
                    if let Some(b) = blobs.first() {
540
                        if let Ok(oxy_core::subrepo::Pin::Oxy { repo: id, heads, .. }) = repo
541
                            .read_blob(b)
542
                            .map_err(crate::repos::RepoError::Oxy)
543
                            .and_then(|bytes| oxy_core::subrepo::Pin::decode(&bytes).map_err(|e| crate::repos::RepoError::Oxy(e.into())))
544
                        {
545
                            out.push((path.clone(), id, heads));
546
                        }
547
                    }
548
                }
549
            }
550
            Ok(out)
551
        })
552
        .await
25091e5c Andy McCall ↶ 553
        .unwrap_or_default()
554
}
555
556
/// How deep subrepos of subrepos are followed.
557
const MAX_SUBREPO_DEPTH: usize = 3;
558
559
/// The Oxygen subrepos the run's state pins, and those their states pin in
560
/// turn (parents first, paths joined), that live on this hub and that the
561
/// run's pusher may read: what the job's workspace needs beside its own
afd0be38 Andy McCall ↶ 562
/// tree. A pin the hub cannot serve is left out, with what it pins, and
563
/// the third list says why, for the job's log.
564
async fn subrepo_inputs(
565
    app: &Shared,
566
    row: &db::repos::RepoRow,
567
    run: &fdb::Run,
568
) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>, Vec<String>) {
569
    let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![], vec![]) };
25091e5c Andy McCall ↶ 570
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
fd3db644 Andy McCall ↶ 571
    let pusher = db::accounts::account_by_name(&app.db, &run.actor).await.ok().flatten();
25091e5c Andy McCall ↶ 572
    let rows = db::repos::list(&app.db).await.unwrap_or_default();
fd3db644 Andy McCall ↶ 573
    let mut inputs = Vec::new();
574
    let mut ids = Vec::new();
afd0be38 Andy McCall ↶ 575
    let mut left_out = Vec::new();
576
    let actor = if run.actor.is_empty() { "the run's starter".to_string() } else { run.actor.clone() };
25091e5c Andy McCall ↶ 577
    // (path prefix, repository, heads, depth), parents before children.
578
    let mut queue = std::collections::VecDeque::from([(String::new(), name, heads, 0usize)]);
579
    while let Some((prefix, name, heads, depth)) = queue.pop_front() {
580
        if depth == MAX_SUBREPO_DEPTH {
581
            continue;
582
        }
583
        for (path, id, pinned) in pins_in(app, name, heads).await {
584
            let path = if prefix.is_empty() { path } else { format!("{prefix}/{path}") };
afd0be38 Andy McCall ↶ 585
            let mut hosted = false;
25091e5c Andy McCall ↶ 586
            for other in &rows {
587
                let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue };
588
                let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false);
589
                if !matches {
590
                    continue;
591
                }
afd0be38 Andy McCall ↶ 592
                hosted = true;
25091e5c Andy McCall ↶ 593
                let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten();
594
                if crate::auth::roles::allows(role, crate::auth::Access::Read) {
595
                    inputs.push(flow_core::api::SubrepoInput {
596
                        path: path.clone(),
597
                        repo: format!("{}/{}", other.owner, other.name),
598
                        clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name),
599
                        heads: pinned.iter().map(|h| h.to_hex()).collect(),
600
                    });
601
                    if !ids.contains(&other.id) {
602
                        ids.push(other.id);
603
                    }
604
                    queue.push_back((path.clone(), other_name, pinned.clone(), depth + 1));
afd0be38 Andy McCall ↶ 605
                } else {
606
                    left_out.push(format!(
607
                        "subrepo {path} ({}/{}) is left out: {actor} cannot read it, so the job has no files there",
608
                        other.owner, other.name
609
                    ));
25091e5c Andy McCall ↶ 610
                }
611
                break;
afd0be38 Andy McCall ↶ 612
            }
613
            if !hosted {
614
                left_out.push(format!("subrepo {path} is left out: the repository it pins is not on this hub"));
fd3db644 Andy McCall ↶ 615
            }
616
        }
617
    }
afd0be38 Andy McCall ↶ 618
    (inputs, ids, left_out)
42fa9554 Andy McCall ↶ 619
}
620
621
async fn running_job(app: &Shared, id: i64) -> Result<fdb::Job, ApiError> {
622
    match fdb::job(&app.db, id).await? {
623
        Some(j) if j.status == "running" => Ok(j),
624
        Some(_) => Err(ApiError(StatusCode::CONFLICT, "this job is not running".into())),
625
        None => Err(ApiError(StatusCode::NOT_FOUND, "no such job".into())),
626
    }
627
}
628
629
/// `POST /api/v1/flow/jobs/{id}/log`.
630
pub async fn log(
631
    State(app): State<Shared>,
632
    headers: HeaderMap,
633
    Path(id): Path<i64>,
634
    Json(mut batch): Json<LogBatch>,
635
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 636
    runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 637
    let job = running_job(&app, id).await?;
745d225d Andy McCall ↶ 638
    // An empty batch asks only whether the job still runs: a cancelled
639
    // one answers 409 and its runner stops it (oxygenflow#32).
640
    if batch.records.is_empty() {
641
        return Ok(StatusCode::NO_CONTENT);
642
    }
42fa9554 Andy McCall ↶ 643
    for r in &mut batch.records {
644
        if r.text.len() > MAX_RECORD_TEXT {
645
            let mut cut = MAX_RECORD_TEXT;
646
            while !r.text.is_char_boundary(cut) {
647
                cut -= 1;
648
            }
649
            r.text.truncate(cut);
650
        }
651
        if !matches!(r.stream.as_str(), "out" | "err" | "console" | "flow") {
652
            r.stream = "console".into();
653
        }
654
    }
655
    let kept = fdb::append_log(&app.db, job.id, &batch.records, MAX_RECORDS as i64).await?;
656
    if kept < batch.records.len() && job.log_count < MAX_RECORDS as i64 {
657
        tracing::info!("flow: job {} reached the log limit", job.id);
658
    }
659
    Ok(StatusCode::NO_CONTENT)
660
}
661
a3c8288b Andy McCall ↶ 662
/// Largest artifact the hub keeps; the runner refuses larger ones first.
663
pub const MAX_ARTIFACT_BYTES: usize = 512 << 20;
664
665
/// `POST /api/v1/flow/jobs/{id}/artifacts/{name}`: the bytes of an artifact
666
/// the job declares (OxygenFlow spec §5.2), kept as a blob of the run's
667
/// repository. Answers `{"blob"}`.
668
pub async fn artifact(
669
    State(app): State<Shared>,
670
    headers: HeaderMap,
671
    Path((id, name)): Path<(i64, String)>,
672
    body: axum::body::Bytes,
673
) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 674
    runner_auth(&app, &headers).await?;
a3c8288b Andy McCall ↶ 675
    let job = running_job(&app, id).await?;
676
    let spec: flow_core::Job =
677
        serde_json::from_str(&job.spec).map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}")))?;
678
    let Some(out) = spec.outputs.iter().find(|o| o.name == name) else {
679
        return Err(ApiError(StatusCode::BAD_REQUEST, format!("job {} declares no artifact {name:?}", job.name)));
680
    };
681
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
682
    let row =
683
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
684
    let repo = RepoName::new(&row.owner, &row.name).ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "bad repository name".into()))?;
685
    let content = blake3::hash(&body).to_hex().to_string();
686
    let size = body.len() as i64;
687
    let blob = app
688
        .repos
689
        .write(&repo, move |r, _| Ok(r.store_blob(&body)?))
690
        .await
691
        .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("keep the artifact: {e}")))?;
692
    let a = fdb::Artifact { job_id: job.id, name, path: out.path.clone(), blob: blob.to_hex(), size, content, created_ms: db::now_ms() };
693
    fdb::add_artifact(&app.db, &a).await?;
694
    Ok(Json(serde_json::json!({ "blob": a.blob })).into_response())
695
}
696
42fa9554 Andy McCall ↶ 697
/// `POST /api/v1/flow/jobs/{id}/finish`.
698
pub async fn finish(
699
    State(app): State<Shared>,
700
    headers: HeaderMap,
701
    Path(id): Path<i64>,
702
    Json(f): Json<Finish>,
703
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 704
    let me = runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 705
    let job = running_job(&app, id).await?;
706
    let status = match (f.exit_code, &f.error) {
707
        (Some(0), None) => "passed",
708
        (Some(_), _) => "failed",
709
        (None, _) => "error",
710
    };
711
    let (receipt, receipt_hash) = match &f.receipt {
712
        Some(r) if r.job_name == job.name && Some(r.exit_code as i32) == f.exit_code => {
a3c8288b Andy McCall ↶ 713
            // The receipt names exactly the artifacts uploaded, in name order.
714
            let kept: Vec<String> = fdb::artifacts(&app.db, job.id).await?.into_iter().map(|a| a.blob).collect();
715
            if r.outputs.iter().map(hex::encode).collect::<Vec<_>>() != kept {
716
                return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt names other outputs than the artifacts uploaded".into()));
717
            }
42fa9554 Andy McCall ↶ 718
            (serde_json::to_string(r).unwrap_or_default(), hex::encode(r.hash()))
719
        }
720
        Some(_) => return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt does not match the job".into())),
721
        None => (String::new(), String::new()),
722
    };
723
    let error = f.error.clone().unwrap_or_default();
724
    let changed = fdb::finish(
725
        &app.db,
726
        job.id,
727
        fdb::Outcome { status, exit_code: f.exit_code.map(i64::from), error: &error, receipt: &receipt, receipt_hash: &receipt_hash },
728
    )
729
    .await?;
730
    if let Some(run) = fdb::run_by_id(&app.db, job.run_id).await? {
731
        if let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? {
cc19697f Andy McCall ↶ 732
            if let (Some(r), Some(signed)) = (&f.receipt, &f.signed_receipt) {
b6b7e4cf Andy McCall ↶ 733
                let (attest, signer, note) = match keep_receipt(&app, &me, &row, &run, r, signed).await {
cc19697f Andy McCall ↶ 734
                    Ok((attest, signer)) => (attest, signer, String::new()),
735
                    Err(e) => {
736
                        tracing::warn!("flow: job {} receipt not kept: {e}", job.id);
737
                        (String::new(), String::new(), e)
738
                    }
739
                };
740
                fdb::set_receipt(&app.db, job.id, &attest, &signer, &note).await?;
741
            }
42fa9554 Andy McCall ↶ 742
        }
743
    }
070a7b64 Andy McCall ↶ 744
    report_changed(&app, job.run_id, changed).await?;
42fa9554 Andy McCall ↶ 745
    Ok(StatusCode::NO_CONTENT)
070a7b64 Andy McCall ↶ 746
}
747
748
/// Report the checks of jobs whose status changed.
749
async fn report_changed(app: &Shared, run_id: i64, changed: Vec<fdb::Job>) -> sqlx::Result<()> {
750
    let Some(run) = fdb::run_by_id(&app.db, run_id).await? else {
751
        return Ok(());
752
    };
753
    let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? else {
754
        return Ok(());
755
    };
756
    for j in changed {
757
        let summary = match j.status.as_str() {
758
            "passed" => format!("passed in {}", duration(&j)),
759
            "failed" => format!("exit {} after {}", j.exit_code.unwrap_or(-1), duration(&j)),
760
            "skipped" => "skipped: a dependency did not pass".into(),
745d225d Andy McCall ↶ 761
            "cancelled" => j.error.clone(),
070a7b64 Andy McCall ↶ 762
            _ => format!("could not run: {}", j.error),
763
        };
764
        report_check(app, &row, &run.state, run.number, &j.name, check_status(&j.status), &summary).await;
765
    }
766
    Ok(())
767
}
768
5dead1f4 Andy McCall ↶ 769
/// The site's job time limits (oxygenhub#89): set on Admin → Runners →
770
/// Settings, else from `hub.toml`.
771
pub struct Limits {
772
    /// For a job whose pipeline names none.
773
    pub default_s: u64,
774
    /// The most a pipeline may ask for.
775
    pub max_s: u64,
776
    /// Grace past a job's limit before the hub gives it up: a runner stops
777
    /// a job at its limit and reports it within moments, so one still
778
    /// running this long after is one the runner no longer reports (it was
779
    /// restarted, or lost the hub while the job ended, oxygenhub#71).
780
    pub grace_ms: i64,
781
}
782
783
pub const SETTING_DEFAULT: &str = "flow.job_timeout_minutes";
784
pub const SETTING_MAX: &str = "flow.job_timeout_max_minutes";
785
pub const SETTING_GRACE: &str = "flow.lost_grace_minutes";
786
787
pub async fn limits(app: &Shared) -> Limits {
788
    let minutes = |key: &'static str| async move {
789
        db::accounts::setting(&app.db, key).await.ok().flatten().and_then(|v| v.parse::<u64>().ok()).filter(|m| *m > 0)
790
    };
791
    let default_m = minutes(SETTING_DEFAULT).await.unwrap_or(app.config.flow.job_timeout_minutes.max(1));
792
    let max_m = minutes(SETTING_MAX).await.unwrap_or(default_m.max(6 * 60)).max(default_m);
793
    let grace_m = minutes(SETTING_GRACE).await.unwrap_or(5);
794
    Limits { default_s: default_m * 60, max_s: max_m * 60, grace_ms: grace_m as i64 * 60_000 }
795
}
070a7b64 Andy McCall ↶ 796
5dead1f4 Andy McCall ↶ 797
/// End every job still running past its time limit plus the grace:
070a7b64 Andy McCall ↶ 798
/// it is marked an error with the reason, the jobs that depended on it are
799
/// skipped and its run settled, as when a runner reports an error.
800
pub async fn reap_lost_jobs(app: &Shared) -> sqlx::Result<usize> {
5dead1f4 Andy McCall ↶ 801
    let limits = limits(app).await;
802
    let lost = fdb::running_past_limit(&app.db, db::now_ms(), limits.default_s, limits.grace_ms).await?;
070a7b64 Andy McCall ↶ 803
    let n = lost.len();
804
    for job in lost {
805
        let error = format!(
806
            "lost: still running {} minutes after it started, past the time limit; the runner{} stopped reporting it",
807
            (db::now_ms() - job.started_ms.unwrap_or(0)) / 60_000,
808
            if job.runner.is_empty() { String::new() } else { format!(" {}", job.runner) },
809
        );
810
        let changed =
811
            fdb::finish(&app.db, job.id, fdb::Outcome { status: "error", exit_code: None, error: &error, receipt: "", receipt_hash: "" })
812
                .await?;
813
        tracing::warn!("flow: job {} {error}", job.id);
814
        report_changed(app, job.run_id, changed).await?;
815
    }
816
    Ok(n)
eef677c4 Andy McCall ↶ 817
}
818
819
/// End the jobs running on `runner` that it does not hold (oxygenhub#71):
820
/// it restarted since claiming them, or never received the claim. A
821
/// runner claims one job at a time, so a job it holds is always listed.
822
async fn end_jobs_not_held(app: &Shared, runner: &db::runners::Runner, holding: &[i64]) -> sqlx::Result<()> {
823
    for job in fdb::running_on(&app.db, runner.id).await? {
824
        if holding.contains(&job.id) {
825
            continue;
826
        }
827
        let error = format!("lost: the runner {} no longer holds this job (it restarted, or never received it)", runner.name);
828
        let changed =
829
            fdb::finish(&app.db, job.id, fdb::Outcome { status: "error", exit_code: None, error: &error, receipt: "", receipt_hash: "" })
830
                .await?;
831
        tracing::warn!("flow: job {} {error}", job.id);
832
        report_changed(app, job.run_id, changed).await?;
833
    }
834
    Ok(())
348f8c4b Andy McCall ↶ 835
}
836
837
/// Start the scheduled runs due at `when` (oxygenflow#24): on each
838
/// repository that runs Flow, the jobs of its default bookmark's pipeline
839
/// with a `schedule` line due then. Returns how many runs started.
840
pub async fn run_due(app: &Shared, when: flow_core::pipeline::When) -> usize {
841
    let Ok(rows) = db::repos::list(&app.db).await else { return 0 };
842
    let mut started = 0;
843
    for row in rows.into_iter().filter(|r| enabled(app, r)) {
844
        let Some(repo) = RepoName::new(&row.owner, &row.name) else { continue };
845
        let b = row.default_bookmark.clone();
846
        let found = app
847
            .repos
848
            .read(&repo, move |r, _| {
849
                let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) };
850
                Ok(Some((st, r.store().get_state(&st)?.heads)))
851
            })
852
            .await;
853
        let Ok(Some((state, heads))) = found else { continue };
854
        if heads.is_empty() {
855
            continue;
856
        }
857
        match start_run_at(app, &row, &row.default_bookmark, &state.to_hex(), &heads, &row.owner, "schedule", None, Some(when)).await {
858
            Ok(Some(_)) => started += 1,
859
            Ok(None) => {}
860
            Err(e) => tracing::warn!("flow: scheduled run of {}/{}: {e}", row.owner, row.name),
861
        }
862
    }
863
    started
864
}
865
866
/// The hub's clock as a [`When`](flow_core::pipeline::When).
867
pub fn when_now() -> flow_core::pipeline::When {
868
    let now = time::OffsetDateTime::now_utc();
869
    flow_core::pipeline::When {
870
        minute: now.minute() as u32,
871
        hour: now.hour() as u32,
872
        day: now.day() as u32,
873
        month: u8::from(now.month()) as u32,
874
        weekday: now.weekday().number_days_from_sunday() as u32,
875
    }
876
}
877
878
/// Start the scheduled runs due each minute, once a minute.
879
pub async fn scheduler(app: Shared) {
880
    let mut last = None;
881
    loop {
882
        tokio::time::sleep(std::time::Duration::from_secs(15)).await;
883
        let minute = time::OffsetDateTime::now_utc().unix_timestamp() / 60;
884
        if last == Some(minute) {
885
            continue;
886
        }
887
        last = Some(minute);
888
        let n = run_due(&app, when_now()).await;
889
        if n > 0 {
890
            tracing::info!("flow: {n} scheduled run(s) started");
891
        }
892
    }
070a7b64 Andy McCall ↶ 893
}
894
895
/// Look for lost jobs once a minute.
896
pub async fn reaper(app: Shared) {
897
    loop {
898
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
899
        if let Err(e) = reap_lost_jobs(&app).await {
900
            tracing::warn!("flow: looking for lost jobs: {e}");
901
        }
902
    }
cc19697f Andy McCall ↶ 903
}
904
905
/// Check a runner's signed receipt for a job of `run` and keep it in the
906
/// repository (Oxygen spec §14.6): the attestation's hex and the runner's
907
/// name, or why it was not kept. It must be the job's receipt, for the
908
/// run's state and repository, signed by the key the runner's token first
909
/// signed with.
910
pub async fn keep_receipt(
911
    app: &Shared,
b6b7e4cf Andy McCall ↶ 912
    runner: &db::runners::Runner,
cc19697f Andy McCall ↶ 913
    row: &db::repos::RepoRow,
914
    run: &fdb::Run,
915
    receipt: &flow_core::Receipt,
916
    signed_hex: &str,
917
) -> Result<(String, String), String> {
918
    use oxy_core::receipt::SignedReceipt;
919
    let bytes = hex::decode(signed_hex).map_err(|_| "the signed receipt is not hex".to_string())?;
920
    let sr = SignedReceipt::decode(&bytes).map_err(|e| format!("not a signed receipt: {e}"))?;
921
    sr.verify().map_err(|e| format!("it does not verify: {e}"))?;
922
    if sr.receipt.hash().0 != receipt.hash() {
923
        return Err("it signs another receipt than the job's".into());
924
    }
925
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
926
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(heads).encode());
927
    if sr.state() != state {
928
        return Err("it is for another state than the run's".into());
929
    }
930
    let key = sr.runner.hash().to_hex();
b6b7e4cf Andy McCall ↶ 931
    let bound = db::runners::bind_key(&app.db, runner.id, &key, &sr.runner.name).await.map_err(|e| e.to_string())?;
cc19697f Andy McCall ↶ 932
    if bound != key {
933
        return Err(format!("signed by {}, not the key this runner signs with ({})", &key[..12], &bound[..12.min(bound.len())]));
934
    }
935
    let name = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
936
    let signer = sr.runner.name.clone();
937
    let attest = app.repos.write(&name, move |repo, _| Ok(repo.add_receipt(&sr)?)).await.map_err(|e| e.to_string())?;
938
    Ok((attest.to_hex(), signer))
42fa9554 Andy McCall ↶ 939
}
940
941
/// A job's wall time, as the pages show it.
942
pub fn duration(j: &fdb::Job) -> String {
943
    match (j.started_ms, j.finished_ms) {
944
        (Some(a), Some(b)) => human_ms(b - a),
945
        (Some(a), None) => human_ms(db::now_ms() - a),
946
        _ => "–".into(),
947
    }
948
}
949
950
pub fn human_ms(ms: i64) -> String {
951
    let ms = ms.max(0);
952
    if ms < 1000 {
953
        format!("{ms} ms")
954
    } else if ms < 60_000 {
955
        format!("{:.1} s", ms as f64 / 1000.0)
956
    } else {
957
        format!("{} min {} s", ms / 60_000, (ms % 60_000) / 1000)
958
    }
959
}
bf3d0754 Andy McCall ↶ 960
961
/// Where a live run stands among the runners' work: what its viewers may
962
/// know of it. Jobs of other repositories are named only when those are
963
/// public; the rest are counted.
964
#[derive(Clone, Debug, Default, PartialEq, Eq)]
965
pub struct QueueInfo {
966
    /// 1-based place of the run's first queued job among all queued jobs.
967
    pub position: Option<usize>,
968
    /// Jobs running now, and queued jobs ahead of the run's first one.
969
    pub running: Vec<String>,
970
    pub ahead: Vec<String>,
971
    /// Of those, jobs in repositories the page may not name.
972
    pub hidden: usize,
973
}
974
975
impl QueueInfo {
976
    /// "3rd in line" and the like.
977
    pub fn place(&self) -> Option<String> {
978
        self.position.map(|n| {
979
            let suffix = match (n % 10, n % 100) {
980
                (1, 11) | (2, 12) | (3, 13) => "th",
981
                (1, _) => "st",
982
                (2, _) => "nd",
983
                (3, _) => "rd",
984
                _ => "th",
985
            };
986
            format!("{n}{suffix} in line")
987
        })
988
    }
989
}
990
991
/// The queue as `run` (of repository `repo_id`) sees it, from `queue`.
992
pub fn queue_info(queue: &[fdb::QueueEntry], repo_id: i64, run_id: i64) -> QueueInfo {
993
    let mut info = QueueInfo::default();
994
    let first = queue.iter().filter(|e| e.status == "queued").position(|e| e.run_id == run_id);
995
    info.position = first.map(|i| i + 1);
996
    let label = |e: &fdb::QueueEntry| format!("{}/{} #{} {}", e.owner, e.name, e.run_number, e.job);
997
    let mut queued_seen = 0;
998
    for e in queue {
999
        if e.run_id == run_id {
1000
            continue;
1001
        }
1002
        let ahead = match e.status.as_str() {
1003
            "running" => true,
1004
            _ => {
1005
                queued_seen += 1;
1006
                first.is_some_and(|f| queued_seen <= f)
1007
            }
1008
        };
1009
        if !ahead {
1010
            continue;
1011
        }
1012
        if e.repo_id != repo_id && e.visibility != "public" {
1013
            info.hidden += 1;
1014
        } else if e.status == "running" {
1015
            info.running.push(label(e));
1016
        } else {
1017
            info.ahead.push(label(e));
1018
        }
1019
    }
1020
    info
1021
}
1022
1023
#[cfg(test)]
1024
mod queue_tests {
1025
    use super::*;
1026
1027
    fn e(job_id: i64, run_id: i64, repo_id: i64, vis: &str, job: &str, status: &str) -> fdb::QueueEntry {
1028
        fdb::QueueEntry {
1029
            job_id,
1030
            run_id,
1031
            repo_id,
1032
            owner: "o".into(),
1033
            name: format!("r{repo_id}"),
1034
            visibility: vis.into(),
1035
            run_number: run_id,
1036
            job: job.into(),
1037
            status: status.into(),
b66e70b7 Andy McCall ↶ 1038
            runner: String::new(),
c4c87f48 Andy McCall ↶ 1039
            spec: String::new(),
1040
            sandbox: String::new(),
bf3d0754 Andy McCall ↶ 1041
        }
1042
    }
1043
1044
    #[test]
1045
    fn a_run_learns_its_place_and_what_is_ahead_of_it() {
1046
        // Running first, then queued in claim order (as `fdb::queue` returns them).
1047
        let q = vec![
1048
            e(1, 10, 1, "public", "build", "running"),
1049
            e(2, 11, 2, "private", "test", "running"),
1050
            e(3, 10, 1, "public", "test", "queued"),
1051
            e(4, 12, 3, "private", "lint", "queued"),
1052
            e(5, 13, 4, "public", "build", "queued"),
1053
            e(6, 13, 4, "public", "test", "queued"),
1054
            e(7, 14, 5, "public", "late", "queued"),
1055
        ];
1056
        let info = queue_info(&q, 4, 13);
1057
        assert_eq!(info.position, Some(3));
1058
        assert_eq!(info.place().as_deref(), Some("3rd in line"));
1059
        assert_eq!(info.running, ["o/r1 #10 build"]);
1060
        assert_eq!(info.ahead, ["o/r1 #10 test"], "queued ahead, not behind");
1061
        assert_eq!(info.hidden, 2, "private repositories are counted, not named");
1062
        // A private repository's own jobs are named to its viewers.
1063
        assert_eq!(queue_info(&q, 3, 12).ahead, ["o/r1 #10 test"]);
1064
        assert_eq!(queue_info(&q, 3, 12).position, Some(2));
1065
        // A run with nothing queued has no place.
1066
        assert_eq!(queue_info(&q, 1, 99).position, None);
1067
        let ordinal = |n| QueueInfo { position: Some(n), ..Default::default() }.place().unwrap();
1068
        assert_eq!(
1069
            [ordinal(1), ordinal(2), ordinal(11), ordinal(22), ordinal(113)],
1070
            ["1st in line", "2nd in line", "11th in line", "22nd in line", "113th in line"]
1071
        );
1072
    }
1073
}