OXYGENHUB
# ced6d3afc5cf oxygenhub / crates / hub / src / flow.rs
32.7 KiB · Rust · last changed Runners are offered only jobs whose image they hold (closes #73) 1 month ago source blame history raw
42fa9554 Andy McCall ↶ 1
//! OxygenFlow on the hub (hub spec §13.2): runs started by pushes, the
2
//! runner API, job tokens and the checks that mirror jobs. Pipelines are
3
//! parsed by `flow-core`, the same code `flow check` runs.
4
5
use crate::api::ApiError;
6
use crate::db::{self, flow as fdb};
7
use crate::repos::RepoName;
8
use crate::Shared;
9
use axum::extract::{Path, State};
10
use axum::http::{HeaderMap, StatusCode};
11
use axum::response::{IntoResponse, Response};
12
use axum::Json;
13
use flow_core::api::{Claim, ClaimRequest, Finish, LogBatch, MAX_RECORDS, MAX_RECORD_TEXT};
14
use flow_core::{Pipeline, PIPELINE_PATH};
15
use oxy_core::hash::ChangeHash;
16
17
/// Prefix of job tokens, so the wire protocol can tell them from personal tokens.
18
pub const JOB_TOKEN_PREFIX: &str = "flowjob_";
19
/// A job token is refused this long after its job started, whatever its state.
20
pub const JOB_TOKEN_MAX_AGE_MS: i64 = 2 * 3600 * 1000;
21
9008bf6c Andy McCall ↶ 22
/// Whether pushes to this repository start runs: its own setting when a
23
/// site administrator made one, else hub.toml's `[flow] repos`.
24
pub fn enabled(app: &Shared, row: &db::repos::RepoRow) -> bool {
25
    match row.flow.as_str() {
26
        "on" => true,
27
        "off" => false,
28
        _ => app.config.flow.allows(&row.owner, &row.name),
29
    }
30
}
31
42fa9554 Andy McCall ↶ 32
/// A bookmark moved by a push: start a run when the repository is allowed
33
/// and the new state has a pipeline.
34
pub async fn on_push(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, state: &str, heads: &[ChangeHash], actor: &str) {
9008bf6c Andy McCall ↶ 35
    if bookmark.starts_with("tags/") || heads.is_empty() || !enabled(app, row) {
42fa9554 Andy McCall ↶ 36
        return;
37
    }
1edac4b6 Andy McCall ↶ 38
    if let Err(e) = start_run(app, row, bookmark, state, heads, actor, "push").await {
39
        tracing::warn!("flow: starting a run of {}/{}: {e}", row.owner, row.name);
40
    }
41
}
42
43
/// "Run pipeline" (hub spec §13.2): run a bookmark's current state without
44
/// moving it. Returns the new run's number.
45
pub async fn run_bookmark(app: &Shared, row: &db::repos::RepoRow, bookmark: &str, actor: &str) -> Result<i64, String> {
46
    if !enabled(app, row) {
47
        return Err("OxygenFlow is not enabled for this repository".into());
48
    }
49
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
50
    let b = bookmark.to_string();
51
    let found = app
52
        .repos
53
        .read(&repo, move |r, _| {
54
            let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) };
55
            Ok(Some((st, r.store().get_state(&st)?.heads)))
56
        })
57
        .await
58
        .map_err(|e| e.to_string())?;
59
    let Some((state, heads)) = found else { return Err(format!("no bookmark {bookmark}")) };
60
    if heads.is_empty() {
61
        return Err(format!("{bookmark} has no changes yet"));
62
    }
63
    start_run(app, row, bookmark, &state.to_hex(), &heads, actor, "manual")
64
        .await?
65
        .ok_or_else(|| format!("{bookmark} has no {PIPELINE_PATH}"))
66
}
67
4f692533 Andy McCall ↶ 68
/// "Run pipeline" on one committed change: the state whose only head is
69
/// `change`, whatever bookmarks hold it. Returns the new run's number.
70
pub async fn run_change(app: &Shared, row: &db::repos::RepoRow, change: &str, actor: &str) -> Result<i64, String> {
71
    if !enabled(app, row) {
72
        return Err("OxygenFlow is not enabled for this repository".into());
73
    }
74
    let h = ChangeHash::from_hex(change.trim().trim_start_matches('#')).ok_or("give the full hash of a change")?;
75
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
df15b747 Andy McCall ↶ 76
    let known = app.repos.read(&repo, move |r, _| Ok(r.store().get_change_meta(&h).is_ok())).await.map_err(|e| e.to_string())?;
4f692533 Andy McCall ↶ 77
    if !known {
78
        return Err(format!("no change {} in this repository", &change.trim()[..12.min(change.trim().len())]));
79
    }
80
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(vec![h]).encode());
81
    start_run(app, row, "", &state.to_hex(), &[h], actor, "manual")
82
        .await?
83
        .ok_or_else(|| format!("that change's state has no {PIPELINE_PATH}"))
84
}
85
1edac4b6 Andy McCall ↶ 86
/// Re-run a finished run: the same state, bookmark and pipeline, as a new
87
/// run whose checks replace the old ones. Returns the new run's number.
88
pub async fn rerun(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run, actor: &str) -> Result<i64, String> {
89
    if !enabled(app, row) {
90
        return Err("OxygenFlow is not enabled for this repository".into());
91
    }
92
    if run.live() {
93
        return Err(format!("run #{} is still {}", run.number, run.status));
94
    }
95
    let heads: Vec<ChangeHash> = run.head_list().into_iter().filter_map(ChangeHash::from_hex).collect();
96
    if heads.is_empty() {
97
        return Err("the run names no changes".into());
98
    }
99
    start_run(app, row, &run.bookmark, &run.state, &heads, actor, &format!("rerun:{}", run.number))
100
        .await?
101
        .ok_or_else(|| format!("that state has no {PIPELINE_PATH}"))
102
}
103
104
/// Start a run of the pipeline in the state `heads` make (whose state
105
/// hash is `state`), recorded on `bookmark`: for a push, "Run pipeline"
106
/// (`trigger` = `manual`) and re-runs (`rerun:N`). `actor` is the account
107
/// whose access the run's subrepo reads are checked against. Returns the
108
/// run's number, or `None` when the state has no pipeline. The caller
109
/// checks that Flow is enabled.
110
pub async fn start_run(
111
    app: &Shared,
112
    row: &db::repos::RepoRow,
113
    bookmark: &str,
114
    state: &str,
115
    heads: &[ChangeHash],
116
    actor: &str,
117
    trigger: &str,
118
) -> Result<Option<i64>, String> {
119
    let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
42fa9554 Andy McCall ↶ 120
    let h = heads.to_vec();
121
    let text = app
122
        .repos
123
        .read(&repo, move |r, cache| {
124
            let loaded = cache.load(r, &h)?;
125
            Ok(match loaded.snapshot.files.get(PIPELINE_PATH).map(|f| &f.content) {
126
                Some(oxy_core::state::FileContent::Text { bytes, .. }) => Some(String::from_utf8_lossy(bytes).into_owned()),
127
                _ => None,
128
            })
129
        })
1edac4b6 Andy McCall ↶ 130
        .await
131
        .map_err(|e| format!("reading the pipeline: {e}"))?;
132
    let Some(text) = text else { return Ok(None) };
42fa9554 Andy McCall ↶ 133
    let mut hex: Vec<String> = heads.iter().map(|h| h.to_hex()).collect();
134
    hex.sort();
135
    let heads_s = hex.join("+");
1edac4b6 Andy McCall ↶ 136
    let base = db::flow::NewRun { repo_id: row.id, bookmark, state, heads: &heads_s, pipeline_hash: "", actor, trigger, error: None };
137
    let id = match Pipeline::parse(&text) {
42fa9554 Andy McCall ↶ 138
        Err(problems) => {
139
            let msg = problems.to_string();
1edac4b6 Andy McCall ↶ 140
            db::flow::create_run(&app.db, db::flow::NewRun { error: Some(&msg), ..base }, &[]).await.map_err(|e| e.to_string())?
42fa9554 Andy McCall ↶ 141
        }
142
        Ok(p) => {
143
            let depths = p.depths();
144
            let jobs: Vec<fdb::NewJob> = p
145
                .order()
146
                .into_iter()
147
                .map(|j| {
148
                    let sb = &p.sandboxes[&j.sandbox];
149
                    fdb::NewJob {
150
                        name: j.name.clone(),
151
                        depth: depths[j.name.as_str()] as i64,
152
                        depends: j.depends_on.clone(),
153
                        spec: serde_json::to_string(j).unwrap_or_default(),
154
                        sandbox: serde_json::to_string(sb).unwrap_or_default(),
155
                        runtime: sb.runtime.as_str().into(),
156
                    }
157
                })
158
                .collect();
159
            let hash = hex::encode(p.hash());
1edac4b6 Andy McCall ↶ 160
            let id =
161
                db::flow::create_run(&app.db, db::flow::NewRun { pipeline_hash: &hash, ..base }, &jobs).await.map_err(|e| e.to_string())?;
162
            if let Ok(Some(run)) = fdb::run_by_id(&app.db, id).await {
42fa9554 Andy McCall ↶ 163
                for j in &jobs {
a0d4bbfb Andy McCall ↶ 164
                    report_check(app, row, state, run.number, &j.name, "queued", "waiting for a runner").await;
42fa9554 Andy McCall ↶ 165
                }
166
            }
167
            id
168
        }
169
    };
1edac4b6 Andy McCall ↶ 170
    let run = fdb::run_by_id(&app.db, id).await.map_err(|e| e.to_string())?.ok_or("the run vanished")?;
171
    Ok(Some(run.number))
42fa9554 Andy McCall ↶ 172
}
173
a0d4bbfb Andy McCall ↶ 174
/// Record a job's check; it links to the run's page by the run's number.
175
async fn report_check(app: &Shared, row: &db::repos::RepoRow, state: &str, run_number: i64, job: &str, status: &str, summary: &str) {
176
    let url = format!("{}/{}/{}/flow/{run_number}", app.config.base_url(), row.owner, row.name);
42fa9554 Andy McCall ↶ 177
    let name = format!("flow/{job}");
178
    let c = db::hooks::upsert_check(
179
        &app.db,
180
        row.id,
181
        db::hooks::CheckReport { state, name: &name, status, summary, url: &url, reporter: None, seal: "" },
182
    )
183
    .await;
184
    if let Ok(c) = c {
185
        let api = crate::api::check_of(&c);
186
        app.events.proposal(row.id, hub_api::ProposalEvent { number: 0, kind: "check".into(), check: Some(api.clone()) });
187
        crate::hooks::emit(app, row, "check", "flow", serde_json::json!({ "check": api })).await;
188
    }
189
}
190
191
/// The check status a job's status shows as.
192
fn check_status(job_status: &str) -> &'static str {
193
    match job_status {
194
        "queued" => "queued",
195
        "running" => "running",
196
        "passed" => "passed",
197
        "skipped" => "skipped",
198
        _ => "failed",
199
    }
200
}
201
b6b7e4cf Andy McCall ↶ 202
/// The runner a bearer token belongs to: a registered runner, or one
203
/// configured with a hub.toml token (tracked from its first request). A
204
/// disabled runner is refused.
205
async fn runner_auth(app: &Shared, headers: &HeaderMap) -> Result<db::runners::Runner, ApiError> {
42fa9554 Andy McCall ↶ 206
    let Some(t) = oxy_server::bearer_token(headers) else {
207
        return Err(ApiError(StatusCode::UNAUTHORIZED, "a runner token is required".into()));
b6b7e4cf Andy McCall ↶ 208
    };
209
    let hash = blake3::hash(t.as_bytes()).to_hex().to_string();
210
    let runner = match db::runners::by_token(&app.db, &hash).await? {
211
        Some(r) => r,
212
        None if app.config.flow.runner_tokens.iter().any(|x| crate::auth::mac_equal(x, t)) => {
213
            db::runners::configured(&app.db, &hash, "runner").await?
214
        }
215
        None => return Err(ApiError(StatusCode::UNAUTHORIZED, "not a runner token".into())),
42fa9554 Andy McCall ↶ 216
    };
b6b7e4cf Andy McCall ↶ 217
    if runner.state == "disabled" {
218
        return Err(ApiError(StatusCode::FORBIDDEN, format!("runner {} is disabled on this hub", runner.name)));
42fa9554 Andy McCall ↶ 219
    }
b6b7e4cf Andy McCall ↶ 220
    Ok(runner)
42fa9554 Andy McCall ↶ 221
}
222
b6b7e4cf Andy McCall ↶ 223
/// `POST /api/v1/flow/runners/register`: a one-time registration token
224
/// from Admin → Runners buys a runner credential, binding the runner's
225
/// signing identity (hub spec §13.2).
226
pub async fn register(
227
    State(app): State<Shared>,
228
    Json(req): Json<flow_core::api::Register>,
229
) -> Result<Json<flow_core::api::Registered>, ApiError> {
230
    let bytes = hex::decode(req.identity.trim()).map_err(|_| ApiError(StatusCode::BAD_REQUEST, "identity: not hex".into()))?;
231
    let identity = oxy_core::identity::Identity::decode(&bytes).map_err(|e| ApiError(StatusCode::BAD_REQUEST, format!("identity: {e}")))?;
232
    let name: String = req.name.chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')).take(64).collect();
233
    if name.is_empty() {
234
        return Err(ApiError(StatusCode::BAD_REQUEST, "give the runner a name of letters, digits, - _ or .".into()));
235
    }
236
    let credential = format!("flowrunner_{}", crate::auth::random_hex(32));
237
    let hash = blake3::hash(credential.as_bytes()).to_hex().to_string();
238
    let reg = blake3::hash(req.token.trim().as_bytes()).to_hex().to_string();
239
    let runner =
240
        db::runners::register(&app.db, &reg, &name, &hash, &identity.hash().to_hex(), &identity.name, "").await?.ok_or_else(|| {
241
            ApiError(StatusCode::FORBIDDEN, "that registration token is unknown, used or expired; create one on Admin → Runners".into())
242
        })?;
243
    db::runners::seen(&app.db, runner.id, None, Some(&req.info)).await?;
244
    tracing::info!("flow: runner {} registered (key {})", runner.name, &runner.key[..12]);
245
    Ok(Json(flow_core::api::Registered { id: runner.id, name: runner.name, credential }))
246
}
247
42fa9554 Andy McCall ↶ 248
/// `POST /api/v1/flow/claim`.
249
pub async fn claim(State(app): State<Shared>, headers: HeaderMap, Json(req): Json<ClaimRequest>) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 250
    let me = runner_auth(&app, &headers).await?;
251
    let claimed: String = req.runner.chars().filter(|c| !c.is_control()).take(64).collect();
252
    db::runners::seen(&app.db, me.id, Some(&claimed), req.info.as_ref()).await?;
253
    // A draining runner finishes what it has and takes nothing new.
254
    if me.state == "draining" {
255
        return Ok(StatusCode::NO_CONTENT.into_response());
256
    }
257
    let runner = if me.source == "registered" { me.name.clone() } else { claimed };
42fa9554 Andy McCall ↶ 258
    let secret = format!("{JOB_TOKEN_PREFIX}{}", crate::auth::random_hex(32));
b0612e7d Andy McCall ↶ 259
    let room = fdb::Room {
260
        memory_mib: req.free_memory_mib,
261
        cpus: req.free_cpus,
262
        images: req.images.as_ref().map(|i| i.iter().map(|h| h.to_ascii_lowercase()).collect()),
263
    };
c4c87f48 Andy McCall ↶ 264
    // The labels it reports now, or those it registered with.
265
    let labels: Vec<String> = match &req.info {
266
        Some(i) => i.labels.clone(),
267
        None => me.label_list().into_iter().map(String::from).collect(),
268
    };
269
    let Some(job) =
270
        fdb::claim_within(&app.db, &req.runtimes, &runner, &crate::auth::token_hash(&secret), room, &me.scope(), &labels).await?
271
    else {
42fa9554 Andy McCall ↶ 272
        return Ok(StatusCode::NO_CONTENT.into_response());
273
    };
b6b7e4cf Andy McCall ↶ 274
    db::runners::set_job_runner(&app.db, job.id, me.id).await?;
42fa9554 Andy McCall ↶ 275
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
276
    let row =
277
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
a0d4bbfb Andy McCall ↶ 278
    report_check(&app, &row, &run.state, run.number, &job.name, "running", &format!("running on {runner}")).await;
42fa9554 Andy McCall ↶ 279
    let bad = |e: serde_json::Error| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}"));
280
    let claim = Claim {
281
        job_id: job.id,
282
        run_id: run.id,
283
        repo: format!("{}/{}", row.owner, row.name),
284
        clone_url: format!("{}/{}/{}", app.config.base_url(), row.owner, row.name),
285
        token: secret,
286
        heads: run.head_list().into_iter().map(String::from).collect(),
287
        pipeline_hash: run.pipeline_hash.clone(),
288
        job: serde_json::from_str(&job.spec).map_err(bad)?,
289
        sandbox: serde_json::from_str(&job.sandbox).map_err(bad)?,
290
        timeout_s: app.config.flow.job_timeout_minutes.max(1) * 60,
fd3db644 Andy McCall ↶ 291
        subrepos: vec![],
9a065bc0 Andy McCall ↶ 292
        state: run.state.clone(),
293
        bookmark: run.bookmark.clone(),
294
        run_number: run.number,
42fa9554 Andy McCall ↶ 295
    };
fd3db644 Andy McCall ↶ 296
    let mut claim = claim;
297
    let (inputs, ids) = subrepo_inputs(&app, &row, &run).await;
298
    if !ids.is_empty() {
299
        fdb::set_also_read(&app.db, job.id, &ids).await?;
300
    }
301
    claim.subrepos = inputs;
42fa9554 Andy McCall ↶ 302
    Ok(Json(claim).into_response())
fd3db644 Andy McCall ↶ 303
}
304
25091e5c Andy McCall ↶ 305
/// The Oxygen subrepo pins in `name`'s state `heads`: (path, repository id, pinned heads).
306
async fn pins_in(
307
    app: &Shared,
308
    name: crate::repos::RepoName,
309
    heads: Vec<oxy_core::hash::ChangeHash>,
310
) -> Vec<(String, [u8; 16], Vec<oxy_core::hash::ChangeHash>)> {
311
    app.repos
fd3db644 Andy McCall ↶ 312
        .read(&name, move |repo, cache| {
313
            let loaded = cache.load(repo, &heads)?;
314
            let mut out = Vec::new();
315
            for (path, f) in &loaded.snapshot.files {
316
                if f.kind != oxy_core::objects::NodeKind::Subrepo {
317
                    continue;
318
                }
319
                if let oxy_core::state::FileContent::Opaque { blobs } = &f.content {
320
                    if let Some(b) = blobs.first() {
321
                        if let Ok(oxy_core::subrepo::Pin::Oxy { repo: id, heads, .. }) = repo
322
                            .read_blob(b)
323
                            .map_err(crate::repos::RepoError::Oxy)
324
                            .and_then(|bytes| oxy_core::subrepo::Pin::decode(&bytes).map_err(|e| crate::repos::RepoError::Oxy(e.into())))
325
                        {
326
                            out.push((path.clone(), id, heads));
327
                        }
328
                    }
329
                }
330
            }
331
            Ok(out)
332
        })
333
        .await
25091e5c Andy McCall ↶ 334
        .unwrap_or_default()
335
}
336
337
/// How deep subrepos of subrepos are followed.
338
const MAX_SUBREPO_DEPTH: usize = 3;
339
340
/// The Oxygen subrepos the run's state pins, and those their states pin in
341
/// turn (parents first, paths joined), that live on this hub and that the
342
/// run's pusher may read: what the job's workspace needs beside its own
343
/// tree. A pin the hub cannot serve is left out, with what it pins; the job
344
/// then fails on the missing files with an ordinary error.
345
async fn subrepo_inputs(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>) {
346
    let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![]) };
347
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
fd3db644 Andy McCall ↶ 348
    let pusher = db::accounts::account_by_name(&app.db, &run.actor).await.ok().flatten();
25091e5c Andy McCall ↶ 349
    let rows = db::repos::list(&app.db).await.unwrap_or_default();
fd3db644 Andy McCall ↶ 350
    let mut inputs = Vec::new();
351
    let mut ids = Vec::new();
25091e5c Andy McCall ↶ 352
    // (path prefix, repository, heads, depth), parents before children.
353
    let mut queue = std::collections::VecDeque::from([(String::new(), name, heads, 0usize)]);
354
    while let Some((prefix, name, heads, depth)) = queue.pop_front() {
355
        if depth == MAX_SUBREPO_DEPTH {
356
            continue;
357
        }
358
        for (path, id, pinned) in pins_in(app, name, heads).await {
359
            let path = if prefix.is_empty() { path } else { format!("{prefix}/{path}") };
360
            for other in &rows {
361
                let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue };
362
                let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false);
363
                if !matches {
364
                    continue;
365
                }
366
                let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten();
367
                if crate::auth::roles::allows(role, crate::auth::Access::Read) {
368
                    inputs.push(flow_core::api::SubrepoInput {
369
                        path: path.clone(),
370
                        repo: format!("{}/{}", other.owner, other.name),
371
                        clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name),
372
                        heads: pinned.iter().map(|h| h.to_hex()).collect(),
373
                    });
374
                    if !ids.contains(&other.id) {
375
                        ids.push(other.id);
376
                    }
377
                    queue.push_back((path.clone(), other_name, pinned.clone(), depth + 1));
378
                }
379
                break;
fd3db644 Andy McCall ↶ 380
            }
381
        }
382
    }
383
    (inputs, ids)
42fa9554 Andy McCall ↶ 384
}
385
386
async fn running_job(app: &Shared, id: i64) -> Result<fdb::Job, ApiError> {
387
    match fdb::job(&app.db, id).await? {
388
        Some(j) if j.status == "running" => Ok(j),
389
        Some(_) => Err(ApiError(StatusCode::CONFLICT, "this job is not running".into())),
390
        None => Err(ApiError(StatusCode::NOT_FOUND, "no such job".into())),
391
    }
392
}
393
394
/// `POST /api/v1/flow/jobs/{id}/log`.
395
pub async fn log(
396
    State(app): State<Shared>,
397
    headers: HeaderMap,
398
    Path(id): Path<i64>,
399
    Json(mut batch): Json<LogBatch>,
400
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 401
    runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 402
    let job = running_job(&app, id).await?;
403
    for r in &mut batch.records {
404
        if r.text.len() > MAX_RECORD_TEXT {
405
            let mut cut = MAX_RECORD_TEXT;
406
            while !r.text.is_char_boundary(cut) {
407
                cut -= 1;
408
            }
409
            r.text.truncate(cut);
410
        }
411
        if !matches!(r.stream.as_str(), "out" | "err" | "console" | "flow") {
412
            r.stream = "console".into();
413
        }
414
    }
415
    let kept = fdb::append_log(&app.db, job.id, &batch.records, MAX_RECORDS as i64).await?;
416
    if kept < batch.records.len() && job.log_count < MAX_RECORDS as i64 {
417
        tracing::info!("flow: job {} reached the log limit", job.id);
418
    }
419
    Ok(StatusCode::NO_CONTENT)
420
}
421
a3c8288b Andy McCall ↶ 422
/// Largest artifact the hub keeps; the runner refuses larger ones first.
423
pub const MAX_ARTIFACT_BYTES: usize = 512 << 20;
424
425
/// `POST /api/v1/flow/jobs/{id}/artifacts/{name}`: the bytes of an artifact
426
/// the job declares (OxygenFlow spec §5.2), kept as a blob of the run's
427
/// repository. Answers `{"blob"}`.
428
pub async fn artifact(
429
    State(app): State<Shared>,
430
    headers: HeaderMap,
431
    Path((id, name)): Path<(i64, String)>,
432
    body: axum::body::Bytes,
433
) -> Result<Response, ApiError> {
b6b7e4cf Andy McCall ↶ 434
    runner_auth(&app, &headers).await?;
a3c8288b Andy McCall ↶ 435
    let job = running_job(&app, id).await?;
436
    let spec: flow_core::Job =
437
        serde_json::from_str(&job.spec).map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("stored job: {e}")))?;
438
    let Some(out) = spec.outputs.iter().find(|o| o.name == name) else {
439
        return Err(ApiError(StatusCode::BAD_REQUEST, format!("job {} declares no artifact {name:?}", job.name)));
440
    };
441
    let run = fdb::run_by_id(&app.db, job.run_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such run".into()))?;
442
    let row =
443
        db::repos::get_by_id(&app.db, run.repo_id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "no such repository".into()))?;
444
    let repo = RepoName::new(&row.owner, &row.name).ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "bad repository name".into()))?;
445
    let content = blake3::hash(&body).to_hex().to_string();
446
    let size = body.len() as i64;
447
    let blob = app
448
        .repos
449
        .write(&repo, move |r, _| Ok(r.store_blob(&body)?))
450
        .await
451
        .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, format!("keep the artifact: {e}")))?;
452
    let a = fdb::Artifact { job_id: job.id, name, path: out.path.clone(), blob: blob.to_hex(), size, content, created_ms: db::now_ms() };
453
    fdb::add_artifact(&app.db, &a).await?;
454
    Ok(Json(serde_json::json!({ "blob": a.blob })).into_response())
455
}
456
42fa9554 Andy McCall ↶ 457
/// `POST /api/v1/flow/jobs/{id}/finish`.
458
pub async fn finish(
459
    State(app): State<Shared>,
460
    headers: HeaderMap,
461
    Path(id): Path<i64>,
462
    Json(f): Json<Finish>,
463
) -> Result<StatusCode, ApiError> {
b6b7e4cf Andy McCall ↶ 464
    let me = runner_auth(&app, &headers).await?;
42fa9554 Andy McCall ↶ 465
    let job = running_job(&app, id).await?;
466
    let status = match (f.exit_code, &f.error) {
467
        (Some(0), None) => "passed",
468
        (Some(_), _) => "failed",
469
        (None, _) => "error",
470
    };
471
    let (receipt, receipt_hash) = match &f.receipt {
472
        Some(r) if r.job_name == job.name && Some(r.exit_code as i32) == f.exit_code => {
a3c8288b Andy McCall ↶ 473
            // The receipt names exactly the artifacts uploaded, in name order.
474
            let kept: Vec<String> = fdb::artifacts(&app.db, job.id).await?.into_iter().map(|a| a.blob).collect();
475
            if r.outputs.iter().map(hex::encode).collect::<Vec<_>>() != kept {
476
                return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt names other outputs than the artifacts uploaded".into()));
477
            }
42fa9554 Andy McCall ↶ 478
            (serde_json::to_string(r).unwrap_or_default(), hex::encode(r.hash()))
479
        }
480
        Some(_) => return Err(ApiError(StatusCode::BAD_REQUEST, "the receipt does not match the job".into())),
481
        None => (String::new(), String::new()),
482
    };
483
    let error = f.error.clone().unwrap_or_default();
484
    let changed = fdb::finish(
485
        &app.db,
486
        job.id,
487
        fdb::Outcome { status, exit_code: f.exit_code.map(i64::from), error: &error, receipt: &receipt, receipt_hash: &receipt_hash },
488
    )
489
    .await?;
490
    if let Some(run) = fdb::run_by_id(&app.db, job.run_id).await? {
491
        if let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? {
cc19697f Andy McCall ↶ 492
            if let (Some(r), Some(signed)) = (&f.receipt, &f.signed_receipt) {
b6b7e4cf Andy McCall ↶ 493
                let (attest, signer, note) = match keep_receipt(&app, &me, &row, &run, r, signed).await {
cc19697f Andy McCall ↶ 494
                    Ok((attest, signer)) => (attest, signer, String::new()),
495
                    Err(e) => {
496
                        tracing::warn!("flow: job {} receipt not kept: {e}", job.id);
497
                        (String::new(), String::new(), e)
498
                    }
499
                };
500
                fdb::set_receipt(&app.db, job.id, &attest, &signer, &note).await?;
501
            }
42fa9554 Andy McCall ↶ 502
        }
503
    }
070a7b64 Andy McCall ↶ 504
    report_changed(&app, job.run_id, changed).await?;
42fa9554 Andy McCall ↶ 505
    Ok(StatusCode::NO_CONTENT)
070a7b64 Andy McCall ↶ 506
}
507
508
/// Report the checks of jobs whose status changed.
509
async fn report_changed(app: &Shared, run_id: i64, changed: Vec<fdb::Job>) -> sqlx::Result<()> {
510
    let Some(run) = fdb::run_by_id(&app.db, run_id).await? else {
511
        return Ok(());
512
    };
513
    let Some(row) = db::repos::get_by_id(&app.db, run.repo_id).await? else {
514
        return Ok(());
515
    };
516
    for j in changed {
517
        let summary = match j.status.as_str() {
518
            "passed" => format!("passed in {}", duration(&j)),
519
            "failed" => format!("exit {} after {}", j.exit_code.unwrap_or(-1), duration(&j)),
520
            "skipped" => "skipped: a dependency did not pass".into(),
521
            _ => format!("could not run: {}", j.error),
522
        };
523
        report_check(app, &row, &run.state, run.number, &j.name, check_status(&j.status), &summary).await;
524
    }
525
    Ok(())
526
}
527
528
/// Grace past the job time limit before the hub gives a job up: a runner
529
/// stops a job at the limit and reports it within moments, so one still
530
/// running this long after is one the runner no longer reports (it was
531
/// restarted, or lost the hub while the job ended, oxygenhub#71).
532
const LOST_GRACE_MS: i64 = 5 * 60 * 1000;
533
534
/// End every job still running past the job time limit plus the grace:
535
/// it is marked an error with the reason, the jobs that depended on it are
536
/// skipped and its run settled, as when a runner reports an error.
537
pub async fn reap_lost_jobs(app: &Shared) -> sqlx::Result<usize> {
538
    let limit_ms = app.config.flow.job_timeout_minutes.max(1) as i64 * 60_000 + LOST_GRACE_MS;
539
    let lost = fdb::running_since_before(&app.db, db::now_ms() - limit_ms).await?;
540
    let n = lost.len();
541
    for job in lost {
542
        let error = format!(
543
            "lost: still running {} minutes after it started, past the time limit; the runner{} stopped reporting it",
544
            (db::now_ms() - job.started_ms.unwrap_or(0)) / 60_000,
545
            if job.runner.is_empty() { String::new() } else { format!(" {}", job.runner) },
546
        );
547
        let changed =
548
            fdb::finish(&app.db, job.id, fdb::Outcome { status: "error", exit_code: None, error: &error, receipt: "", receipt_hash: "" })
549
                .await?;
550
        tracing::warn!("flow: job {} {error}", job.id);
551
        report_changed(app, job.run_id, changed).await?;
552
    }
553
    Ok(n)
554
}
555
556
/// Look for lost jobs once a minute.
557
pub async fn reaper(app: Shared) {
558
    loop {
559
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
560
        if let Err(e) = reap_lost_jobs(&app).await {
561
            tracing::warn!("flow: looking for lost jobs: {e}");
562
        }
563
    }
cc19697f Andy McCall ↶ 564
}
565
566
/// Check a runner's signed receipt for a job of `run` and keep it in the
567
/// repository (Oxygen spec §14.6): the attestation's hex and the runner's
568
/// name, or why it was not kept. It must be the job's receipt, for the
569
/// run's state and repository, signed by the key the runner's token first
570
/// signed with.
571
pub async fn keep_receipt(
572
    app: &Shared,
b6b7e4cf Andy McCall ↶ 573
    runner: &db::runners::Runner,
cc19697f Andy McCall ↶ 574
    row: &db::repos::RepoRow,
575
    run: &fdb::Run,
576
    receipt: &flow_core::Receipt,
577
    signed_hex: &str,
578
) -> Result<(String, String), String> {
579
    use oxy_core::receipt::SignedReceipt;
580
    let bytes = hex::decode(signed_hex).map_err(|_| "the signed receipt is not hex".to_string())?;
581
    let sr = SignedReceipt::decode(&bytes).map_err(|e| format!("not a signed receipt: {e}"))?;
582
    sr.verify().map_err(|e| format!("it does not verify: {e}"))?;
583
    if sr.receipt.hash().0 != receipt.hash() {
584
        return Err("it signs another receipt than the job's".into());
585
    }
586
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
587
    let state = oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(heads).encode());
588
    if sr.state() != state {
589
        return Err("it is for another state than the run's".into());
590
    }
591
    let key = sr.runner.hash().to_hex();
b6b7e4cf Andy McCall ↶ 592
    let bound = db::runners::bind_key(&app.db, runner.id, &key, &sr.runner.name).await.map_err(|e| e.to_string())?;
cc19697f Andy McCall ↶ 593
    if bound != key {
594
        return Err(format!("signed by {}, not the key this runner signs with ({})", &key[..12], &bound[..12.min(bound.len())]));
595
    }
596
    let name = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?;
597
    let signer = sr.runner.name.clone();
598
    let attest = app.repos.write(&name, move |repo, _| Ok(repo.add_receipt(&sr)?)).await.map_err(|e| e.to_string())?;
599
    Ok((attest.to_hex(), signer))
42fa9554 Andy McCall ↶ 600
}
601
602
/// A job's wall time, as the pages show it.
603
pub fn duration(j: &fdb::Job) -> String {
604
    match (j.started_ms, j.finished_ms) {
605
        (Some(a), Some(b)) => human_ms(b - a),
606
        (Some(a), None) => human_ms(db::now_ms() - a),
607
        _ => "–".into(),
608
    }
609
}
610
611
pub fn human_ms(ms: i64) -> String {
612
    let ms = ms.max(0);
613
    if ms < 1000 {
614
        format!("{ms} ms")
615
    } else if ms < 60_000 {
616
        format!("{:.1} s", ms as f64 / 1000.0)
617
    } else {
618
        format!("{} min {} s", ms / 60_000, (ms % 60_000) / 1000)
619
    }
620
}
bf3d0754 Andy McCall ↶ 621
622
/// Where a live run stands among the runners' work: what its viewers may
623
/// know of it. Jobs of other repositories are named only when those are
624
/// public; the rest are counted.
625
#[derive(Clone, Debug, Default, PartialEq, Eq)]
626
pub struct QueueInfo {
627
    /// 1-based place of the run's first queued job among all queued jobs.
628
    pub position: Option<usize>,
629
    /// Jobs running now, and queued jobs ahead of the run's first one.
630
    pub running: Vec<String>,
631
    pub ahead: Vec<String>,
632
    /// Of those, jobs in repositories the page may not name.
633
    pub hidden: usize,
634
}
635
636
impl QueueInfo {
637
    /// "3rd in line" and the like.
638
    pub fn place(&self) -> Option<String> {
639
        self.position.map(|n| {
640
            let suffix = match (n % 10, n % 100) {
641
                (1, 11) | (2, 12) | (3, 13) => "th",
642
                (1, _) => "st",
643
                (2, _) => "nd",
644
                (3, _) => "rd",
645
                _ => "th",
646
            };
647
            format!("{n}{suffix} in line")
648
        })
649
    }
650
}
651
652
/// The queue as `run` (of repository `repo_id`) sees it, from `queue`.
653
pub fn queue_info(queue: &[fdb::QueueEntry], repo_id: i64, run_id: i64) -> QueueInfo {
654
    let mut info = QueueInfo::default();
655
    let first = queue.iter().filter(|e| e.status == "queued").position(|e| e.run_id == run_id);
656
    info.position = first.map(|i| i + 1);
657
    let label = |e: &fdb::QueueEntry| format!("{}/{} #{} {}", e.owner, e.name, e.run_number, e.job);
658
    let mut queued_seen = 0;
659
    for e in queue {
660
        if e.run_id == run_id {
661
            continue;
662
        }
663
        let ahead = match e.status.as_str() {
664
            "running" => true,
665
            _ => {
666
                queued_seen += 1;
667
                first.is_some_and(|f| queued_seen <= f)
668
            }
669
        };
670
        if !ahead {
671
            continue;
672
        }
673
        if e.repo_id != repo_id && e.visibility != "public" {
674
            info.hidden += 1;
675
        } else if e.status == "running" {
676
            info.running.push(label(e));
677
        } else {
678
            info.ahead.push(label(e));
679
        }
680
    }
681
    info
682
}
683
684
#[cfg(test)]
685
mod queue_tests {
686
    use super::*;
687
688
    fn e(job_id: i64, run_id: i64, repo_id: i64, vis: &str, job: &str, status: &str) -> fdb::QueueEntry {
689
        fdb::QueueEntry {
690
            job_id,
691
            run_id,
692
            repo_id,
693
            owner: "o".into(),
694
            name: format!("r{repo_id}"),
695
            visibility: vis.into(),
696
            run_number: run_id,
697
            job: job.into(),
698
            status: status.into(),
b66e70b7 Andy McCall ↶ 699
            runner: String::new(),
c4c87f48 Andy McCall ↶ 700
            spec: String::new(),
701
            sandbox: String::new(),
bf3d0754 Andy McCall ↶ 702
        }
703
    }
704
705
    #[test]
706
    fn a_run_learns_its_place_and_what_is_ahead_of_it() {
707
        // Running first, then queued in claim order (as `fdb::queue` returns them).
708
        let q = vec![
709
            e(1, 10, 1, "public", "build", "running"),
710
            e(2, 11, 2, "private", "test", "running"),
711
            e(3, 10, 1, "public", "test", "queued"),
712
            e(4, 12, 3, "private", "lint", "queued"),
713
            e(5, 13, 4, "public", "build", "queued"),
714
            e(6, 13, 4, "public", "test", "queued"),
715
            e(7, 14, 5, "public", "late", "queued"),
716
        ];
717
        let info = queue_info(&q, 4, 13);
718
        assert_eq!(info.position, Some(3));
719
        assert_eq!(info.place().as_deref(), Some("3rd in line"));
720
        assert_eq!(info.running, ["o/r1 #10 build"]);
721
        assert_eq!(info.ahead, ["o/r1 #10 test"], "queued ahead, not behind");
722
        assert_eq!(info.hidden, 2, "private repositories are counted, not named");
723
        // A private repository's own jobs are named to its viewers.
724
        assert_eq!(queue_info(&q, 3, 12).ahead, ["o/r1 #10 test"]);
725
        assert_eq!(queue_info(&q, 3, 12).position, Some(2));
726
        // A run with nothing queued has no place.
727
        assert_eq!(queue_info(&q, 1, 99).position, None);
728
        let ordinal = |n| QueueInfo { position: Some(n), ..Default::default() }.place().unwrap();
729
        assert_eq!(
730
            [ordinal(1), ordinal(2), ordinal(11), ordinal(22), ordinal(113)],
731
            ["1st in line", "2nd in line", "11th in line", "22nd in line", "113th in line"]
732
        );
733
    }
734
}