OXYGENHUB
◆ v0.6.0 oxygenhub / crates / hub / src / db / flow.rs
50.5 KiB · Rust · last changed Newest run wins: a proposal's new run supersedes its older ones 16 days ago source blame history raw
1 
//! OxygenFlow runs, jobs and logs (hub spec §13.2).
2
3 
use super::{now_ms, Pool};
4 
use sqlx::FromRow;
5
6 
#[derive(Clone, Debug, FromRow)]
7 
pub struct Run {
8 
    pub id: i64,
9 
    pub repo_id: i64,
10 
    /// The run's number within its repository, from 1 (migration 0019).
11 
    #[sqlx(default)]
12 
    pub number: i64,
13 
    /// `push`, `proposal`, `tag:NAME`, `manual`, `schedule` or
14 
    /// `rerun:N:EVENT` (migration 0020).
15 
    #[sqlx(default)]
16 
    pub trigger: String,
17 
    pub bookmark: String,
18 
    pub state: String,
19 
    pub heads: String,
20 
    pub pipeline_hash: String,
21 
    pub status: String,
22 
    pub error: String,
23 
    pub actor: String,
24 
    pub created_ms: i64,
25 
    pub finished_ms: Option<i64>,
26 
    /// `urgent` when expedited (oxygenhub#244), else empty: the run follows
27 
    /// its repository's, organisation's or the site's level.
28 
    #[sqlx(default)]
29 
    pub priority: String,
30 
    /// Who expedited it and why.
31 
    #[sqlx(default)]
32 
    pub priority_note: String,
33 
}
34
35 
/// The priority levels, most urgent first (oxygenhub#244).
36 
pub const PRIORITIES: [&str; 4] = ["urgent", "high", "normal", "low"];
37
38 
/// SQL for the level a run's jobs get from where they are, as a rank
39 
/// (0 urgent … 3 low): the run's own (expedited), else its repository's,
40 
/// its organisation's, the site's (`settings.flow_priority`), else normal.
41 
macro_rules! level_rank {
42 
    () => {
43 
        "CASE COALESCE(NULLIF(r.priority, ''), NULLIF(rep.flow_priority, ''), \
44 
           NULLIF((SELECT o.flow_priority FROM organisations o WHERE o.name = rep.owner), ''), \
45 
           NULLIF((SELECT s.value FROM settings s WHERE s.key = 'flow_priority'), ''), 'normal') \
46 
         WHEN 'urgent' THEN 0 WHEN 'high' THEN 1 WHEN 'low' THEN 3 ELSE 2 END"
47 
    };
48 
}
49
50 
/// SQL for a queued job's level as a rank (0 urgent … 3 low): the level
51 
/// its run gets from where it is, lowered by the job's own priority from
52 
/// its pipeline (`low`, `normal`), never raised by it; an expedited run is
53 
/// urgent whatever its jobs say (oxygenhub#244). Needs `j` (the job), `r`
54 
/// (its run) and `rep` (its repository) in the query.
55 
pub const PRIORITY_RANK: &str = concat!(
56 
    "CASE WHEN r.priority <> '' THEN ",
57 
    level_rank!(),
58 
    " WHEN j.priority = 'low' THEN 3 WHEN j.priority = 'normal' AND (",
59 
    level_rank!(),
60 
    ") < 2 THEN 2 ELSE ",
61 
    level_rank!(),
62 
    " END"
63 
);
64
65 
/// SQL for why a job stands where it does, beside [`PRIORITY_RANK`]: an
66 
/// expedited run's note, else the pipeline's word for a job it lowered.
67 
pub const PRIORITY_NOTE: &str =
68 
    "CASE WHEN r.priority <> '' THEN r.priority_note WHEN j.priority <> '' THEN 'pipeline: ' || j.priority ELSE r.priority_note END";
69
70 
/// How many runs of `owner`'s repositories are expedited and still live.
71 
pub async fn expedited_live(pool: &Pool, owner: &str) -> sqlx::Result<i64> {
72 
    sqlx::query_scalar(
73 
        "SELECT COUNT(*) FROM flow_runs r JOIN repositories rep ON rep.id = r.repo_id \
74 
         WHERE rep.owner = $1 AND r.priority = 'urgent' AND r.status IN ('queued', 'running')",
75 
    )
76 
    .bind(owner)
77 
    .fetch_one(pool)
78 
    .await
79 
}
80
81 
/// Raise run `run_id` to urgent, saying who and why; a run already ended
82 
/// is left alone. Whether it changed.
83 
pub async fn expedite(pool: &Pool, run_id: i64, note: &str) -> sqlx::Result<bool> {
84 
    let r = sqlx::query("UPDATE flow_runs SET priority = 'urgent', priority_note = $1 WHERE id = $2 AND status IN ('queued', 'running')")
85 
        .bind(note)
86 
        .bind(run_id)
87 
        .execute(pool)
88 
        .await?;
89 
    Ok(r.rows_affected() > 0)
90 
}
91
92 
impl Run {
93 
    /// How the run page says what started it.
94 
    pub fn started_by(&self) -> String {
95 
        match self.trigger.as_str() {
96 
            "manual" => format!("started by {}", self.actor),
97 
            "schedule" => "started on schedule".to_string(),
98 
            t if t.starts_with("tag:") => format!("tag {} pushed by {}", &t[4..], self.actor),
99 
            t if t.starts_with("rerun:") => format!("re-run of #{} by {}", self.rerun_of().unwrap_or(""), self.actor),
100 
            _ => format!("pushed by {}", self.actor),
101 
        }
102 
    }
103
104 
    /// The run a rerun repeats: `N` of `rerun:N` or `rerun:N:EVENT`.
105 
    pub fn rerun_of(&self) -> Option<&str> {
106 
        let rest = self.trigger.strip_prefix("rerun:")?;
107 
        Some(rest.split(':').next().unwrap_or(rest))
108 
    }
109
110 
    /// The trigger as people read it: `rerun:N`, without the event it repeats.
111 
    pub fn trigger_label(&self) -> String {
112 
        match self.rerun_of() {
113 
            Some(n) => format!("rerun:{n}"),
114 
            None => self.trigger.clone(),
115 
        }
116 
    }
117
118 
    pub fn head_list(&self) -> Vec<&str> {
119 
        self.heads.split('+').filter(|h| !h.is_empty()).collect()
120 
    }
121
122 
    pub fn live(&self) -> bool {
123 
        matches!(self.status.as_str(), "queued" | "running")
124 
    }
125 
}
126
127 
#[derive(Clone, Debug, Default, FromRow)]
128 
pub struct Job {
129 
    pub id: i64,
130 
    pub run_id: i64,
131 
    pub name: String,
132 
    pub depth: i64,
133 
    pub depends: String,
134 
    pub spec: String,
135 
    pub sandbox: String,
136 
    pub runtime: String,
137 
    pub status: String,
138 
    pub runner: String,
139 
    pub token_hash: String,
140 
    pub started_ms: Option<i64>,
141 
    pub finished_ms: Option<i64>,
142 
    pub exit_code: Option<i64>,
143 
    pub error: String,
144 
    pub receipt: String,
145 
    pub receipt_hash: String,
146 
    pub log_count: i64,
147 
    /// Comma-separated ids of other repositories the job token may read.
148 
    pub also_read: String,
149 
    /// The receipt attestation kept in the repository (Oxygen spec §14.6).
150 
    pub receipt_attest: String,
151 
    /// Who signed it: the runner's identity name.
152 
    pub receipt_signer: String,
153 
    /// Why a signed receipt was not kept.
154 
    pub receipt_note: String,
155 
    /// Why a queued job no runner can take waits (spec §11.4), from the
156 
    /// last placement pass; empty when it is only waiting its turn.
157 
    pub waiting: String,
158 
    /// What it took, as its runner reported (oxygenflow#41), in MiB.
159 
    pub peak_disk_mib: Option<i64>,
160 
    pub peak_memory_mib: Option<i64>,
161 
    /// The free disk it needs, learned from past runs of the same job.
162 
    pub need_disk_mib: Option<i64>,
163 
    /// Why it ran on its runner (oxygenflow#42).
164 
    pub placed: String,
165 
    /// `waiting` for a person, `approved`, or empty (oxygenflow#57).
166 
    pub approval: String,
167 
    /// Who approved or rejected it, and when.
168 
    pub approved_by: String,
169 
    pub approved_ms: Option<i64>,
170 
    /// The agent token it was decided through, if one (oxygenhub#229).
171 
    #[sqlx(default)]
172 
    pub approved_via: String,
173 
    /// The environment it deploys to, or empty (oxygenflow#59).
174 
    pub environment: String,
175 
    /// The approval attestation that released it, when sealed (#235).
176 
    pub approval_seal: String,
177 
    /// When it entered the queue (made, or run again), and when it became
178 
    /// ready to run: set at its claim (migration 0080, oxygenhub#257).
179 
    #[sqlx(default)]
180 
    pub queued_ms: Option<i64>,
181 
    #[sqlx(default)]
182 
    pub ready_ms: Option<i64>,
183 
}
184
185 
impl Job {
186 
    pub fn depends_on(&self) -> Vec<String> {
187 
        serde_json::from_str(&self.depends).unwrap_or_default()
188 
    }
189
190 
    pub fn done(&self) -> bool {
191 
        matches!(self.status.as_str(), "passed" | "failed" | "skipped" | "error" | "cancelled")
192 
    }
193 
}
194
195 
#[derive(Clone, Debug, FromRow)]
196 
pub struct LogLine {
197 
    pub seq: i64,
198 
    pub t: i64,
199 
    pub stream: String,
200 
    pub text: String,
201 
}
202
203 
/// A job as the trigger creates it.
204 
pub struct NewJob {
205 
    pub name: String,
206 
    pub depth: i64,
207 
    pub depends: Vec<String>,
208 
    pub spec: String,
209 
    pub sandbox: String,
210 
    pub runtime: String,
211 
    /// The free disk it needs, from [`learned_disk_need`].
212 
    pub need_disk_mib: Option<i64>,
213 
}
214
215 
/// Runs of a job whose peaks set its next need.
216 
const NEED_FROM_RUNS: i64 = 5;
217
218 
/// The free disk a new run of job `name` in repository `repo_id` needs
219 
/// (oxygenflow#41): the most its last runs took, plus a quarter; `None`
220 
/// for a job that has not run here yet.
221 
pub async fn learned_disk_need(pool: &Pool, repo_id: i64, name: &str) -> sqlx::Result<Option<i64>> {
222 
    let peak: Option<i64> = sqlx::query_scalar(
223 
        "SELECT MAX(peak_disk_mib) FROM (SELECT j.peak_disk_mib FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id \
224 
         WHERE r.repo_id = $1 AND j.name = $2 AND j.peak_disk_mib IS NOT NULL ORDER BY j.id DESC LIMIT $3) recent",
225 
    )
226 
    .bind(repo_id)
227 
    .bind(name)
228 
    .bind(NEED_FROM_RUNS)
229 
    .fetch_one(pool)
230 
    .await?;
231 
    Ok(peak.map(|p| p + p / 4 + 64))
232 
}
233
234 
/// Record why job `job_id` ran where it did.
235 
pub async fn set_placed(pool: &Pool, job_id: i64, placed: &str) -> sqlx::Result<()> {
236 
    sqlx::query("UPDATE flow_jobs SET placed = $1 WHERE id = $2").bind(placed).bind(job_id).execute(pool).await?;
237 
    Ok(())
238 
}
239
240 
/// Keep what a job took, as its runner reported.
241 
pub async fn set_peaks(pool: &Pool, job_id: i64, disk_mib: Option<u64>, memory_mib: Option<u64>) -> sqlx::Result<()> {
242 
    sqlx::query("UPDATE flow_jobs SET peak_disk_mib = $1, peak_memory_mib = $2 WHERE id = $3")
243 
        .bind(disk_mib.map(|v| v.min(i64::MAX as u64) as i64))
244 
        .bind(memory_mib.map(|v| v.min(i64::MAX as u64) as i64))
245 
        .bind(job_id)
246 
        .execute(pool)
247 
        .await?;
248 
    Ok(())
249 
}
250
251 
pub struct NewRun<'a> {
252 
    pub repo_id: i64,
253 
    pub bookmark: &'a str,
254 
    pub state: &'a str,
255 
    pub heads: &'a str,
256 
    pub pipeline_hash: &'a str,
257 
    pub actor: &'a str,
258 
    /// `push`, `proposal`, `tag:NAME`, `manual`, `schedule` or
259 
    /// `rerun:N:EVENT`.
260 
    pub trigger: &'a str,
261 
    /// Problems when the pipeline is invalid: the run is recorded in error.
262 
    pub error: Option<&'a str>,
263 
}
264
265 
pub async fn create_run(pool: &Pool, r: NewRun<'_>, jobs: &[NewJob]) -> sqlx::Result<i64> {
266 
    let mut tx = super::begin_write(pool).await?;
267 
    let now = now_ms();
268 
    let status = if r.error.is_some() { "error" } else { "queued" };
269 
    let id: i64 = sqlx::query_scalar(
270 
        "INSERT INTO flow_runs (repo_id, number, bookmark, state, heads, pipeline_hash, status, error, actor, trigger, created_ms, finished_ms) \
271 
         VALUES ($1, (SELECT COALESCE(MAX(number), 0) + 1 FROM flow_runs WHERE repo_id = $2), $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id",
272 
    )
273 
    .bind(r.repo_id)
274 
    .bind(r.repo_id)
275 
    .bind(r.bookmark)
276 
    .bind(r.state)
277 
    .bind(r.heads)
278 
    .bind(r.pipeline_hash)
279 
    .bind(status)
280 
    .bind(r.error.unwrap_or(""))
281 
    .bind(r.actor)
282 
    .bind(r.trigger)
283 
    .bind(now)
284 
    .bind(r.error.map(|_| now))
285 
    .fetch_one(&mut *tx)
286 
    .await?;
287 
    for j in jobs {
288 
        sqlx::query(
289 
            "INSERT INTO flow_jobs (run_id, name, depth, depends, spec, sandbox, runtime, status, need_disk_mib, approval, environment, priority, queued_ms) \
290 
             VALUES ($1, $2, $3, $4, $5, $6, $7, 'queued', $8, $9, $10, $11, $12)",
291 
        )
292 
        .bind(id)
293 
        .bind(&j.name)
294 
        .bind(j.depth)
295 
        .bind(serde_json::to_string(&j.depends).unwrap_or_else(|_| "[]".into()))
296 
        .bind(&j.spec)
297 
        .bind(&j.sandbox)
298 
        .bind(&j.runtime)
299 
        .bind(j.need_disk_mib)
300 
        .bind(if needs_approval(&j.spec) { "waiting" } else { "" })
301 
        .bind(environment_of(&j.spec))
302 
        .bind(priority_of(&j.spec))
303 
        .bind(now)
304 
        .execute(&mut *tx)
305 
        .await?;
306 
    }
307 
    tx.commit().await?;
308 
    Ok(id)
309 
}
310
311 
/// The environment a compiled job deploys to, or empty (oxygenflow#59).
312 
fn environment_of(spec: &str) -> String {
313 
    serde_json::from_str::<serde_json::Value>(spec).ok().and_then(|v| v["environment"].as_str().map(String::from)).unwrap_or_default()
314 
}
315
316 
/// A compiled job's own priority (`low`, `normal`), or empty; anything
317 
/// else is not one a pipeline may give (oxygenhub#244).
318 
fn priority_of(spec: &str) -> String {
319 
    serde_json::from_str::<serde_json::Value>(spec)
320 
        .ok()
321 
        .and_then(|v| v["priority"].as_str().map(String::from))
322 
        .filter(|p| p == "low" || p == "normal")
323 
        .unwrap_or_default()
324 
}
325
326 
/// Whether a compiled job asks for approval (oxygenflow#57).
327 
fn needs_approval(spec: &str) -> bool {
328 
    serde_json::from_str::<serde_json::Value>(spec).is_ok_and(|v| !v["approval"].is_null())
329 
}
330
331 
/// The labels a compiled job and its sandbox ask a runner for (`runs-on`).
332 
fn labels_in(spec: &str, sandbox: &str) -> Vec<String> {
333 
    let mut out: Vec<String> = [spec, sandbox]
334 
        .iter()
335 
        .filter_map(|j| serde_json::from_str::<serde_json::Value>(j).ok())
336 
        .flat_map(|v| v["runs_on"].as_array().cloned().unwrap_or_default())
337 
        .filter_map(|l| l.as_str().map(String::from))
338 
        .collect();
339 
    out.sort();
340 
    out.dedup();
341 
    out
342 
}
343
344 
impl Job {
345 
    /// The runner labels this job needs.
346 
    pub fn needs(&self) -> Vec<String> {
347 
        labels_in(&self.spec, &self.sandbox)
348 
    }
349
350 
    /// The architecture the job's sandbox asks for (OxygenFlow spec §11.3).
351 
    pub fn arch(&self) -> Option<String> {
352 
        arch_in(&self.sandbox)
353 
    }
354 
}
355
356 
/// The architecture a sandbox's JSON asks for, if any.
357 
fn arch_in(sandbox: &str) -> Option<String> {
358 
    serde_json::from_str::<serde_json::Value>(sandbox).ok()?["arch"].as_str().map(String::from)
359 
}
360
361 
/// A job waiting for or held by a runner, with where it comes from.
362 
#[derive(Clone, Debug, FromRow)]
363 
pub struct QueueEntry {
364 
    pub job_id: i64,
365 
    pub run_id: i64,
366 
    pub repo_id: i64,
367 
    pub owner: String,
368 
    pub name: String,
369 
    pub visibility: String,
370 
    pub run_number: i64,
371 
    pub job: String,
372 
    pub status: String,
373 
    /// The runner holding it, when running.
374 
    pub runner: String,
375 
    pub spec: String,
376 
    pub sandbox: String,
377 
    /// When it started running.
378 
    pub started_ms: Option<i64>,
379 
    /// Its level (0 urgent … 3 low, oxygenhub#244) and, when expedited, who
380 
    /// did it and why.
381 
    pub priority_rank: i64,
382 
    pub priority_note: String,
383 
}
384
385 
impl QueueEntry {
386 
    pub fn priority(&self) -> &'static str {
387 
        PRIORITIES.get(self.priority_rank as usize).copied().unwrap_or("normal")
388 
    }
389 
}
390
391 
impl QueueEntry {
392 
    /// The runner labels the job needs.
393 
    pub fn needs(&self) -> Vec<String> {
394 
        labels_in(&self.spec, &self.sandbox)
395 
    }
396
397 
    /// The architecture the job's sandbox asks for (OxygenFlow spec §11.3).
398 
    pub fn arch(&self) -> Option<String> {
399 
        arch_in(&self.sandbox)
400 
    }
401 
}
402
403 
/// Every job of a live run that is running or queued: running ones first,
404 
/// then queued ones in the order runners claim them (hub spec §13.2).
405 
pub async fn queue(pool: &Pool) -> sqlx::Result<Vec<QueueEntry>> {
406 
    sqlx::query_as(&format!(
407 
        "SELECT j.id AS job_id, j.run_id, r.repo_id, rep.owner, rep.name, rep.visibility, r.number AS run_number, j.name AS job, j.status, j.runner, j.spec, j.sandbox, j.started_ms, \
408 
           CAST({PRIORITY_RANK} AS BIGINT) AS priority_rank, {PRIORITY_NOTE} AS priority_note \
409 
         FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id JOIN repositories rep ON rep.id = r.repo_id \
410 
         WHERE j.status IN ('queued', 'running') AND j.approval != 'waiting' AND r.status IN ('queued', 'running') \
411 
         ORDER BY CASE j.status WHEN 'running' THEN 0 ELSE 1 END, \
412 
           CASE j.status WHEN 'running' THEN 0 ELSE {PRIORITY_RANK} END, \
413 
           CASE j.status WHEN 'running' THEN 0 ELSE ROW_NUMBER() OVER (PARTITION BY r.repo_id, j.status ORDER BY j.run_id, j.depth, j.id) \
414 
           + (SELECT COUNT(*) FROM flow_jobs x JOIN flow_runs xr ON xr.id = x.run_id WHERE x.status = 'running' AND xr.repo_id = r.repo_id) END, \
415 
           j.run_id, j.depth, j.id"
416 
    ))
417 
    .fetch_all(pool)
418 
    .await
419 
}
420
421 
/// A repository's runs, newest first, numbered below `before`.
422 
pub async fn runs_by_number(pool: &Pool, repo_id: i64, limit: i64, before: Option<i64>) -> sqlx::Result<Vec<Run>> {
423 
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = $1 AND number < $2 ORDER BY number DESC LIMIT $3")
424 
        .bind(repo_id)
425 
        .bind(before.unwrap_or(i64::MAX))
426 
        .bind(limit)
427 
        .fetch_all(pool)
428 
        .await
429 
}
430
431 
/// The latest run on this bookmark of this repository, for what a push
432 
/// changed since (a job's scope).
433 
/// The run a push to `bookmark` is scoped against: the newest that was not
434 
/// cancelled, nor a push run still going (a new push supersedes it), so
435 
/// that what a cancelled run would have checked is checked by the next.
436 
pub async fn latest_on(pool: &Pool, repo_id: i64, bookmark: &str) -> sqlx::Result<Option<Run>> {
437 
    sqlx::query_as(
438 
        "SELECT * FROM flow_runs WHERE repo_id = $1 AND bookmark = $2 AND heads != '' AND status != 'cancelled'
439 
         AND NOT (trigger = 'push' AND status IN ('queued', 'running')) ORDER BY id DESC LIMIT 1",
440 
    )
441 
    .bind(repo_id)
442 
    .bind(bookmark)
443 
    .fetch_optional(pool)
444 
    .await
445 
}
446
447 
pub async fn runs(pool: &Pool, repo_id: i64, limit: i64, before: Option<i64>) -> sqlx::Result<Vec<Run>> {
448 
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = $1 AND id < $2 ORDER BY id DESC LIMIT $3")
449 
        .bind(repo_id)
450 
        .bind(before.unwrap_or(i64::MAX))
451 
        .bind(limit)
452 
        .fetch_all(pool)
453 
        .await
454 
}
455
456 
/// A repository's run by its number there.
457 
pub async fn run(pool: &Pool, repo_id: i64, number: i64) -> sqlx::Result<Option<Run>> {
458 
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = $1 AND number = $2").bind(repo_id).bind(number).fetch_optional(pool).await
459 
}
460
461 
pub async fn run_by_id(pool: &Pool, id: i64) -> sqlx::Result<Option<Run>> {
462 
    sqlx::query_as("SELECT * FROM flow_runs WHERE id = $1").bind(id).fetch_optional(pool).await
463 
}
464
465 
pub async fn jobs(pool: &Pool, run_id: i64) -> sqlx::Result<Vec<Job>> {
466 
    sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = $1 ORDER BY depth, name").bind(run_id).fetch_all(pool).await
467 
}
468
469 
pub async fn job(pool: &Pool, id: i64) -> sqlx::Result<Option<Job>> {
470 
    sqlx::query_as("SELECT * FROM flow_jobs WHERE id = $1").bind(id).fetch_optional(pool).await
471 
}
472
473 
/// The running job a job token belongs to, while it may still be used.
474 
pub async fn job_by_token(pool: &Pool, token_hash: &str, max_age_ms: i64) -> sqlx::Result<Option<Job>> {
475 
    sqlx::query_as("SELECT * FROM flow_jobs WHERE token_hash = $1 AND status = 'running' AND started_ms >= $2")
476 
        .bind(token_hash)
477 
        .bind(now_ms() - max_age_ms)
478 
        .fetch_optional(pool)
479 
        .await
480 
}
481
482 
/// Take the oldest queued job whose dependencies all passed and whose tier
483 
/// the runner offers, and mark it running for `runner` with a token.
484 
/// Let a job's token also read these repositories (its pinned subrepos).
485 
pub async fn set_also_read(pool: &Pool, job_id: i64, repo_ids: &[i64]) -> sqlx::Result<()> {
486 
    let list: Vec<String> = repo_ids.iter().map(|i| i.to_string()).collect();
487 
    sqlx::query("UPDATE flow_jobs SET also_read = $1 WHERE id = $2").bind(list.join(",")).bind(job_id).execute(pool).await?;
488 
    Ok(())
489 
}
490
491 
impl Job {
492 
    pub fn also_read_ids(&self) -> Vec<i64> {
493 
        self.also_read.split(',').filter_map(|x| x.trim().parse().ok()).collect()
494 
    }
495 
}
496
497 
/// What a runner can take on: free memory (MiB) and vCPUs, `None` for no
498 
/// limit, and the rootfs images it holds, `None` when it does not say.
499 
#[derive(Clone, Debug, Default)]
500 
pub struct Room {
501 
    pub memory_mib: Option<u32>,
502 
    pub cpus: Option<u32>,
503 
    pub images: Option<std::collections::HashSet<String>>,
504 
    /// The runner's architecture (`x86_64`, `riscv64`, …), empty when unknown.
505 
    pub arch: String,
506 
    /// Each image's architecture as runners have reported it (`flow_images`).
507 
    pub image_archs: std::collections::HashMap<String, String>,
508 
    /// Jobs this runner declined, not to be offered to it again (spec §11.5).
509 
    pub declined: std::collections::HashSet<i64>,
510 
    /// Images the hub holds that this runner would fetch when it lacks
511 
    /// them (oxygenflow#39); empty for a runner that does not fetch.
512 
    pub fetchable: std::collections::HashSet<String>,
513 
    /// Its free disk now, from its claim's facts (oxygenflow#40); `None`
514 
    /// when it does not say.
515 
    pub free_disk_mib: Option<u64>,
516 
}
517
518 
impl Room {
519 
    /// Whether a job needing `need` MiB of disk fits in its free disk.
520 
    pub fn disk_fits(&self, need: Option<i64>) -> bool {
521 
        match (need, self.free_disk_mib) {
522 
            (Some(n), Some(free)) => n.max(0) as u64 <= free,
523 
            _ => true,
524 
        }
525 
    }
526 
}
527
528 
impl Room {
529 
    /// Whether a job's sandbox (its stored JSON) fits: its memory and vCPUs,
530 
    /// and its rootfs image among the runner's (oxygenhub#73).
531 
    pub fn fits(&self, sandbox_json: &str) -> bool {
532 
        let v: serde_json::Value = serde_json::from_str(sandbox_json).unwrap_or_default();
533 
        let need = |k: &str| v[k].as_u64().unwrap_or(0);
534 
        let rootfs = v["rootfs"].as_str().unwrap_or("");
535 
        // The architecture: the sandbox's own ask, else the image's as read
536 
        // or learned from its passes (oxygenhub#173); one nobody has read
537 
        // or run yet matches (OxygenFlow spec §11.3).
538 
        let wanted = v["arch"].as_str().map(String::from).or_else(|| self.image_archs.get(rootfs).cloned());
539 
        let arch_ok = self.arch.is_empty() || wanted.as_deref().is_none_or(|w| w == self.arch);
540 
        arch_ok
541 
            && self.memory_mib.is_none_or(|m| need("memory_mib") <= m as u64)
542 
            && self.cpus.is_none_or(|c| need("cpus") <= c as u64)
543 
            && (rootfs.is_empty() || self.images.as_ref().is_none_or(|have| have.contains(rootfs)) || self.fetchable.contains(rootfs))
544 
    }
545 
}
546
547 
/// Remember which architecture each image is for, as a runner reports it.
548 
pub async fn record_image_archs(pool: &Pool, archs: &std::collections::BTreeMap<String, String>) -> sqlx::Result<()> {
549 
    for (hash, arch) in archs.iter().take(500) {
550 
        if hash.len() != 64 || !flow_core::arch_ok(arch) {
551 
            continue;
552 
        }
553 
        sqlx::query("INSERT INTO flow_images (hash, arch, seen_ms) VALUES ($1, $2, $3) ON CONFLICT (hash) DO UPDATE SET arch = excluded.arch, seen_ms = excluded.seen_ms")
554 
        .bind(hash.to_ascii_lowercase())
555 
        .bind(arch)
556 
        .bind(now_ms())
557 
        .execute(pool)
558 
        .await?;
559 
    }
560 
    Ok(())
561 
}
562
563 
/// Give every image nobody has read the architecture of the runners its
564 
/// jobs passed on (oxygenhub#173), for the jobs before the hub learned at
565 
/// each pass. Never overrides a known architecture.
566 
pub async fn learn_archs_from_runs(pool: &Pool) -> sqlx::Result<u64> {
567 
    let rootfs = crate::db::json_text("j.sandbox", "rootfs");
568 
    let r = sqlx::query(&format!(
569 
        "INSERT INTO flow_images (hash, arch, seen_ms)
570 
         SELECT lower({rootfs}), MIN(r.arch), $1 FROM flow_jobs j JOIN flow_runners r ON r.id = j.runner_id
571 
         WHERE j.status = 'passed' AND r.arch != '' AND length({rootfs}) = 64
572 
         GROUP BY lower({rootfs})
573 
         ON CONFLICT (hash) DO NOTHING"
574 
    ))
575 
    .bind(now_ms())
576 
    .execute(pool)
577 
    .await?;
578 
    Ok(r.rows_affected())
579 
}
580
581 
/// Every image's architecture the hub knows.
582 
pub async fn image_archs(pool: &Pool) -> sqlx::Result<std::collections::HashMap<String, String>> {
583 
    let rows: Vec<(String, String)> = sqlx::query_as("SELECT hash, arch FROM flow_images").fetch_all(pool).await?;
584 
    Ok(rows.into_iter().collect())
585 
}
586
587 
pub async fn claim(pool: &Pool, runtimes: &[String], runner: &str, token_hash: &str) -> sqlx::Result<Option<Job>> {
588 
    claim_within(pool, runtimes, runner, token_hash, Room::default(), &Default::default(), &[]).await
589 
}
590
591 
/// The first queued job a runner with these tiers, this room and this
592 
/// scope may take, now running under `runner`. Repositories take turns
593 
/// (fair share, oxygenhub#244): a job's turn is its place in its own
594 
/// repository's queue plus that repository's running jobs, lowest first,
595 
/// oldest first among equals, so one repository's burst can't hold up the
596 
/// rest. A scoped runner never sees
597 
/// another scope's jobs (oxygenhub#41).
598 
pub async fn claim_within(
599 
    pool: &Pool,
600 
    runtimes: &[String],
601 
    runner: &str,
602 
    token_hash: &str,
603 
    room: Room,
604 
    scope: &super::runners::Scope,
605 
    labels: &[String],
606 
) -> sqlx::Result<Option<Job>> {
607 
    claim_ranked(pool, runtimes, runner, token_hash, room, scope, labels, &|_| false).await
608 
}
609
610 
/// [`claim_within`], passing over a job `elsewhere` says a better runner
611 
/// should take (oxygenflow#42).
612 
#[allow(clippy::too_many_arguments)]
613 
pub async fn claim_ranked(
614 
    pool: &Pool,
615 
    runtimes: &[String],
616 
    runner: &str,
617 
    token_hash: &str,
618 
    room: Room,
619 
    scope: &super::runners::Scope,
620 
    labels: &[String],
621 
    elsewhere: &(dyn Fn(&Job) -> bool + Sync),
622 
) -> sqlx::Result<Option<Job>> {
623 
    let mut tx = super::begin_write(pool).await?;
624 
    let queued: Vec<Job> = sqlx::query_as(&format!(
625 
        "SELECT j.* FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id JOIN repositories rep ON rep.id = r.repo_id \
626 
         WHERE j.status = 'queued' AND j.approval != 'waiting' AND r.status IN ('queued', 'running') \
627 
         AND ($1 = '' OR rep.owner = $2) AND ($3 IS NULL OR r.repo_id = $4) \
628 
         AND NOT ($5 = '' AND $6 IS NULL AND rep.flow = 'own') \
629 
         ORDER BY {PRIORITY_RANK}, ROW_NUMBER() OVER (PARTITION BY r.repo_id ORDER BY j.run_id, j.depth, j.id) \
630 
           + (SELECT COUNT(*) FROM flow_jobs x JOIN flow_runs xr ON xr.id = x.run_id WHERE x.status = 'running' AND xr.repo_id = r.repo_id), \
631 
           j.run_id, j.depth, j.id LIMIT 200"
632 
    ))
633 
    .bind(&scope.owner)
634 
    .bind(&scope.owner)
635 
    .bind(scope.repo_id)
636 
    .bind(scope.repo_id)
637 
    // A repository on its own runners only (#225): the site's runners skip it.
638 
    .bind(&scope.owner)
639 
    .bind(scope.repo_id)
640 
    .fetch_all(&mut *tx)
641 
    .await?;
642 
    for j in queued {
643 
        if !runtimes.iter().any(|r| r == &j.runtime)
644 
            || !room.fits(&j.sandbox)
645 
            || !room.disk_fits(j.need_disk_mib)
646 
            || room.declined.contains(&j.id)
647 
        {
648 
            continue;
649 
        }
650 
        // Every label the job and its sandbox ask for (`runs-on`).
651 
        if !j.needs().iter().all(|l| labels.contains(l)) {
652 
            continue;
653 
        }
654 
        let deps = j.depends_on();
655 
        // When it became ready: queued, its dependencies passed, approved.
656 
        let mut ready = j.queued_ms.unwrap_or(0).max(j.approved_ms.unwrap_or(0));
657 
        if !deps.is_empty() {
658 
            let mut passed = 0i64;
659 
            for d in &deps {
660 
                let st: Option<(String, Option<i64>)> =
661 
                    sqlx::query_as("SELECT status, finished_ms FROM flow_jobs WHERE run_id = $1 AND name = $2")
662 
                        .bind(j.run_id)
663 
                        .bind(d)
664 
                        .fetch_optional(&mut *tx)
665 
                        .await?;
666 
                if let Some((status, finished)) = st {
667 
                    if status == "passed" {
668 
                        passed += 1;
669 
                        ready = ready.max(finished.unwrap_or(0));
670 
                    }
671 
                }
672 
            }
673 
            if passed != deps.len() as i64 {
674 
                continue;
675 
            }
676 
        }
677 
        if elsewhere(&j) {
678 
            continue;
679 
        }
680 
        let now = now_ms();
681 
        let done = sqlx::query(
682 
            "UPDATE flow_jobs SET status = 'running', runner = $1, token_hash = $2, started_ms = $3, waiting = '', ready_ms = $5 \
683 
             WHERE id = $4 AND status = 'queued'",
684 
        )
685 
        .bind(runner)
686 
        .bind(token_hash)
687 
        .bind(now)
688 
        .bind(j.id)
689 
        .bind(if ready > 0 { ready.min(now) } else { now })
690 
        .execute(&mut *tx)
691 
        .await?;
692 
        if done.rows_affected() == 1 {
693 
            sqlx::query("UPDATE flow_runs SET status = 'running' WHERE id = $1 AND status = 'queued'")
694 
                .bind(j.run_id)
695 
                .execute(&mut *tx)
696 
                .await?;
697 
            tx.commit().await?;
698 
            return job(pool, j.id).await;
699 
        }
700 
    }
701 
    tx.commit().await?;
702 
    Ok(None)
703 
}
704
705 
/// How long a decline keeps a job from its runner when the runner's
706 
/// profile stays the same: a reason such as memory in use passes.
707 
pub const DECLINE_HOLDS_MS: i64 = 10 * 60_000;
708
709 
/// The jobs runner `runner_id` declined with this profile, recently.
710 
pub async fn declined_by(pool: &Pool, runner_id: i64, profile: &str) -> sqlx::Result<std::collections::HashSet<i64>> {
711 
    let rows: Vec<(i64,)> = sqlx::query_as("SELECT job_id FROM flow_declines WHERE runner_id = $1 AND profile = $2 AND ms > $3")
712 
        .bind(runner_id)
713 
        .bind(profile)
714 
        .bind(now_ms() - DECLINE_HOLDS_MS)
715 
        .fetch_all(pool)
716 
        .await?;
717 
    Ok(rows.into_iter().map(|(j,)| j).collect())
718 
}
719
720 
/// A decline: one runner's reason for handing a job back.
721 
#[derive(Clone, Debug, sqlx::FromRow)]
722 
pub struct DeclineRow {
723 
    pub job_id: i64,
724 
    pub runner_id: i64,
725 
    pub runner: String,
726 
    pub reason: String,
727 
    pub ms: i64,
728 
}
729
730 
/// Runner `runner_id` hands back job `job_id`, running on it: the decline
731 
/// is kept (one per job and runner, the latest), and the job is queued
732 
/// again. Returns every decline the job has had, or `None` when the job
733 
/// was not running on that runner.
734 
pub async fn decline(
735 
    pool: &Pool,
736 
    job_id: i64,
737 
    runner_id: i64,
738 
    runner: &str,
739 
    profile: &str,
740 
    reason: &str,
741 
) -> sqlx::Result<Option<Vec<DeclineRow>>> {
742 
    let mut tx = super::begin_write(pool).await?;
743 
    let back = sqlx::query(
744 
        "UPDATE flow_jobs SET status = 'queued', runner = '', token_hash = '', started_ms = NULL, runner_id = NULL \
745 
         WHERE id = $1 AND status = 'running' AND runner_id = $2",
746 
    )
747 
    .bind(job_id)
748 
    .bind(runner_id)
749 
    .execute(&mut *tx)
750 
    .await?;
751 
    if back.rows_affected() != 1 {
752 
        tx.rollback().await?;
753 
        return Ok(None);
754 
    }
755 
    sqlx::query(
756 
        "INSERT INTO flow_declines (job_id, runner_id, runner, profile, reason, ms) VALUES ($1, $2, $3, $4, $5, $6) \
757 
         ON CONFLICT (job_id, runner_id) DO UPDATE SET runner = excluded.runner, profile = excluded.profile, reason = excluded.reason, ms = excluded.ms",
758 
    )
759 
    .bind(job_id)
760 
    .bind(runner_id)
761 
    .bind(runner)
762 
    .bind(profile)
763 
    .bind(reason)
764 
    .bind(now_ms())
765 
    .execute(&mut *tx)
766 
    .await?;
767 
    let all: Vec<DeclineRow> =
768 
        sqlx::query_as("SELECT job_id, runner_id, runner, reason, ms FROM flow_declines WHERE job_id = $1 ORDER BY ms")
769 
            .bind(job_id)
770 
            .fetch_all(&mut *tx)
771 
            .await?;
772 
    tx.commit().await?;
773 
    Ok(Some(all))
774 
}
775
776 
/// The declines of job `job_id`, oldest first.
777 
pub async fn declines_of_job(pool: &Pool, job_id: i64) -> sqlx::Result<Vec<DeclineRow>> {
778 
    sqlx::query_as("SELECT job_id, runner_id, runner, reason, ms FROM flow_declines WHERE job_id = $1 ORDER BY ms")
779 
        .bind(job_id)
780 
        .fetch_all(pool)
781 
        .await
782 
}
783
784 
/// A runner's latest declines, newest first, with where the job is.
785 
#[derive(Clone, Debug, sqlx::FromRow)]
786 
pub struct RunnerDecline {
787 
    pub job_id: i64,
788 
    pub job: String,
789 
    pub run_number: i64,
790 
    pub owner: String,
791 
    pub name: String,
792 
    pub reason: String,
793 
    pub ms: i64,
794 
}
795
796 
pub async fn declines_of_runner(pool: &Pool, runner_id: i64, limit: i64) -> sqlx::Result<Vec<RunnerDecline>> {
797 
    sqlx::query_as(
798 
        "SELECT d.job_id, j.name AS job, r.number AS run_number, rep.owner, rep.name, d.reason, d.ms \
799 
         FROM flow_declines d JOIN flow_jobs j ON j.id = d.job_id JOIN flow_runs r ON r.id = j.run_id \
800 
         JOIN repositories rep ON rep.id = r.repo_id WHERE d.runner_id = $1 ORDER BY d.ms DESC LIMIT $2",
801 
    )
802 
    .bind(runner_id)
803 
    .bind(limit)
804 
    .fetch_all(pool)
805 
    .await
806 
}
807
808 
/// Queued jobs whose dependencies have passed, in their live runs, with
809 
/// their repositories (owner, id): the ones a runner could take now.
810 
/// Every queued job's priority rank, 0 (urgent) to 3 (low), by job id
811 
/// (#259: how many ready jobs come first).
812 
pub async fn queued_ranks(pool: &Pool) -> sqlx::Result<std::collections::HashMap<i64, i64>> {
813 
    let rows: Vec<(i64, i64)> = sqlx::query_as(&format!(
814 
        "SELECT j.id, CAST({PRIORITY_RANK} AS BIGINT) FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id JOIN repositories rep ON rep.id = r.repo_id \
815 
         WHERE j.status = 'queued'"
816 
    ))
817 
    .fetch_all(pool)
818 
    .await?;
819 
    Ok(rows.into_iter().collect())
820 
}
821
822 
/// How many jobs each runner is running now, by runner id (#259).
823 
pub async fn running_by_runner(pool: &Pool) -> sqlx::Result<std::collections::HashMap<i64, i64>> {
824 
    let rows: Vec<(i64, i64)> =
825 
        sqlx::query_as("SELECT runner_id, COUNT(*) FROM flow_jobs WHERE status = 'running' AND runner_id IS NOT NULL GROUP BY runner_id")
826 
            .fetch_all(pool)
827 
            .await?;
828 
    Ok(rows.into_iter().collect())
829 
}
830
831 
pub async fn ready_queued(pool: &Pool) -> sqlx::Result<Vec<(Job, String, i64)>> {
832 
    let queued: Vec<Job> = sqlx::query_as(
833 
        "SELECT j.* FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id \
834 
         WHERE j.status = 'queued' AND j.approval != 'waiting' AND r.status IN ('queued', 'running') ORDER BY j.run_id, j.depth, j.id LIMIT 500",
835 
    )
836 
    .fetch_all(pool)
837 
    .await?;
838 
    let mut out = Vec::new();
839 
    for j in queued {
840 
        let mut ready = true;
841 
        for d in j.depends_on() {
842 
            let st: Option<String> = sqlx::query_scalar("SELECT status FROM flow_jobs WHERE run_id = $1 AND name = $2")
843 
                .bind(j.run_id)
844 
                .bind(&d)
845 
                .fetch_optional(pool)
846 
                .await?;
847 
            if st.as_deref() != Some("passed") {
848 
                ready = false;
849 
                break;
850 
            }
851 
        }
852 
        if !ready {
853 
            continue;
854 
        }
855 
        let repo: Option<(String, i64)> =
856 
            sqlx::query_as("SELECT rep.owner, rep.id FROM flow_runs r JOIN repositories rep ON rep.id = r.repo_id WHERE r.id = $1")
857 
                .bind(j.run_id)
858 
                .fetch_optional(pool)
859 
                .await?;
860 
        if let Some((owner, id)) = repo {
861 
            out.push((j, owner, id));
862 
        }
863 
    }
864 
    Ok(out)
865 
}
866
867 
pub async fn set_waiting(pool: &Pool, job_id: i64, waiting: &str) -> sqlx::Result<()> {
868 
    sqlx::query("UPDATE flow_jobs SET waiting = $1 WHERE id = $2 AND waiting <> $3")
869 
        .bind(waiting)
870 
        .bind(job_id)
871 
        .bind(waiting)
872 
        .execute(pool)
873 
        .await?;
874 
    Ok(())
875 
}
876
877 
/// Append records; returns how many were kept (the rest exceed the cap).
878 
pub async fn append_log(pool: &Pool, job_id: i64, records: &[flow_core::api::Record], cap: i64) -> sqlx::Result<usize> {
879 
    let mut tx = super::begin_write(pool).await?;
880 
    let count: i64 = sqlx::query_scalar("SELECT log_count FROM flow_jobs WHERE id = $1").bind(job_id).fetch_one(&mut *tx).await?;
881 
    let room = (cap - count).max(0) as usize;
882 
    let keep = &records[..records.len().min(room)];
883 
    for (i, r) in keep.iter().enumerate() {
884 
        sqlx::query("INSERT INTO flow_logs (job_id, seq, t, stream, text) VALUES ($1, $2, $3, $4, $5)")
885 
            .bind(job_id)
886 
            .bind(count + i as i64 + 1)
887 
            .bind(r.t as i64)
888 
            .bind(&r.stream)
889 
            .bind(&r.text)
890 
            .execute(&mut *tx)
891 
            .await?;
892 
    }
893 
    sqlx::query("UPDATE flow_jobs SET log_count = log_count + $1 WHERE id = $2")
894 
        .bind(keep.len() as i64)
895 
        .bind(job_id)
896 
        .execute(&mut *tx)
897 
        .await?;
898 
    tx.commit().await?;
899 
    Ok(keep.len())
900 
}
901
902 
pub async fn log(pool: &Pool, job_id: i64, after: i64, limit: i64) -> sqlx::Result<Vec<LogLine>> {
903 
    sqlx::query_as("SELECT seq, t, stream, text FROM flow_logs WHERE job_id = $1 AND seq > $2 ORDER BY seq LIMIT $3")
904 
        .bind(job_id)
905 
        .bind(after)
906 
        .bind(limit)
907 
        .fetch_all(pool)
908 
        .await
909 
}
910
911 
/// Jobs running since before `ms` (the oldest first).
912 
/// Jobs still running at `now` past their own time limit (`default_s` for
913 
/// one claimed without) plus `grace_ms`.
914 
pub async fn running_past_limit(pool: &Pool, now: i64, default_s: u64, grace_ms: i64) -> sqlx::Result<Vec<Job>> {
915 
    sqlx::query_as(
916 
        "SELECT * FROM flow_jobs WHERE status = 'running' \
917 
         AND COALESCE(started_ms, 0) + (CASE WHEN timeout_s > 0 THEN timeout_s ELSE $1 END) * 1000 + $2 < $3 ORDER BY started_ms",
918 
    )
919 
    .bind(default_s as i64)
920 
    .bind(grace_ms)
921 
    .bind(now)
922 
    .fetch_all(pool)
923 
    .await
924 
}
925
926 
/// Record the limit a claimed job runs under.
927 
pub async fn set_job_timeout(pool: &Pool, job_id: i64, timeout_s: u64) -> sqlx::Result<()> {
928 
    sqlx::query("UPDATE flow_jobs SET timeout_s = $1 WHERE id = $2").bind(timeout_s as i64).bind(job_id).execute(pool).await?;
929 
    Ok(())
930 
}
931
932 
pub async fn running_since_before(pool: &Pool, ms: i64) -> sqlx::Result<Vec<Job>> {
933 
    sqlx::query_as("SELECT * FROM flow_jobs WHERE status = 'running' AND COALESCE(started_ms, 0) < $1 ORDER BY started_ms")
934 
        .bind(ms)
935 
        .fetch_all(pool)
936 
        .await
937 
}
938
939 
/// The jobs running on the runner with this id.
940 
pub async fn running_on(pool: &Pool, runner_id: i64) -> sqlx::Result<Vec<Job>> {
941 
    sqlx::query_as("SELECT * FROM flow_jobs WHERE status = 'running' AND runner_id = $1 ORDER BY id").bind(runner_id).fetch_all(pool).await
942 
}
943
944 
pub struct Outcome<'a> {
945 
    pub status: &'a str,
946 
    pub exit_code: Option<i64>,
947 
    pub error: &'a str,
948 
    pub receipt: &'a str,
949 
    pub receipt_hash: &'a str,
950 
}
951
952 
/// Record a job's end, skip the jobs that can no longer run, and settle the
953 
/// run. Returns the jobs whose status changed (for their checks).
954 
pub async fn finish(pool: &Pool, job_id: i64, o: Outcome<'_>) -> sqlx::Result<Vec<Job>> {
955 
    let mut tx = super::begin_write(pool).await?;
956 
    let now = now_ms();
957 
    let j: Job = sqlx::query_as("SELECT * FROM flow_jobs WHERE id = $1").bind(job_id).fetch_one(&mut *tx).await?;
958 
    sqlx::query("UPDATE flow_jobs SET status = $1, exit_code = $2, error = $3, receipt = $4, receipt_hash = $5, finished_ms = $6, token_hash = '' WHERE id = $7")
959 
        .bind(o.status)
960 
        .bind(o.exit_code)
961 
        .bind(o.error)
962 
        .bind(o.receipt)
963 
        .bind(o.receipt_hash)
964 
        .bind(now)
965 
        .bind(job_id)
966 
        .execute(&mut *tx)
967 
        .await?;
968 
    // A pass teaches what its image is for, when nobody has read it
969 
    // (oxygenhub#173): the architecture of the runner it passed on.
970 
    if o.status == "passed" {
971 
        let rootfs = serde_json::from_str::<serde_json::Value>(&j.sandbox)
972 
            .ok()
973 
            .and_then(|v| v["rootfs"].as_str().map(str::to_ascii_lowercase))
974 
            .unwrap_or_default();
975 
        if rootfs.len() == 64 {
976 
            sqlx::query(
977 
                "INSERT INTO flow_images (hash, arch, seen_ms) SELECT $1, r.arch, $2 FROM flow_jobs j JOIN flow_runners r ON r.id = j.runner_id \
978 
                 WHERE j.id = $3 AND r.arch != '' ON CONFLICT (hash) DO NOTHING",
979 
            )
980 
            .bind(&rootfs)
981 
            .bind(now)
982 
            .bind(job_id)
983 
            .execute(&mut *tx)
984 
            .await?;
985 
        }
986 
    }
987 
    let mut changed = vec![job_id];
988 
    // Dependents of anything that did not pass are skipped, transitively.
989 
    loop {
990 
        let all: Vec<Job> = sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = $1").bind(j.run_id).fetch_all(&mut *tx).await?;
991 
        let blocked: Vec<i64> = all
992 
            .iter()
993 
            .filter(|x| x.status == "queued")
994 
            .filter(|x| {
995 
                x.depends_on()
996 
                    .iter()
997 
                    .any(|d| all.iter().any(|y| &y.name == d && matches!(y.status.as_str(), "failed" | "skipped" | "error" | "cancelled")))
998 
            })
999 
            .map(|x| x.id)
1000 
            .collect();
1001 
        if blocked.is_empty() {
1002 
            break;
1003 
        }
1004 
        for id in blocked {
1005 
            sqlx::query("UPDATE flow_jobs SET status = 'skipped', finished_ms = $1 WHERE id = $2")
1006 
                .bind(now)
1007 
                .bind(id)
1008 
                .execute(&mut *tx)
1009 
                .await?;
1010 
            changed.push(id);
1011 
        }
1012 
    }
1013 
    let all: Vec<Job> = sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = $1").bind(j.run_id).fetch_all(&mut *tx).await?;
1014 
    if all.iter().all(Job::done) {
1015 
        let status = if all.iter().all(|x| x.status == "passed") { "passed" } else { "failed" };
1016 
        sqlx::query("UPDATE flow_runs SET status = $1, finished_ms = $2 WHERE id = $3")
1017 
            .bind(status)
1018 
            .bind(now)
1019 
            .bind(j.run_id)
1020 
            .execute(&mut *tx)
1021 
            .await?;
1022 
    }
1023 
    tx.commit().await?;
1024 
    let mut out = Vec::new();
1025 
    for id in changed {
1026 
        if let Some(x) = job(pool, id).await? {
1027 
            out.push(x);
1028 
        }
1029 
    }
1030 
    Ok(out)
1031 
}
1032
1033 
/// Cancel a run (oxygenflow#32): its jobs not yet done end as cancelled,
1034 
/// and so does the run. A runner learns of it when the hub next refuses
1035 
/// its log lines, and stops the job. Returns the jobs whose status changed.
1036 
pub async fn cancel_run(pool: &Pool, run_id: i64, why: &str) -> sqlx::Result<Vec<Job>> {
1037 
    let now = now_ms();
1038 
    let mut tx = pool.begin().await?;
1039 
    let ids: Vec<(i64,)> = sqlx::query_as("SELECT id FROM flow_jobs WHERE run_id = $1 AND status IN ('queued', 'running')")
1040 
        .bind(run_id)
1041 
        .fetch_all(&mut *tx)
1042 
        .await?;
1043 
    sqlx::query(
1044 
        "UPDATE flow_jobs SET status = 'cancelled', error = $1, finished_ms = $2, token_hash = ''
1045 
         WHERE run_id = $3 AND status IN ('queued', 'running')",
1046 
    )
1047 
    .bind(why)
1048 
    .bind(now)
1049 
    .bind(run_id)
1050 
    .execute(&mut *tx)
1051 
    .await?;
1052 
    sqlx::query("UPDATE flow_runs SET status = 'cancelled', finished_ms = $1 WHERE id = $2 AND status IN ('queued', 'running')")
1053 
        .bind(now)
1054 
        .bind(run_id)
1055 
        .execute(&mut *tx)
1056 
        .await?;
1057 
    tx.commit().await?;
1058 
    let mut out = Vec::new();
1059 
    for (id,) in ids {
1060 
        if let Some(x) = job(pool, id).await? {
1061 
            out.push(x);
1062 
        }
1063 
    }
1064 
    Ok(out)
1065 
}
1066
1067 
/// Run job `name` of finished run `run_id` again, with every job that
1068 
/// depends on it (oxygenflow#31): back to the queue as if new, their logs,
1069 
/// artifacts and reports gone, and the run running again. The jobs they
1070 
/// depend on keep their results. The jobs reset; none when there is no
1071 
/// such job.
1072 
pub async fn rerun_job(pool: &Pool, run_id: i64, name: &str) -> sqlx::Result<Vec<Job>> {
1073 
    let all: Vec<Job> = sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = $1").bind(run_id).fetch_all(pool).await?;
1074 
    if !all.iter().any(|j| j.name == name) {
1075 
        return Ok(Vec::new());
1076 
    }
1077 
    let mut again = vec![name.to_string()];
1078 
    loop {
1079 
        let more: Vec<String> = all
1080 
            .iter()
1081 
            .filter(|j| !again.contains(&j.name) && j.depends_on().iter().any(|d| again.contains(d)))
1082 
            .map(|j| j.name.clone())
1083 
            .collect();
1084 
        if more.is_empty() {
1085 
            break;
1086 
        }
1087 
        again.extend(more);
1088 
    }
1089 
    let ids: Vec<i64> = all.iter().filter(|j| again.contains(&j.name)).map(|j| j.id).collect();
1090 
    let mut tx = pool.begin().await?;
1091 
    for id in &ids {
1092 
        for table in ["flow_logs", "flow_artifacts", "flow_declines", "test_reports", "coverage_reports", "reports"] {
1093 
            sqlx::query(&format!("DELETE FROM {table} WHERE job_id = $1")).bind(id).execute(&mut *tx).await?;
1094 
        }
1095 
        sqlx::query(
1096 
            "UPDATE flow_jobs SET status = 'queued', runner = '', token_hash = '', started_ms = NULL, finished_ms = NULL,
1097 
             exit_code = NULL, error = '', receipt = '', receipt_hash = '', log_count = 0, also_read = '', receipt_attest = '',
1098 
             receipt_signer = '', receipt_note = '', runner_id = NULL, waiting = '', placed = '', peak_disk_mib = NULL,
1099 
             peak_memory_mib = NULL, approval = CASE WHEN approval = '' THEN '' ELSE 'waiting' END, approved_by = '',
1100 
             approved_ms = NULL, approval_seal = '', approved_via = '', queued_ms = $2, ready_ms = NULL WHERE id = $1",
1101 
        )
1102 
        .bind(id)
1103 
        .bind(now_ms())
1104 
        .execute(&mut *tx)
1105 
        .await?;
1106 
    }
1107 
    sqlx::query("UPDATE flow_runs SET status = 'running', finished_ms = NULL WHERE id = $1").bind(run_id).execute(&mut *tx).await?;
1108 
    tx.commit().await?;
1109 
    let mut out = Vec::new();
1110 
    for id in ids {
1111 
        if let Some(x) = job(pool, id).await? {
1112 
            out.push(x);
1113 
        }
1114 
    }
1115 
    Ok(out)
1116 
}
1117
1118 
/// The runs of `bookmark` started by `trigger` still going, older than run
1119 
/// `number`: those a newer run supersedes (#267). With `running_too`
1120 
/// false, only those still queued.
1121 
pub async fn live_runs_before(
1122 
    pool: &Pool,
1123 
    repo_id: i64,
1124 
    bookmark: &str,
1125 
    trigger: &str,
1126 
    number: i64,
1127 
    running_too: bool,
1128 
) -> sqlx::Result<Vec<Run>> {
1129 
    sqlx::query_as(
1130 
        "SELECT * FROM flow_runs WHERE repo_id = $1 AND bookmark = $2 AND number < $3 AND trigger = $4
1131 
         AND (status = 'queued' OR ($5 AND status = 'running')) ORDER BY id",
1132 
    )
1133 
    .bind(repo_id)
1134 
    .bind(bookmark)
1135 
    .bind(number)
1136 
    .bind(trigger)
1137 
    .bind(running_too)
1138 
    .fetch_all(pool)
1139 
    .await
1140 
}
1141
1142 
/// The runs on `bookmark` that no runner has started yet: queued, with
1143 
/// the given trigger, created before `before_ms` (#263).
1144 
pub async fn unstarted_runs_on(pool: &Pool, repo_id: i64, bookmark: &str, trigger: &str, before_ms: i64) -> sqlx::Result<Vec<Run>> {
1145 
    sqlx::query_as(
1146 
        "SELECT * FROM flow_runs WHERE repo_id = $1 AND bookmark = $2 AND trigger = $3 AND status = 'queued' AND created_ms <= $4 ORDER BY id",
1147 
    )
1148 
    .bind(repo_id)
1149 
    .bind(bookmark)
1150 
    .bind(trigger)
1151 
    .bind(before_ms)
1152 
    .fetch_all(pool)
1153 
    .await
1154 
}
1155
1156 
/// The newest run of exactly this state (heads joined with `+`, sorted).
1157 
pub async fn latest_for_heads(pool: &Pool, repo_id: i64, heads: &str) -> sqlx::Result<Option<Run>> {
1158 
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = $1 AND heads = $2 ORDER BY id DESC LIMIT 1")
1159 
        .bind(repo_id)
1160 
        .bind(heads)
1161 
        .fetch_optional(pool)
1162 
        .await
1163 
}
1164
1165 
/// Record what became of job `job_id`'s signed receipt.
1166 
pub async fn set_receipt(pool: &Pool, job_id: i64, attest: &str, signer: &str, note: &str) -> sqlx::Result<()> {
1167 
    sqlx::query("UPDATE flow_jobs SET receipt_attest = $1, receipt_signer = $2, receipt_note = $3 WHERE id = $4")
1168 
        .bind(attest)
1169 
        .bind(signer)
1170 
        .bind(note)
1171 
        .bind(job_id)
1172 
        .execute(pool)
1173 
        .await?;
1174 
    Ok(())
1175 
}
1176
1177 
/// A job's artifact, kept as a blob of the run's repository.
1178 
#[derive(Clone, Debug, FromRow)]
1179 
pub struct Artifact {
1180 
    pub job_id: i64,
1181 
    pub name: String,
1182 
    pub path: String,
1183 
    pub blob: String,
1184 
    pub size: i64,
1185 
    /// BLAKE3 of the bytes.
1186 
    pub content: String,
1187 
    pub created_ms: i64,
1188 
}
1189
1190 
pub async fn add_artifact(pool: &Pool, a: &Artifact) -> sqlx::Result<()> {
1191 
    sqlx::query(
1192 
        "INSERT INTO flow_artifacts (job_id, name, path, blob, size, content, created_ms) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (job_id, name) DO UPDATE SET path = excluded.path, blob = excluded.blob, size = excluded.size, content = excluded.content, created_ms = excluded.created_ms",
1193 
    )
1194 
    .bind(a.job_id)
1195 
    .bind(&a.name)
1196 
    .bind(&a.path)
1197 
    .bind(&a.blob)
1198 
    .bind(a.size)
1199 
    .bind(&a.content)
1200 
    .bind(a.created_ms)
1201 
    .execute(pool)
1202 
    .await?;
1203 
    Ok(())
1204 
}
1205
1206 
/// Every artifact blob a repository's runs keep, for garbage collection to
1207 
/// keep too (#126): the database is the only thing that names them.
1208 
pub async fn artifact_blobs(pool: &Pool, repo_id: i64) -> sqlx::Result<Vec<String>> {
1209 
    sqlx::query_scalar(
1210 
        "SELECT DISTINCT a.blob FROM flow_artifacts a JOIN flow_jobs j ON j.id = a.job_id JOIN flow_runs r ON r.id = j.run_id WHERE r.repo_id = $1",
1211 
    )
1212 
    .bind(repo_id)
1213 
    .fetch_all(pool)
1214 
    .await
1215 
}
1216
1217 
/// Job `job_id`'s artifacts, by name.
1218 
pub async fn artifacts(pool: &Pool, job_id: i64) -> sqlx::Result<Vec<Artifact>> {
1219 
    sqlx::query_as("SELECT * FROM flow_artifacts WHERE job_id = $1 ORDER BY name").bind(job_id).fetch_all(pool).await
1220 
}
1221
1222 
/// An artifact with the run and job that made it, for a repository's list
1223 
/// of every run's outputs (oxygenhub#254).
1224 
#[derive(Clone, Debug, FromRow)]
1225 
pub struct RepoArtifact {
1226 
    pub run_number: i64,
1227 
    pub bookmark: String,
1228 
    pub job: String,
1229 
    pub name: String,
1230 
    pub path: String,
1231 
    pub size: i64,
1232 
    pub content: String,
1233 
    pub created_ms: i64,
1234 
}
1235
1236 
/// A repository's newest artifacts, at most `limit`.
1237 
pub async fn repo_artifacts(pool: &Pool, repo_id: i64, limit: i64) -> sqlx::Result<Vec<RepoArtifact>> {
1238 
    sqlx::query_as(
1239 
        "SELECT r.number AS run_number, r.bookmark, j.name AS job, a.name, a.path, a.size, a.content, a.created_ms
1240 
         FROM flow_artifacts a JOIN flow_jobs j ON j.id = a.job_id JOIN flow_runs r ON r.id = j.run_id
1241 
         WHERE r.repo_id = $1 ORDER BY a.created_ms DESC, r.number DESC, j.name, a.name LIMIT $2",
1242 
    )
1243 
    .bind(repo_id)
1244 
    .bind(limit)
1245 
    .fetch_all(pool)
1246 
    .await
1247 
}
1248
1249 
/// Where an artifact with given content came from: the run and job that
1250 
/// built it, and the job's signed receipt when one was kept.
1251 
#[derive(Clone, Debug, FromRow)]
1252 
pub struct Provenance {
1253 
    pub run_number: i64,
1254 
    pub job: String,
1255 
    pub name: String,
1256 
    pub receipt_attest: String,
1257 
    pub receipt_signer: String,
1258 
}
1259
1260 
/// The build of the bytes whose BLAKE3 is `content` in repository
1261 
/// `repo_id`: a passed job, one with a kept signed receipt first, newest
1262 
/// run first.
1263 
pub async fn provenance(pool: &Pool, repo_id: i64, content: &str) -> sqlx::Result<Option<Provenance>> {
1264 
    sqlx::query_as(
1265 
        "SELECT r.number AS run_number, j.name AS job, a.name AS name, j.receipt_attest, j.receipt_signer \
1266 
         FROM flow_artifacts a JOIN flow_jobs j ON j.id = a.job_id JOIN flow_runs r ON r.id = j.run_id \
1267 
         WHERE r.repo_id = $1 AND a.content = $2 AND j.status = 'passed' \
1268 
         ORDER BY (j.receipt_attest != '') DESC, r.number DESC LIMIT 1",
1269 
    )
1270 
    .bind(repo_id)
1271 
    .bind(content)
1272 
    .fetch_optional(pool)
1273 
    .await
1274 
}
1275
1276 
/// Approve job `id` if it is still waiting for a person (oxygenflow#57);
1277 
/// whether it was.
1278 
pub async fn approve_job(pool: &Pool, id: i64, who: &str, seal: &str, via: &str) -> sqlx::Result<bool> {
1279 
    let r = sqlx::query(
1280 
        "UPDATE flow_jobs SET approval = 'approved', approved_by = $1, approved_ms = $2, approval_seal = $3, approved_via = $5 \
1281 
         WHERE id = $4 AND status = 'queued' AND approval = 'waiting'",
1282 
    )
1283 
    .bind(who)
1284 
    .bind(now_ms())
1285 
    .bind(seal)
1286 
    .bind(id)
1287 
    .bind(via)
1288 
    .execute(pool)
1289 
    .await?;
1290 
    Ok(r.rows_affected() > 0)
1291 
}
1292
1293 
/// Note who rejected job `id` (it is then finished as cancelled).
1294 
pub async fn note_rejection(pool: &Pool, id: i64, who: &str, via: &str) -> sqlx::Result<()> {
1295 
    sqlx::query("UPDATE flow_jobs SET approved_by = $1, approved_ms = $2, approved_via = $4 WHERE id = $3")
1296 
        .bind(who)
1297 
        .bind(now_ms())
1298 
        .bind(id)
1299 
        .bind(via)
1300 
        .execute(pool)
1301 
        .await?;
1302 
    Ok(())
1303 
}
1304
1305 
/// Jobs waiting for a person, in runs still going.
1306 
pub async fn waiting_for_approval(pool: &Pool) -> sqlx::Result<Vec<Job>> {
1307 
    sqlx::query_as(
1308 
        "SELECT j.* FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id \
1309 
         WHERE j.status = 'queued' AND j.approval = 'waiting' AND r.status IN ('queued', 'running')",
1310 
    )
1311 
    .fetch_all(pool)
1312 
    .await
1313 
}
1314
1315 
/// One deployment: a job for an environment that passed.
1316 
#[derive(Clone, Debug, FromRow)]
1317 
pub struct Deployment {
1318 
    pub environment: String,
1319 
    pub job: String,
1320 
    pub run_number: i64,
1321 
    pub state: String,
1322 
    pub bookmark: String,
1323 
    pub actor: String,
1324 
    pub approved_by: String,
1325 
    pub finished_ms: i64,
1326 
}
1327
1328 
/// A repository's deployments, newest first (oxygenflow#59).
1329 
pub async fn deployments(pool: &Pool, repo_id: i64, limit: i64) -> sqlx::Result<Vec<Deployment>> {
1330 
    sqlx::query_as(
1331 
        "SELECT j.environment, j.name AS job, r.number AS run_number, r.state, r.bookmark, r.actor, j.approved_by, \
1332 
         COALESCE(j.finished_ms, 0) AS finished_ms FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id \
1333 
         WHERE r.repo_id = $1 AND j.environment != '' AND j.status = 'passed' ORDER BY j.finished_ms DESC LIMIT $2",
1334 
    )
1335 
    .bind(repo_id)
1336 
    .bind(limit)
1337 
    .fetch_all(pool)
1338 
    .await
1339 
}