crates/hub/src/db/flow.rs at 55673c650986 · andymccall/oxygenhub

# 55673c650986 oxygenhub / crates / hub / src / db / flow.rs
42.9 KiB · Rust · last changed Flow: repositories take turns on the runners (fair share) 21 days ago source blame history raw
//! OxygenFlow runs, jobs and logs (hub spec §13.2).

use super::{now_ms, Pool};
use sqlx::FromRow;

#[derive(Clone, Debug, FromRow)]
pub struct Run {
    pub id: i64,
    pub repo_id: i64,
    /// The run's number within its repository, from 1 (migration 0019).
    #[sqlx(default)]
    pub number: i64,
    /// `push`, `manual` or `rerun:N` (migration 0020).
    #[sqlx(default)]
    pub trigger: String,
    pub bookmark: String,
    pub state: String,
    pub heads: String,
    pub pipeline_hash: String,
    pub status: String,
    pub error: String,
    pub actor: String,
    pub created_ms: i64,
    pub finished_ms: Option<i64>,
}

impl Run {
    /// How the run page says what started it.
    pub fn started_by(&self) -> String {
        match self.trigger.as_str() {
            "manual" => format!("started by {}", self.actor),
            "schedule" => "started on schedule".to_string(),
            t if t.starts_with("tag:") => format!("tag {} pushed by {}", &t[4..], self.actor),
            t if t.starts_with("rerun:") => format!("re-run of #{} by {}", self.rerun_of().unwrap_or(""), self.actor),
            _ => format!("pushed by {}", self.actor),
        }
    }

    /// The run a rerun repeats: `N` of `rerun:N` or `rerun:N:EVENT`.
    pub fn rerun_of(&self) -> Option<&str> {
        let rest = self.trigger.strip_prefix("rerun:")?;
        Some(rest.split(':').next().unwrap_or(rest))
    }

    /// The trigger as people read it: `rerun:N`, without the event it repeats.
    pub fn trigger_label(&self) -> String {
        match self.rerun_of() {
            Some(n) => format!("rerun:{n}"),
            None => self.trigger.clone(),
        }
    }

    pub fn head_list(&self) -> Vec<&str> {
        self.heads.split('+').filter(|h| !h.is_empty()).collect()
    }

    pub fn live(&self) -> bool {
        matches!(self.status.as_str(), "queued" | "running")
    }
}

#[derive(Clone, Debug, Default, FromRow)]
pub struct Job {
    pub id: i64,
    pub run_id: i64,
    pub name: String,
    pub depth: i64,
    pub depends: String,
    pub spec: String,
    pub sandbox: String,
    pub runtime: String,
    pub status: String,
    pub runner: String,
    pub token_hash: String,
    pub started_ms: Option<i64>,
    pub finished_ms: Option<i64>,
    pub exit_code: Option<i64>,
    pub error: String,
    pub receipt: String,
    pub receipt_hash: String,
    pub log_count: i64,
    /// Comma-separated ids of other repositories the job token may read.
    pub also_read: String,
    /// The receipt attestation kept in the repository (Oxygen spec §14.6).
    pub receipt_attest: String,
    /// Who signed it: the runner's identity name.
    pub receipt_signer: String,
    /// Why a signed receipt was not kept.
    pub receipt_note: String,
    /// Why a queued job no runner can take waits (spec §11.4), from the
    /// last placement pass; empty when it is only waiting its turn.
    pub waiting: String,
    /// What it took, as its runner reported (oxygenflow#41), in MiB.
    pub peak_disk_mib: Option<i64>,
    pub peak_memory_mib: Option<i64>,
    /// The free disk it needs, learned from past runs of the same job.
    pub need_disk_mib: Option<i64>,
    /// Why it ran on its runner (oxygenflow#42).
    pub placed: String,
    /// `waiting` for a person, `approved`, or empty (oxygenflow#57).
    pub approval: String,
    /// Who approved or rejected it, and when.
    pub approved_by: String,
    pub approved_ms: Option<i64>,
    /// The environment it deploys to, or empty (oxygenflow#59).
    pub environment: String,
    /// The approval attestation that released it, when sealed (#235).
    pub approval_seal: String,
}

impl Job {
    pub fn depends_on(&self) -> Vec<String> {
        serde_json::from_str(&self.depends).unwrap_or_default()
    }

    pub fn done(&self) -> bool {
        matches!(self.status.as_str(), "passed" | "failed" | "skipped" | "error" | "cancelled")
    }
}

#[derive(Clone, Debug, FromRow)]
pub struct LogLine {
    pub seq: i64,
    pub t: i64,
    pub stream: String,
    pub text: String,
}

/// A job as the trigger creates it.
pub struct NewJob {
    pub name: String,
    pub depth: i64,
    pub depends: Vec<String>,
    pub spec: String,
    pub sandbox: String,
    pub runtime: String,
    /// The free disk it needs, from [`learned_disk_need`].
    pub need_disk_mib: Option<i64>,
}

/// Runs of a job whose peaks set its next need.
const NEED_FROM_RUNS: i64 = 5;

/// The free disk a new run of job `name` in repository `repo_id` needs
/// (oxygenflow#41): the most its last runs took, plus a quarter; `None`
/// for a job that has not run here yet.
pub async fn learned_disk_need(pool: &Pool, repo_id: i64, name: &str) -> sqlx::Result<Option<i64>> {
    let peak: Option<i64> = sqlx::query_scalar(
        "SELECT MAX(peak_disk_mib) FROM (SELECT j.peak_disk_mib FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id \
         WHERE r.repo_id = $1 AND j.name = $2 AND j.peak_disk_mib IS NOT NULL ORDER BY j.id DESC LIMIT $3)",
    )
    .bind(repo_id)
    .bind(name)
    .bind(NEED_FROM_RUNS)
    .fetch_one(pool)
    .await?;
    Ok(peak.map(|p| p + p / 4 + 64))
}

/// Record why job `job_id` ran where it did.
pub async fn set_placed(pool: &Pool, job_id: i64, placed: &str) -> sqlx::Result<()> {
    sqlx::query("UPDATE flow_jobs SET placed = $1 WHERE id = $2").bind(placed).bind(job_id).execute(pool).await?;
    Ok(())
}

/// Keep what a job took, as its runner reported.
pub async fn set_peaks(pool: &Pool, job_id: i64, disk_mib: Option<u64>, memory_mib: Option<u64>) -> sqlx::Result<()> {
    sqlx::query("UPDATE flow_jobs SET peak_disk_mib = $1, peak_memory_mib = $2 WHERE id = $3")
        .bind(disk_mib.map(|v| v.min(i64::MAX as u64) as i64))
        .bind(memory_mib.map(|v| v.min(i64::MAX as u64) as i64))
        .bind(job_id)
        .execute(pool)
        .await?;
    Ok(())
}

pub struct NewRun<'a> {
    pub repo_id: i64,
    pub bookmark: &'a str,
    pub state: &'a str,
    pub heads: &'a str,
    pub pipeline_hash: &'a str,
    pub actor: &'a str,
    /// `push`, `manual` or `rerun:N`.
    pub trigger: &'a str,
    /// Problems when the pipeline is invalid: the run is recorded in error.
    pub error: Option<&'a str>,
}

pub async fn create_run(pool: &Pool, r: NewRun<'_>, jobs: &[NewJob]) -> sqlx::Result<i64> {
    let mut tx = super::begin_write(pool).await?;
    let now = now_ms();
    let status = if r.error.is_some() { "error" } else { "queued" };
    let id: i64 = sqlx::query_scalar(
        "INSERT INTO flow_runs (repo_id, number, bookmark, state, heads, pipeline_hash, status, error, actor, trigger, created_ms, finished_ms) \
         VALUES ($1, (SELECT COALESCE(MAX(number), 0) + 1 FROM flow_runs WHERE repo_id = $2), $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id",
    )
    .bind(r.repo_id)
    .bind(r.repo_id)
    .bind(r.bookmark)
    .bind(r.state)
    .bind(r.heads)
    .bind(r.pipeline_hash)
    .bind(status)
    .bind(r.error.unwrap_or(""))
    .bind(r.actor)
    .bind(r.trigger)
    .bind(now)
    .bind(r.error.map(|_| now))
    .fetch_one(&mut *tx)
    .await?;
    for j in jobs {
        sqlx::query(
            "INSERT INTO flow_jobs (run_id, name, depth, depends, spec, sandbox, runtime, status, need_disk_mib, approval, environment) \
             VALUES ($1, $2, $3, $4, $5, $6, $7, 'queued', $8, $9, $10)",
        )
        .bind(id)
        .bind(&j.name)
        .bind(j.depth)
        .bind(serde_json::to_string(&j.depends).unwrap_or_else(|_| "[]".into()))
        .bind(&j.spec)
        .bind(&j.sandbox)
        .bind(&j.runtime)
        .bind(j.need_disk_mib)
        .bind(if needs_approval(&j.spec) { "waiting" } else { "" })
        .bind(environment_of(&j.spec))
        .execute(&mut *tx)
        .await?;
    }
    tx.commit().await?;
    Ok(id)
}

/// The environment a compiled job deploys to, or empty (oxygenflow#59).
fn environment_of(spec: &str) -> String {
    serde_json::from_str::<serde_json::Value>(spec).ok().and_then(|v| v["environment"].as_str().map(String::from)).unwrap_or_default()
}

/// Whether a compiled job asks for approval (oxygenflow#57).
fn needs_approval(spec: &str) -> bool {
    serde_json::from_str::<serde_json::Value>(spec).is_ok_and(|v| !v["approval"].is_null())
}

/// The labels a compiled job and its sandbox ask a runner for (`runs-on`).
fn labels_in(spec: &str, sandbox: &str) -> Vec<String> {
    let mut out: Vec<String> = [spec, sandbox]
        .iter()
        .filter_map(|j| serde_json::from_str::<serde_json::Value>(j).ok())
        .flat_map(|v| v["runs_on"].as_array().cloned().unwrap_or_default())
        .filter_map(|l| l.as_str().map(String::from))
        .collect();
    out.sort();
    out.dedup();
    out
}

impl Job {
    /// The runner labels this job needs.
    pub fn needs(&self) -> Vec<String> {
        labels_in(&self.spec, &self.sandbox)
    }

    /// The architecture the job's sandbox asks for (OxygenFlow spec §11.3).
    pub fn arch(&self) -> Option<String> {
        arch_in(&self.sandbox)
    }
}

/// The architecture a sandbox's JSON asks for, if any.
fn arch_in(sandbox: &str) -> Option<String> {
    serde_json::from_str::<serde_json::Value>(sandbox).ok()?["arch"].as_str().map(String::from)
}

/// A job waiting for or held by a runner, with where it comes from.
#[derive(Clone, Debug, FromRow)]
pub struct QueueEntry {
    pub job_id: i64,
    pub run_id: i64,
    pub repo_id: i64,
    pub owner: String,
    pub name: String,
    pub visibility: String,
    pub run_number: i64,
    pub job: String,
    pub status: String,
    /// The runner holding it, when running.
    pub runner: String,
    pub spec: String,
    pub sandbox: String,
}

impl QueueEntry {
    /// The runner labels the job needs.
    pub fn needs(&self) -> Vec<String> {
        labels_in(&self.spec, &self.sandbox)
    }

    /// The architecture the job's sandbox asks for (OxygenFlow spec §11.3).
    pub fn arch(&self) -> Option<String> {
        arch_in(&self.sandbox)
    }
}

/// Every job of a live run that is running or queued: running ones first,
/// then queued ones in the order runners claim them (hub spec §13.2).
pub async fn queue(pool: &Pool) -> sqlx::Result<Vec<QueueEntry>> {
    sqlx::query_as(
        "SELECT j.id AS job_id, j.run_id, r.repo_id, p.owner, p.name, p.visibility, r.number AS run_number, j.name AS job, j.status, j.runner, j.spec, j.sandbox \
         FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id JOIN repositories p ON p.id = r.repo_id \
         WHERE j.status IN ('queued', 'running') AND j.approval != 'waiting' AND r.status IN ('queued', 'running') \
         ORDER BY CASE j.status WHEN 'running' THEN 0 ELSE 1 END, \
           CASE j.status WHEN 'running' THEN 0 ELSE ROW_NUMBER() OVER (PARTITION BY r.repo_id, j.status ORDER BY j.run_id, j.depth, j.id) \
           + (SELECT COUNT(*) FROM flow_jobs x JOIN flow_runs xr ON xr.id = x.run_id WHERE x.status = 'running' AND xr.repo_id = r.repo_id) END, \
           j.run_id, j.depth, j.id",
    )
    .fetch_all(pool)
    .await
}

/// A repository's runs, newest first, numbered below `before`.
pub async fn runs_by_number(pool: &Pool, repo_id: i64, limit: i64, before: Option<i64>) -> sqlx::Result<Vec<Run>> {
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = $1 AND number < $2 ORDER BY number DESC LIMIT $3")
        .bind(repo_id)
        .bind(before.unwrap_or(i64::MAX))
        .bind(limit)
        .fetch_all(pool)
        .await
}

/// The latest run on this bookmark of this repository, for what a push
/// changed since (a job's scope).
/// The run a push to `bookmark` is scoped against: the newest that was not
/// cancelled, nor a push run still going (a new push supersedes it), so
/// that what a cancelled run would have checked is checked by the next.
pub async fn latest_on(pool: &Pool, repo_id: i64, bookmark: &str) -> sqlx::Result<Option<Run>> {
    sqlx::query_as(
        "SELECT * FROM flow_runs WHERE repo_id = $1 AND bookmark = $2 AND heads != '' AND status != 'cancelled'
         AND NOT (trigger = 'push' AND status IN ('queued', 'running')) ORDER BY id DESC LIMIT 1",
    )
    .bind(repo_id)
    .bind(bookmark)
    .fetch_optional(pool)
    .await
}

pub async fn runs(pool: &Pool, repo_id: i64, limit: i64, before: Option<i64>) -> sqlx::Result<Vec<Run>> {
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = $1 AND id < $2 ORDER BY id DESC LIMIT $3")
        .bind(repo_id)
        .bind(before.unwrap_or(i64::MAX))
        .bind(limit)
        .fetch_all(pool)
        .await
}

/// A repository's run by its number there.
pub async fn run(pool: &Pool, repo_id: i64, number: i64) -> sqlx::Result<Option<Run>> {
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = $1 AND number = $2").bind(repo_id).bind(number).fetch_optional(pool).await
}

pub async fn run_by_id(pool: &Pool, id: i64) -> sqlx::Result<Option<Run>> {
    sqlx::query_as("SELECT * FROM flow_runs WHERE id = $1").bind(id).fetch_optional(pool).await
}

pub async fn jobs(pool: &Pool, run_id: i64) -> sqlx::Result<Vec<Job>> {
    sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = $1 ORDER BY depth, name").bind(run_id).fetch_all(pool).await
}

pub async fn job(pool: &Pool, id: i64) -> sqlx::Result<Option<Job>> {
    sqlx::query_as("SELECT * FROM flow_jobs WHERE id = $1").bind(id).fetch_optional(pool).await
}

/// The running job a job token belongs to, while it may still be used.
pub async fn job_by_token(pool: &Pool, token_hash: &str, max_age_ms: i64) -> sqlx::Result<Option<Job>> {
    sqlx::query_as("SELECT * FROM flow_jobs WHERE token_hash = $1 AND status = 'running' AND started_ms >= $2")
        .bind(token_hash)
        .bind(now_ms() - max_age_ms)
        .fetch_optional(pool)
        .await
}

/// Take the oldest queued job whose dependencies all passed and whose tier
/// the runner offers, and mark it running for `runner` with a token.
/// Let a job's token also read these repositories (its pinned subrepos).
pub async fn set_also_read(pool: &Pool, job_id: i64, repo_ids: &[i64]) -> sqlx::Result<()> {
    let list: Vec<String> = repo_ids.iter().map(|i| i.to_string()).collect();
    sqlx::query("UPDATE flow_jobs SET also_read = $1 WHERE id = $2").bind(list.join(",")).bind(job_id).execute(pool).await?;
    Ok(())
}

impl Job {
    pub fn also_read_ids(&self) -> Vec<i64> {
        self.also_read.split(',').filter_map(|x| x.trim().parse().ok()).collect()
    }
}

/// What a runner can take on: free memory (MiB) and vCPUs, `None` for no
/// limit, and the rootfs images it holds, `None` when it does not say.
#[derive(Clone, Debug, Default)]
pub struct Room {
    pub memory_mib: Option<u32>,
    pub cpus: Option<u32>,
    pub images: Option<std::collections::HashSet<String>>,
    /// The runner's architecture (`x86_64`, `riscv64`, …), empty when unknown.
    pub arch: String,
    /// Each image's architecture as runners have reported it (`flow_images`).
    pub image_archs: std::collections::HashMap<String, String>,
    /// Jobs this runner declined, not to be offered to it again (spec §11.5).
    pub declined: std::collections::HashSet<i64>,
    /// Images the hub holds that this runner would fetch when it lacks
    /// them (oxygenflow#39); empty for a runner that does not fetch.
    pub fetchable: std::collections::HashSet<String>,
    /// Its free disk now, from its claim's facts (oxygenflow#40); `None`
    /// when it does not say.
    pub free_disk_mib: Option<u64>,
}

impl Room {
    /// Whether a job needing `need` MiB of disk fits in its free disk.
    pub fn disk_fits(&self, need: Option<i64>) -> bool {
        match (need, self.free_disk_mib) {
            (Some(n), Some(free)) => n.max(0) as u64 <= free,
            _ => true,
        }
    }
}

impl Room {
    /// Whether a job's sandbox (its stored JSON) fits: its memory and vCPUs,
    /// and its rootfs image among the runner's (oxygenhub#73).
    pub fn fits(&self, sandbox_json: &str) -> bool {
        let v: serde_json::Value = serde_json::from_str(sandbox_json).unwrap_or_default();
        let need = |k: &str| v[k].as_u64().unwrap_or(0);
        let rootfs = v["rootfs"].as_str().unwrap_or("");
        // The architecture: the sandbox's own ask, else the image's as read
        // or learned from its passes (oxygenhub#173); one nobody has read
        // or run yet matches (OxygenFlow spec §11.3).
        let wanted = v["arch"].as_str().map(String::from).or_else(|| self.image_archs.get(rootfs).cloned());
        let arch_ok = self.arch.is_empty() || wanted.as_deref().is_none_or(|w| w == self.arch);
        arch_ok
            && self.memory_mib.is_none_or(|m| need("memory_mib") <= m as u64)
            && self.cpus.is_none_or(|c| need("cpus") <= c as u64)
            && (rootfs.is_empty() || self.images.as_ref().is_none_or(|have| have.contains(rootfs)) || self.fetchable.contains(rootfs))
    }
}

/// Remember which architecture each image is for, as a runner reports it.
pub async fn record_image_archs(pool: &Pool, archs: &std::collections::BTreeMap<String, String>) -> sqlx::Result<()> {
    for (hash, arch) in archs.iter().take(500) {
        if hash.len() != 64 || !flow_core::arch_ok(arch) {
            continue;
        }
        sqlx::query("INSERT INTO flow_images (hash, arch, seen_ms) VALUES ($1, $2, $3) ON CONFLICT (hash) DO UPDATE SET arch = excluded.arch, seen_ms = excluded.seen_ms")
        .bind(hash.to_ascii_lowercase())
        .bind(arch)
        .bind(now_ms())
        .execute(pool)
        .await?;
    }
    Ok(())
}

/// Give every image nobody has read the architecture of the runners its
/// jobs passed on (oxygenhub#173), for the jobs before the hub learned at
/// each pass. Never overrides a known architecture.
pub async fn learn_archs_from_runs(pool: &Pool) -> sqlx::Result<u64> {
    let r = sqlx::query(
        "INSERT INTO flow_images (hash, arch, seen_ms)
         SELECT lower(json_extract(j.sandbox, '$.rootfs')), MIN(r.arch), $1 FROM flow_jobs j JOIN flow_runners r ON r.id = j.runner_id
         WHERE j.status = 'passed' AND r.arch != '' AND length(json_extract(j.sandbox, '$.rootfs')) = 64
         GROUP BY lower(json_extract(j.sandbox, '$.rootfs'))
         ON CONFLICT (hash) DO NOTHING",
    )
    .bind(now_ms())
    .execute(pool)
    .await?;
    Ok(r.rows_affected())
}

/// Every image's architecture the hub knows.
pub async fn image_archs(pool: &Pool) -> sqlx::Result<std::collections::HashMap<String, String>> {
    let rows: Vec<(String, String)> = sqlx::query_as("SELECT hash, arch FROM flow_images").fetch_all(pool).await?;
    Ok(rows.into_iter().collect())
}

pub async fn claim(pool: &Pool, runtimes: &[String], runner: &str, token_hash: &str) -> sqlx::Result<Option<Job>> {
    claim_within(pool, runtimes, runner, token_hash, Room::default(), &Default::default(), &[]).await
}

/// The first queued job a runner with these tiers, this room and this
/// scope may take, now running under `runner`. Repositories take turns
/// (fair share, oxygenhub#244): a job's turn is its place in its own
/// repository's queue plus that repository's running jobs, lowest first,
/// oldest first among equals, so one repository's burst can't hold up the
/// rest. A scoped runner never sees
/// another scope's jobs (oxygenhub#41).
pub async fn claim_within(
    pool: &Pool,
    runtimes: &[String],
    runner: &str,
    token_hash: &str,
    room: Room,
    scope: &super::runners::Scope,
    labels: &[String],
) -> sqlx::Result<Option<Job>> {
    claim_ranked(pool, runtimes, runner, token_hash, room, scope, labels, &|_| false).await
}

/// [`claim_within`], passing over a job `elsewhere` says a better runner
/// should take (oxygenflow#42).
#[allow(clippy::too_many_arguments)]
pub async fn claim_ranked(
    pool: &Pool,
    runtimes: &[String],
    runner: &str,
    token_hash: &str,
    room: Room,
    scope: &super::runners::Scope,
    labels: &[String],
    elsewhere: &(dyn Fn(&Job) -> bool + Sync),
) -> sqlx::Result<Option<Job>> {
    let mut tx = super::begin_write(pool).await?;
    let queued: Vec<Job> = sqlx::query_as(
        "SELECT j.* FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id JOIN repositories rep ON rep.id = r.repo_id \
         WHERE j.status = 'queued' AND j.approval != 'waiting' AND r.status IN ('queued', 'running') \
         AND ($1 = '' OR rep.owner = $2) AND ($3 IS NULL OR r.repo_id = $4) \
         AND NOT ($5 = '' AND $6 IS NULL AND rep.flow = 'own') \
         ORDER BY ROW_NUMBER() OVER (PARTITION BY r.repo_id ORDER BY j.run_id, j.depth, j.id) \
           + (SELECT COUNT(*) FROM flow_jobs x JOIN flow_runs xr ON xr.id = x.run_id WHERE x.status = 'running' AND xr.repo_id = r.repo_id), \
           j.run_id, j.depth, j.id LIMIT 200",
    )
    .bind(&scope.owner)
    .bind(&scope.owner)
    .bind(scope.repo_id)
    .bind(scope.repo_id)
    // A repository on its own runners only (#225): the site's runners skip it.
    .bind(&scope.owner)
    .bind(scope.repo_id)
    .fetch_all(&mut *tx)
    .await?;
    for j in queued {
        if !runtimes.iter().any(|r| r == &j.runtime)
            || !room.fits(&j.sandbox)
            || !room.disk_fits(j.need_disk_mib)
            || room.declined.contains(&j.id)
        {
            continue;
        }
        // Every label the job and its sandbox ask for (`runs-on`).
        if !j.needs().iter().all(|l| labels.contains(l)) {
            continue;
        }
        let deps = j.depends_on();
        if !deps.is_empty() {
            let mut passed = 0i64;
            for d in &deps {
                let st: Option<String> = sqlx::query_scalar("SELECT status FROM flow_jobs WHERE run_id = $1 AND name = $2")
                    .bind(j.run_id)
                    .bind(d)
                    .fetch_optional(&mut *tx)
                    .await?;
                if st.as_deref() == Some("passed") {
                    passed += 1;
                }
            }
            if passed != deps.len() as i64 {
                continue;
            }
        }
        if elsewhere(&j) {
            continue;
        }
        let now = now_ms();
        let done = sqlx::query(
            "UPDATE flow_jobs SET status = 'running', runner = $1, token_hash = $2, started_ms = $3, waiting = '' WHERE id = $4 AND status = 'queued'",
        )
        .bind(runner)
        .bind(token_hash)
        .bind(now)
        .bind(j.id)
        .execute(&mut *tx)
        .await?;
        if done.rows_affected() == 1 {
            sqlx::query("UPDATE flow_runs SET status = 'running' WHERE id = $1 AND status = 'queued'")
                .bind(j.run_id)
                .execute(&mut *tx)
                .await?;
            tx.commit().await?;
            return job(pool, j.id).await;
        }
    }
    tx.commit().await?;
    Ok(None)
}

/// How long a decline keeps a job from its runner when the runner's
/// profile stays the same: a reason such as memory in use passes.
pub const DECLINE_HOLDS_MS: i64 = 10 * 60_000;

/// The jobs runner `runner_id` declined with this profile, recently.
pub async fn declined_by(pool: &Pool, runner_id: i64, profile: &str) -> sqlx::Result<std::collections::HashSet<i64>> {
    let rows: Vec<(i64,)> = sqlx::query_as("SELECT job_id FROM flow_declines WHERE runner_id = $1 AND profile = $2 AND ms > $3")
        .bind(runner_id)
        .bind(profile)
        .bind(now_ms() - DECLINE_HOLDS_MS)
        .fetch_all(pool)
        .await?;
    Ok(rows.into_iter().map(|(j,)| j).collect())
}

/// A decline: one runner's reason for handing a job back.
#[derive(Clone, Debug, sqlx::FromRow)]
pub struct DeclineRow {
    pub job_id: i64,
    pub runner_id: i64,
    pub runner: String,
    pub reason: String,
    pub ms: i64,
}

/// Runner `runner_id` hands back job `job_id`, running on it: the decline
/// is kept (one per job and runner, the latest), and the job is queued
/// again. Returns every decline the job has had, or `None` when the job
/// was not running on that runner.
pub async fn decline(
    pool: &Pool,
    job_id: i64,
    runner_id: i64,
    runner: &str,
    profile: &str,
    reason: &str,
) -> sqlx::Result<Option<Vec<DeclineRow>>> {
    let mut tx = super::begin_write(pool).await?;
    let back = sqlx::query(
        "UPDATE flow_jobs SET status = 'queued', runner = '', token_hash = '', started_ms = NULL, runner_id = NULL \
         WHERE id = $1 AND status = 'running' AND runner_id = $2",
    )
    .bind(job_id)
    .bind(runner_id)
    .execute(&mut *tx)
    .await?;
    if back.rows_affected() != 1 {
        tx.rollback().await?;
        return Ok(None);
    }
    sqlx::query(
        "INSERT INTO flow_declines (job_id, runner_id, runner, profile, reason, ms) VALUES ($1, $2, $3, $4, $5, $6) \
         ON CONFLICT (job_id, runner_id) DO UPDATE SET runner = excluded.runner, profile = excluded.profile, reason = excluded.reason, ms = excluded.ms",
    )
    .bind(job_id)
    .bind(runner_id)
    .bind(runner)
    .bind(profile)
    .bind(reason)
    .bind(now_ms())
    .execute(&mut *tx)
    .await?;
    let all: Vec<DeclineRow> =
        sqlx::query_as("SELECT job_id, runner_id, runner, reason, ms FROM flow_declines WHERE job_id = $1 ORDER BY ms")
            .bind(job_id)
            .fetch_all(&mut *tx)
            .await?;
    tx.commit().await?;
    Ok(Some(all))
}

/// The declines of job `job_id`, oldest first.
pub async fn declines_of_job(pool: &Pool, job_id: i64) -> sqlx::Result<Vec<DeclineRow>> {
    sqlx::query_as("SELECT job_id, runner_id, runner, reason, ms FROM flow_declines WHERE job_id = $1 ORDER BY ms")
        .bind(job_id)
        .fetch_all(pool)
        .await
}

/// A runner's latest declines, newest first, with where the job is.
#[derive(Clone, Debug, sqlx::FromRow)]
pub struct RunnerDecline {
    pub job_id: i64,
    pub job: String,
    pub run_number: i64,
    pub owner: String,
    pub name: String,
    pub reason: String,
    pub ms: i64,
}

pub async fn declines_of_runner(pool: &Pool, runner_id: i64, limit: i64) -> sqlx::Result<Vec<RunnerDecline>> {
    sqlx::query_as(
        "SELECT d.job_id, j.name AS job, r.number AS run_number, rep.owner, rep.name, d.reason, d.ms \
         FROM flow_declines d JOIN flow_jobs j ON j.id = d.job_id JOIN flow_runs r ON r.id = j.run_id \
         JOIN repositories rep ON rep.id = r.repo_id WHERE d.runner_id = $1 ORDER BY d.ms DESC LIMIT $2",
    )
    .bind(runner_id)
    .bind(limit)
    .fetch_all(pool)
    .await
}

/// Queued jobs whose dependencies have passed, in their live runs, with
/// their repositories (owner, id): the ones a runner could take now.
pub async fn ready_queued(pool: &Pool) -> sqlx::Result<Vec<(Job, String, i64)>> {
    let queued: Vec<Job> = sqlx::query_as(
        "SELECT j.* FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id \
         WHERE j.status = 'queued' AND j.approval != 'waiting' AND r.status IN ('queued', 'running') ORDER BY j.run_id, j.depth, j.id LIMIT 500",
    )
    .fetch_all(pool)
    .await?;
    let mut out = Vec::new();
    for j in queued {
        let mut ready = true;
        for d in j.depends_on() {
            let st: Option<String> = sqlx::query_scalar("SELECT status FROM flow_jobs WHERE run_id = $1 AND name = $2")
                .bind(j.run_id)
                .bind(&d)
                .fetch_optional(pool)
                .await?;
            if st.as_deref() != Some("passed") {
                ready = false;
                break;
            }
        }
        if !ready {
            continue;
        }
        let repo: Option<(String, i64)> =
            sqlx::query_as("SELECT rep.owner, rep.id FROM flow_runs r JOIN repositories rep ON rep.id = r.repo_id WHERE r.id = $1")
                .bind(j.run_id)
                .fetch_optional(pool)
                .await?;
        if let Some((owner, id)) = repo {
            out.push((j, owner, id));
        }
    }
    Ok(out)
}

pub async fn set_waiting(pool: &Pool, job_id: i64, waiting: &str) -> sqlx::Result<()> {
    sqlx::query("UPDATE flow_jobs SET waiting = $1 WHERE id = $2 AND waiting <> $3")
        .bind(waiting)
        .bind(job_id)
        .bind(waiting)
        .execute(pool)
        .await?;
    Ok(())
}

/// Append records; returns how many were kept (the rest exceed the cap).
pub async fn append_log(pool: &Pool, job_id: i64, records: &[flow_core::api::Record], cap: i64) -> sqlx::Result<usize> {
    let mut tx = super::begin_write(pool).await?;
    let count: i64 = sqlx::query_scalar("SELECT log_count FROM flow_jobs WHERE id = $1").bind(job_id).fetch_one(&mut *tx).await?;
    let room = (cap - count).max(0) as usize;
    let keep = &records[..records.len().min(room)];
    for (i, r) in keep.iter().enumerate() {
        sqlx::query("INSERT INTO flow_logs (job_id, seq, t, stream, text) VALUES ($1, $2, $3, $4, $5)")
            .bind(job_id)
            .bind(count + i as i64 + 1)
            .bind(r.t as i64)
            .bind(&r.stream)
            .bind(&r.text)
            .execute(&mut *tx)
            .await?;
    }
    sqlx::query("UPDATE flow_jobs SET log_count = log_count + $1 WHERE id = $2")
        .bind(keep.len() as i64)
        .bind(job_id)
        .execute(&mut *tx)
        .await?;
    tx.commit().await?;
    Ok(keep.len())
}

pub async fn log(pool: &Pool, job_id: i64, after: i64, limit: i64) -> sqlx::Result<Vec<LogLine>> {
    sqlx::query_as("SELECT seq, t, stream, text FROM flow_logs WHERE job_id = $1 AND seq > $2 ORDER BY seq LIMIT $3")
        .bind(job_id)
        .bind(after)
        .bind(limit)
        .fetch_all(pool)
        .await
}

/// Jobs running since before `ms` (the oldest first).
/// Jobs still running at `now` past their own time limit (`default_s` for
/// one claimed without) plus `grace_ms`.
pub async fn running_past_limit(pool: &Pool, now: i64, default_s: u64, grace_ms: i64) -> sqlx::Result<Vec<Job>> {
    sqlx::query_as(
        "SELECT * FROM flow_jobs WHERE status = 'running' \
         AND COALESCE(started_ms, 0) + (CASE WHEN timeout_s > 0 THEN timeout_s ELSE $1 END) * 1000 + $2 < $3 ORDER BY started_ms",
    )
    .bind(default_s as i64)
    .bind(grace_ms)
    .bind(now)
    .fetch_all(pool)
    .await
}

/// Record the limit a claimed job runs under.
pub async fn set_job_timeout(pool: &Pool, job_id: i64, timeout_s: u64) -> sqlx::Result<()> {
    sqlx::query("UPDATE flow_jobs SET timeout_s = $1 WHERE id = $2").bind(timeout_s as i64).bind(job_id).execute(pool).await?;
    Ok(())
}

pub async fn running_since_before(pool: &Pool, ms: i64) -> sqlx::Result<Vec<Job>> {
    sqlx::query_as("SELECT * FROM flow_jobs WHERE status = 'running' AND COALESCE(started_ms, 0) < $1 ORDER BY started_ms")
        .bind(ms)
        .fetch_all(pool)
        .await
}

/// The jobs running on the runner with this id.
pub async fn running_on(pool: &Pool, runner_id: i64) -> sqlx::Result<Vec<Job>> {
    sqlx::query_as("SELECT * FROM flow_jobs WHERE status = 'running' AND runner_id = $1 ORDER BY id").bind(runner_id).fetch_all(pool).await
}

pub struct Outcome<'a> {
    pub status: &'a str,
    pub exit_code: Option<i64>,
    pub error: &'a str,
    pub receipt: &'a str,
    pub receipt_hash: &'a str,
}

/// Record a job's end, skip the jobs that can no longer run, and settle the
/// run. Returns the jobs whose status changed (for their checks).
pub async fn finish(pool: &Pool, job_id: i64, o: Outcome<'_>) -> sqlx::Result<Vec<Job>> {
    let mut tx = super::begin_write(pool).await?;
    let now = now_ms();
    let j: Job = sqlx::query_as("SELECT * FROM flow_jobs WHERE id = $1").bind(job_id).fetch_one(&mut *tx).await?;
    sqlx::query("UPDATE flow_jobs SET status = $1, exit_code = $2, error = $3, receipt = $4, receipt_hash = $5, finished_ms = $6, token_hash = '' WHERE id = $7")
        .bind(o.status)
        .bind(o.exit_code)
        .bind(o.error)
        .bind(o.receipt)
        .bind(o.receipt_hash)
        .bind(now)
        .bind(job_id)
        .execute(&mut *tx)
        .await?;
    // A pass teaches what its image is for, when nobody has read it
    // (oxygenhub#173): the architecture of the runner it passed on.
    if o.status == "passed" {
        let rootfs = serde_json::from_str::<serde_json::Value>(&j.sandbox)
            .ok()
            .and_then(|v| v["rootfs"].as_str().map(str::to_ascii_lowercase))
            .unwrap_or_default();
        if rootfs.len() == 64 {
            sqlx::query(
                "INSERT INTO flow_images (hash, arch, seen_ms) SELECT $1, r.arch, $2 FROM flow_jobs j JOIN flow_runners r ON r.id = j.runner_id \
                 WHERE j.id = $3 AND r.arch != '' ON CONFLICT (hash) DO NOTHING",
            )
            .bind(&rootfs)
            .bind(now)
            .bind(job_id)
            .execute(&mut *tx)
            .await?;
        }
    }
    let mut changed = vec![job_id];
    // Dependents of anything that did not pass are skipped, transitively.
    loop {
        let all: Vec<Job> = sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = $1").bind(j.run_id).fetch_all(&mut *tx).await?;
        let blocked: Vec<i64> = all
            .iter()
            .filter(|x| x.status == "queued")
            .filter(|x| {
                x.depends_on()
                    .iter()
                    .any(|d| all.iter().any(|y| &y.name == d && matches!(y.status.as_str(), "failed" | "skipped" | "error" | "cancelled")))
            })
            .map(|x| x.id)
            .collect();
        if blocked.is_empty() {
            break;
        }
        for id in blocked {
            sqlx::query("UPDATE flow_jobs SET status = 'skipped', finished_ms = $1 WHERE id = $2")
                .bind(now)
                .bind(id)
                .execute(&mut *tx)
                .await?;
            changed.push(id);
        }
    }
    let all: Vec<Job> = sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = $1").bind(j.run_id).fetch_all(&mut *tx).await?;
    if all.iter().all(Job::done) {
        let status = if all.iter().all(|x| x.status == "passed") { "passed" } else { "failed" };
        sqlx::query("UPDATE flow_runs SET status = $1, finished_ms = $2 WHERE id = $3")
            .bind(status)
            .bind(now)
            .bind(j.run_id)
            .execute(&mut *tx)
            .await?;
    }
    tx.commit().await?;
    let mut out = Vec::new();
    for id in changed {
        if let Some(x) = job(pool, id).await? {
            out.push(x);
        }
    }
    Ok(out)
}

/// Cancel a run (oxygenflow#32): its jobs not yet done end as cancelled,
/// and so does the run. A runner learns of it when the hub next refuses
/// its log lines, and stops the job. Returns the jobs whose status changed.
pub async fn cancel_run(pool: &Pool, run_id: i64, why: &str) -> sqlx::Result<Vec<Job>> {
    let now = now_ms();
    let mut tx = pool.begin().await?;
    let ids: Vec<(i64,)> = sqlx::query_as("SELECT id FROM flow_jobs WHERE run_id = $1 AND status IN ('queued', 'running')")
        .bind(run_id)
        .fetch_all(&mut *tx)
        .await?;
    sqlx::query(
        "UPDATE flow_jobs SET status = 'cancelled', error = $1, finished_ms = $2, token_hash = ''
         WHERE run_id = $3 AND status IN ('queued', 'running')",
    )
    .bind(why)
    .bind(now)
    .bind(run_id)
    .execute(&mut *tx)
    .await?;
    sqlx::query("UPDATE flow_runs SET status = 'cancelled', finished_ms = $1 WHERE id = $2 AND status IN ('queued', 'running')")
        .bind(now)
        .bind(run_id)
        .execute(&mut *tx)
        .await?;
    tx.commit().await?;
    let mut out = Vec::new();
    for (id,) in ids {
        if let Some(x) = job(pool, id).await? {
            out.push(x);
        }
    }
    Ok(out)
}

/// Run job `name` of finished run `run_id` again, with every job that
/// depends on it (oxygenflow#31): back to the queue as if new, their logs,
/// artifacts and reports gone, and the run running again. The jobs they
/// depend on keep their results. The jobs reset; none when there is no
/// such job.
pub async fn rerun_job(pool: &Pool, run_id: i64, name: &str) -> sqlx::Result<Vec<Job>> {
    let all: Vec<Job> = sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = $1").bind(run_id).fetch_all(pool).await?;
    if !all.iter().any(|j| j.name == name) {
        return Ok(Vec::new());
    }
    let mut again = vec![name.to_string()];
    loop {
        let more: Vec<String> = all
            .iter()
            .filter(|j| !again.contains(&j.name) && j.depends_on().iter().any(|d| again.contains(d)))
            .map(|j| j.name.clone())
            .collect();
        if more.is_empty() {
            break;
        }
        again.extend(more);
    }
    let ids: Vec<i64> = all.iter().filter(|j| again.contains(&j.name)).map(|j| j.id).collect();
    let mut tx = pool.begin().await?;
    for id in &ids {
        for table in ["flow_logs", "flow_artifacts", "flow_declines", "test_reports", "coverage_reports", "reports"] {
            sqlx::query(&format!("DELETE FROM {table} WHERE job_id = $1")).bind(id).execute(&mut *tx).await?;
        }
        sqlx::query(
            "UPDATE flow_jobs SET status = 'queued', runner = '', token_hash = '', started_ms = NULL, finished_ms = NULL,
             exit_code = NULL, error = '', receipt = '', receipt_hash = '', log_count = 0, also_read = '', receipt_attest = '',
             receipt_signer = '', receipt_note = '', runner_id = NULL, waiting = '', placed = '', peak_disk_mib = NULL,
             peak_memory_mib = NULL, approval = CASE WHEN approval = '' THEN '' ELSE 'waiting' END, approved_by = '',
             approved_ms = NULL, approval_seal = '' WHERE id = $1",
        )
        .bind(id)
        .execute(&mut *tx)
        .await?;
    }
    sqlx::query("UPDATE flow_runs SET status = 'running', finished_ms = NULL WHERE id = $1").bind(run_id).execute(&mut *tx).await?;
    tx.commit().await?;
    let mut out = Vec::new();
    for id in ids {
        if let Some(x) = job(pool, id).await? {
            out.push(x);
        }
    }
    Ok(out)
}

/// The push runs of `bookmark` still going, older than run `number`.
pub async fn live_push_runs_before(pool: &Pool, repo_id: i64, bookmark: &str, number: i64) -> sqlx::Result<Vec<Run>> {
    sqlx::query_as(
        "SELECT * FROM flow_runs WHERE repo_id = $1 AND bookmark = $2 AND number < $3 AND trigger = 'push'
         AND status IN ('queued', 'running') ORDER BY id",
    )
    .bind(repo_id)
    .bind(bookmark)
    .bind(number)
    .fetch_all(pool)
    .await
}

/// The newest run of exactly this state (heads joined with `+`, sorted).
pub async fn latest_for_heads(pool: &Pool, repo_id: i64, heads: &str) -> sqlx::Result<Option<Run>> {
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = $1 AND heads = $2 ORDER BY id DESC LIMIT 1")
        .bind(repo_id)
        .bind(heads)
        .fetch_optional(pool)
        .await
}

/// Record what became of job `job_id`'s signed receipt.
pub async fn set_receipt(pool: &Pool, job_id: i64, attest: &str, signer: &str, note: &str) -> sqlx::Result<()> {
    sqlx::query("UPDATE flow_jobs SET receipt_attest = $1, receipt_signer = $2, receipt_note = $3 WHERE id = $4")
        .bind(attest)
        .bind(signer)
        .bind(note)
        .bind(job_id)
        .execute(pool)
        .await?;
    Ok(())
}

/// A job's artifact, kept as a blob of the run's repository.
#[derive(Clone, Debug, FromRow)]
pub struct Artifact {
    pub job_id: i64,
    pub name: String,
    pub path: String,
    pub blob: String,
    pub size: i64,
    /// BLAKE3 of the bytes.
    pub content: String,
    pub created_ms: i64,
}

pub async fn add_artifact(pool: &Pool, a: &Artifact) -> sqlx::Result<()> {
    sqlx::query(
        "INSERT OR REPLACE INTO flow_artifacts (job_id, name, path, blob, size, content, created_ms) VALUES ($1, $2, $3, $4, $5, $6, $7)",
    )
    .bind(a.job_id)
    .bind(&a.name)
    .bind(&a.path)
    .bind(&a.blob)
    .bind(a.size)
    .bind(&a.content)
    .bind(a.created_ms)
    .execute(pool)
    .await?;
    Ok(())
}

/// Every artifact blob a repository's runs keep, for garbage collection to
/// keep too (#126): the database is the only thing that names them.
pub async fn artifact_blobs(pool: &Pool, repo_id: i64) -> sqlx::Result<Vec<String>> {
    sqlx::query_scalar(
        "SELECT DISTINCT a.blob FROM flow_artifacts a JOIN flow_jobs j ON j.id = a.job_id JOIN flow_runs r ON r.id = j.run_id WHERE r.repo_id = $1",
    )
    .bind(repo_id)
    .fetch_all(pool)
    .await
}

/// Job `job_id`'s artifacts, by name.
pub async fn artifacts(pool: &Pool, job_id: i64) -> sqlx::Result<Vec<Artifact>> {
    sqlx::query_as("SELECT * FROM flow_artifacts WHERE job_id = $1 ORDER BY name").bind(job_id).fetch_all(pool).await
}

/// Where an artifact with given content came from: the run and job that
/// built it, and the job's signed receipt when one was kept.
#[derive(Clone, Debug, FromRow)]
pub struct Provenance {
    pub run_number: i64,
    pub job: String,
    pub name: String,
    pub receipt_attest: String,
    pub receipt_signer: String,
}

/// The build of the bytes whose BLAKE3 is `content` in repository
/// `repo_id`: a passed job, one with a kept signed receipt first, newest
/// run first.
pub async fn provenance(pool: &Pool, repo_id: i64, content: &str) -> sqlx::Result<Option<Provenance>> {
    sqlx::query_as(
        "SELECT r.number AS run_number, j.name AS job, a.name AS name, j.receipt_attest, j.receipt_signer \
         FROM flow_artifacts a JOIN flow_jobs j ON j.id = a.job_id JOIN flow_runs r ON r.id = j.run_id \
         WHERE r.repo_id = $1 AND a.content = $2 AND j.status = 'passed' \
         ORDER BY (j.receipt_attest != '') DESC, r.number DESC LIMIT 1",
    )
    .bind(repo_id)
    .bind(content)
    .fetch_optional(pool)
    .await
}

/// Approve job `id` if it is still waiting for a person (oxygenflow#57);
/// whether it was.
pub async fn approve_job(pool: &Pool, id: i64, who: &str, seal: &str) -> sqlx::Result<bool> {
    let r = sqlx::query(
        "UPDATE flow_jobs SET approval = 'approved', approved_by = $1, approved_ms = $2, approval_seal = $3 \
         WHERE id = $4 AND status = 'queued' AND approval = 'waiting'",
    )
    .bind(who)
    .bind(now_ms())
    .bind(seal)
    .bind(id)
    .execute(pool)
    .await?;
    Ok(r.rows_affected() > 0)
}

/// Note who rejected job `id` (it is then finished as cancelled).
pub async fn note_rejection(pool: &Pool, id: i64, who: &str) -> sqlx::Result<()> {
    sqlx::query("UPDATE flow_jobs SET approved_by = $1, approved_ms = $2 WHERE id = $3")
        .bind(who)
        .bind(now_ms())
        .bind(id)
        .execute(pool)
        .await?;
    Ok(())
}

/// Jobs waiting for a person, in runs still going.
pub async fn waiting_for_approval(pool: &Pool) -> sqlx::Result<Vec<Job>> {
    sqlx::query_as(
        "SELECT j.* FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id \
         WHERE j.status = 'queued' AND j.approval = 'waiting' AND r.status IN ('queued', 'running')",
    )
    .fetch_all(pool)
    .await
}

/// One deployment: a job for an environment that passed.
#[derive(Clone, Debug, FromRow)]
pub struct Deployment {
    pub environment: String,
    pub job: String,
    pub run_number: i64,
    pub state: String,
    pub bookmark: String,
    pub actor: String,
    pub approved_by: String,
    pub finished_ms: i64,
}

/// A repository's deployments, newest first (oxygenflow#59).
pub async fn deployments(pool: &Pool, repo_id: i64, limit: i64) -> sqlx::Result<Vec<Deployment>> {
    sqlx::query_as(
        "SELECT j.environment, j.name AS job, r.number AS run_number, r.state, r.bookmark, r.actor, j.approved_by, \
         COALESCE(j.finished_ms, 0) AS finished_ms FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id \
         WHERE r.repo_id = $1 AND j.environment != '' AND j.status = 'passed' ORDER BY j.finished_ms DESC LIMIT $2",
    )
    .bind(repo_id)
    .bind(limit)
    .fetch_all(pool)
    .await
}
`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet
Keyboard
