//! OxygenFlow runs, jobs and logs (hub spec §13.2). use super::{now_ms, Pool}; use sqlx::FromRow; #[derive(Clone, Debug, FromRow)] pub struct Run { pub id: i64, pub repo_id: i64, /// The run's number within its repository, from 1 (migration 0019). #[sqlx(default)] pub number: i64, /// `push`, `manual` or `rerun:N` (migration 0020). #[sqlx(default)] pub trigger: String, pub bookmark: String, pub state: String, pub heads: String, pub pipeline_hash: String, pub status: String, pub error: String, pub actor: String, pub created_ms: i64, pub finished_ms: Option, } impl Run { /// How the run page says what started it. pub fn started_by(&self) -> String { match self.trigger.as_str() { "manual" => format!("started by {}", self.actor), t if t.starts_with("rerun:") => format!("re-run of #{} by {}", &t[6..], self.actor), _ => format!("pushed by {}", self.actor), } } pub fn head_list(&self) -> Vec<&str> { self.heads.split('+').filter(|h| !h.is_empty()).collect() } pub fn live(&self) -> bool { matches!(self.status.as_str(), "queued" | "running") } } #[derive(Clone, Debug, FromRow)] pub struct Job { pub id: i64, pub run_id: i64, pub name: String, pub depth: i64, pub depends: String, pub spec: String, pub sandbox: String, pub runtime: String, pub status: String, pub runner: String, pub token_hash: String, pub started_ms: Option, pub finished_ms: Option, pub exit_code: Option, pub error: String, pub receipt: String, pub receipt_hash: String, pub log_count: i64, /// Comma-separated ids of other repositories the job token may read. pub also_read: String, /// The receipt attestation kept in the repository (Oxygen spec §14.6). pub receipt_attest: String, /// Who signed it: the runner's identity name. pub receipt_signer: String, /// Why a signed receipt was not kept. pub receipt_note: String, } impl Job { pub fn depends_on(&self) -> Vec { serde_json::from_str(&self.depends).unwrap_or_default() } pub fn done(&self) -> bool { matches!(self.status.as_str(), "passed" | "failed" | "skipped" | "error") } } #[derive(Clone, Debug, FromRow)] pub struct LogLine { pub seq: i64, pub t: i64, pub stream: String, pub text: String, } /// A job as the trigger creates it. pub struct NewJob { pub name: String, pub depth: i64, pub depends: Vec, pub spec: String, pub sandbox: String, pub runtime: String, } pub struct NewRun<'a> { pub repo_id: i64, pub bookmark: &'a str, pub state: &'a str, pub heads: &'a str, pub pipeline_hash: &'a str, pub actor: &'a str, /// `push`, `manual` or `rerun:N`. pub trigger: &'a str, /// Problems when the pipeline is invalid: the run is recorded in error. pub error: Option<&'a str>, } pub async fn create_run(pool: &Pool, r: NewRun<'_>, jobs: &[NewJob]) -> sqlx::Result { let mut tx = super::begin_write(pool).await?; let now = now_ms(); let status = if r.error.is_some() { "error" } else { "queued" }; let id: i64 = sqlx::query_scalar( "INSERT INTO flow_runs (repo_id, number, bookmark, state, heads, pipeline_hash, status, error, actor, trigger, created_ms, finished_ms) \ VALUES (?, (SELECT COALESCE(MAX(number), 0) + 1 FROM flow_runs WHERE repo_id = ?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id", ) .bind(r.repo_id) .bind(r.repo_id) .bind(r.bookmark) .bind(r.state) .bind(r.heads) .bind(r.pipeline_hash) .bind(status) .bind(r.error.unwrap_or("")) .bind(r.actor) .bind(r.trigger) .bind(now) .bind(r.error.map(|_| now)) .fetch_one(&mut *tx) .await?; for j in jobs { sqlx::query( "INSERT INTO flow_jobs (run_id, name, depth, depends, spec, sandbox, runtime, status) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued')", ) .bind(id) .bind(&j.name) .bind(j.depth) .bind(serde_json::to_string(&j.depends).unwrap_or_else(|_| "[]".into())) .bind(&j.spec) .bind(&j.sandbox) .bind(&j.runtime) .execute(&mut *tx) .await?; } tx.commit().await?; Ok(id) } /// The labels a compiled job and its sandbox ask a runner for (`runs-on`). fn labels_in(spec: &str, sandbox: &str) -> Vec { let mut out: Vec = [spec, sandbox] .iter() .filter_map(|j| serde_json::from_str::(j).ok()) .flat_map(|v| v["runs_on"].as_array().cloned().unwrap_or_default()) .filter_map(|l| l.as_str().map(String::from)) .collect(); out.sort(); out.dedup(); out } impl Job { /// The runner labels this job needs. pub fn needs(&self) -> Vec { labels_in(&self.spec, &self.sandbox) } } /// A job waiting for or held by a runner, with where it comes from. #[derive(Clone, Debug, FromRow)] pub struct QueueEntry { pub job_id: i64, pub run_id: i64, pub repo_id: i64, pub owner: String, pub name: String, pub visibility: String, pub run_number: i64, pub job: String, pub status: String, /// The runner holding it, when running. pub runner: String, pub spec: String, pub sandbox: String, } impl QueueEntry { /// The runner labels the job needs. pub fn needs(&self) -> Vec { labels_in(&self.spec, &self.sandbox) } } /// Every job of a live run that is running or queued: running ones first, /// then queued ones in the order runners claim them (hub spec §13.2). pub async fn queue(pool: &Pool) -> sqlx::Result> { sqlx::query_as( "SELECT j.id AS job_id, j.run_id, r.repo_id, p.owner, p.name, p.visibility, r.number AS run_number, j.name AS job, j.status, j.runner, j.spec, j.sandbox \ FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id JOIN repositories p ON p.id = r.repo_id \ WHERE j.status IN ('queued', 'running') AND r.status IN ('queued', 'running') \ ORDER BY CASE j.status WHEN 'running' THEN 0 ELSE 1 END, j.run_id, j.depth, j.id", ) .fetch_all(pool) .await } /// A repository's runs, newest first, numbered below `before`. pub async fn runs_by_number(pool: &Pool, repo_id: i64, limit: i64, before: Option) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = ? AND number < ? ORDER BY number DESC LIMIT ?") .bind(repo_id) .bind(before.unwrap_or(i64::MAX)) .bind(limit) .fetch_all(pool) .await } pub async fn runs(pool: &Pool, repo_id: i64, limit: i64, before: Option) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = ? AND id < ? ORDER BY id DESC LIMIT ?") .bind(repo_id) .bind(before.unwrap_or(i64::MAX)) .bind(limit) .fetch_all(pool) .await } /// A repository's run by its number there. pub async fn run(pool: &Pool, repo_id: i64, number: i64) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = ? AND number = ?").bind(repo_id).bind(number).fetch_optional(pool).await } pub async fn run_by_id(pool: &Pool, id: i64) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_runs WHERE id = ?").bind(id).fetch_optional(pool).await } pub async fn jobs(pool: &Pool, run_id: i64) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = ? ORDER BY depth, name").bind(run_id).fetch_all(pool).await } pub async fn job(pool: &Pool, id: i64) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_jobs WHERE id = ?").bind(id).fetch_optional(pool).await } /// The running job a job token belongs to, while it may still be used. pub async fn job_by_token(pool: &Pool, token_hash: &str, max_age_ms: i64) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_jobs WHERE token_hash = ? AND status = 'running' AND started_ms >= ?") .bind(token_hash) .bind(now_ms() - max_age_ms) .fetch_optional(pool) .await } /// Take the oldest queued job whose dependencies all passed and whose tier /// the runner offers, and mark it running for `runner` with a token. /// Let a job's token also read these repositories (its pinned subrepos). pub async fn set_also_read(pool: &Pool, job_id: i64, repo_ids: &[i64]) -> sqlx::Result<()> { let list: Vec = repo_ids.iter().map(|i| i.to_string()).collect(); sqlx::query("UPDATE flow_jobs SET also_read = ? WHERE id = ?").bind(list.join(",")).bind(job_id).execute(pool).await?; Ok(()) } impl Job { pub fn also_read_ids(&self) -> Vec { self.also_read.split(',').filter_map(|x| x.trim().parse().ok()).collect() } } /// What a runner can take on: free memory (MiB) and vCPUs, `None` for no /// limit, and the rootfs images it holds, `None` when it does not say. #[derive(Clone, Debug, Default)] pub struct Room { pub memory_mib: Option, pub cpus: Option, pub images: Option>, } impl Room { /// Whether a job's sandbox (its stored JSON) fits: its memory and vCPUs, /// and its rootfs image among the runner's (oxygenhub#73). pub fn fits(&self, sandbox_json: &str) -> bool { let v: serde_json::Value = serde_json::from_str(sandbox_json).unwrap_or_default(); let need = |k: &str| v[k].as_u64().unwrap_or(0); let rootfs = v["rootfs"].as_str().unwrap_or(""); self.memory_mib.is_none_or(|m| need("memory_mib") <= m as u64) && self.cpus.is_none_or(|c| need("cpus") <= c as u64) && (rootfs.is_empty() || self.images.as_ref().is_none_or(|have| have.contains(rootfs))) } } pub async fn claim(pool: &Pool, runtimes: &[String], runner: &str, token_hash: &str) -> sqlx::Result> { claim_within(pool, runtimes, runner, token_hash, Room::default(), &Default::default(), &[]).await } /// The oldest runnable job that fits in `room`. /// The first queued job a runner with these tiers, this room and this /// scope may take, now running under `runner`. A scoped runner never sees /// another scope's jobs (oxygenhub#41). pub async fn claim_within( pool: &Pool, runtimes: &[String], runner: &str, token_hash: &str, room: Room, scope: &super::runners::Scope, labels: &[String], ) -> sqlx::Result> { let mut tx = super::begin_write(pool).await?; let queued: Vec = sqlx::query_as( "SELECT j.* FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id JOIN repositories rep ON rep.id = r.repo_id \ WHERE j.status = 'queued' AND r.status IN ('queued', 'running') \ AND (? = '' OR rep.owner = ?) AND (? IS NULL OR r.repo_id = ?) \ ORDER BY j.run_id, j.depth, j.id LIMIT 200", ) .bind(&scope.owner) .bind(&scope.owner) .bind(scope.repo_id) .bind(scope.repo_id) .fetch_all(&mut *tx) .await?; for j in queued { if !runtimes.iter().any(|r| r == &j.runtime) || !room.fits(&j.sandbox) { continue; } // Every label the job and its sandbox ask for (`runs-on`). if !j.needs().iter().all(|l| labels.contains(l)) { continue; } let deps = j.depends_on(); if !deps.is_empty() { let mut passed = 0i64; for d in &deps { let st: Option = sqlx::query_scalar("SELECT status FROM flow_jobs WHERE run_id = ? AND name = ?") .bind(j.run_id) .bind(d) .fetch_optional(&mut *tx) .await?; if st.as_deref() == Some("passed") { passed += 1; } } if passed != deps.len() as i64 { continue; } } let now = now_ms(); let done = sqlx::query( "UPDATE flow_jobs SET status = 'running', runner = ?, token_hash = ?, started_ms = ? WHERE id = ? AND status = 'queued'", ) .bind(runner) .bind(token_hash) .bind(now) .bind(j.id) .execute(&mut *tx) .await?; if done.rows_affected() == 1 { sqlx::query("UPDATE flow_runs SET status = 'running' WHERE id = ? AND status = 'queued'") .bind(j.run_id) .execute(&mut *tx) .await?; tx.commit().await?; return job(pool, j.id).await; } } tx.commit().await?; Ok(None) } /// Append records; returns how many were kept (the rest exceed the cap). pub async fn append_log(pool: &Pool, job_id: i64, records: &[flow_core::api::Record], cap: i64) -> sqlx::Result { let mut tx = super::begin_write(pool).await?; let count: i64 = sqlx::query_scalar("SELECT log_count FROM flow_jobs WHERE id = ?").bind(job_id).fetch_one(&mut *tx).await?; let room = (cap - count).max(0) as usize; let keep = &records[..records.len().min(room)]; for (i, r) in keep.iter().enumerate() { sqlx::query("INSERT INTO flow_logs (job_id, seq, t, stream, text) VALUES (?, ?, ?, ?, ?)") .bind(job_id) .bind(count + i as i64 + 1) .bind(r.t as i64) .bind(&r.stream) .bind(&r.text) .execute(&mut *tx) .await?; } sqlx::query("UPDATE flow_jobs SET log_count = log_count + ? WHERE id = ?") .bind(keep.len() as i64) .bind(job_id) .execute(&mut *tx) .await?; tx.commit().await?; Ok(keep.len()) } pub async fn log(pool: &Pool, job_id: i64, after: i64, limit: i64) -> sqlx::Result> { sqlx::query_as("SELECT seq, t, stream, text FROM flow_logs WHERE job_id = ? AND seq > ? ORDER BY seq LIMIT ?") .bind(job_id) .bind(after) .bind(limit) .fetch_all(pool) .await } /// Jobs running since before `ms` (the oldest first). pub async fn running_since_before(pool: &Pool, ms: i64) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_jobs WHERE status = 'running' AND COALESCE(started_ms, 0) < ? ORDER BY started_ms") .bind(ms) .fetch_all(pool) .await } /// The jobs running on the runner with this id. pub async fn running_on(pool: &Pool, runner_id: i64) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_jobs WHERE status = 'running' AND runner_id = ? ORDER BY id").bind(runner_id).fetch_all(pool).await } pub struct Outcome<'a> { pub status: &'a str, pub exit_code: Option, pub error: &'a str, pub receipt: &'a str, pub receipt_hash: &'a str, } /// Record a job's end, skip the jobs that can no longer run, and settle the /// run. Returns the jobs whose status changed (for their checks). pub async fn finish(pool: &Pool, job_id: i64, o: Outcome<'_>) -> sqlx::Result> { let mut tx = super::begin_write(pool).await?; let now = now_ms(); let j: Job = sqlx::query_as("SELECT * FROM flow_jobs WHERE id = ?").bind(job_id).fetch_one(&mut *tx).await?; sqlx::query("UPDATE flow_jobs SET status = ?, exit_code = ?, error = ?, receipt = ?, receipt_hash = ?, finished_ms = ?, token_hash = '' WHERE id = ?") .bind(o.status) .bind(o.exit_code) .bind(o.error) .bind(o.receipt) .bind(o.receipt_hash) .bind(now) .bind(job_id) .execute(&mut *tx) .await?; let mut changed = vec![job_id]; // Dependents of anything that did not pass are skipped, transitively. loop { let all: Vec = sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = ?").bind(j.run_id).fetch_all(&mut *tx).await?; let blocked: Vec = all .iter() .filter(|x| x.status == "queued") .filter(|x| { x.depends_on() .iter() .any(|d| all.iter().any(|y| &y.name == d && matches!(y.status.as_str(), "failed" | "skipped" | "error"))) }) .map(|x| x.id) .collect(); if blocked.is_empty() { break; } for id in blocked { sqlx::query("UPDATE flow_jobs SET status = 'skipped', finished_ms = ? WHERE id = ?") .bind(now) .bind(id) .execute(&mut *tx) .await?; changed.push(id); } } let all: Vec = sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = ?").bind(j.run_id).fetch_all(&mut *tx).await?; if all.iter().all(Job::done) { let status = if all.iter().all(|x| x.status == "passed") { "passed" } else { "failed" }; sqlx::query("UPDATE flow_runs SET status = ?, finished_ms = ? WHERE id = ?") .bind(status) .bind(now) .bind(j.run_id) .execute(&mut *tx) .await?; } tx.commit().await?; let mut out = Vec::new(); for id in changed { if let Some(x) = job(pool, id).await? { out.push(x); } } Ok(out) } /// The newest run of exactly this state (heads joined with `+`, sorted). pub async fn latest_for_heads(pool: &Pool, repo_id: i64, heads: &str) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = ? AND heads = ? ORDER BY id DESC LIMIT 1") .bind(repo_id) .bind(heads) .fetch_optional(pool) .await } /// Record what became of job `job_id`'s signed receipt. pub async fn set_receipt(pool: &Pool, job_id: i64, attest: &str, signer: &str, note: &str) -> sqlx::Result<()> { sqlx::query("UPDATE flow_jobs SET receipt_attest = ?, receipt_signer = ?, receipt_note = ? WHERE id = ?") .bind(attest) .bind(signer) .bind(note) .bind(job_id) .execute(pool) .await?; Ok(()) } /// A job's artifact, kept as a blob of the run's repository. #[derive(Clone, Debug, FromRow)] pub struct Artifact { pub job_id: i64, pub name: String, pub path: String, pub blob: String, pub size: i64, /// BLAKE3 of the bytes. pub content: String, pub created_ms: i64, } pub async fn add_artifact(pool: &Pool, a: &Artifact) -> sqlx::Result<()> { sqlx::query("INSERT OR REPLACE INTO flow_artifacts (job_id, name, path, blob, size, content, created_ms) VALUES (?, ?, ?, ?, ?, ?, ?)") .bind(a.job_id) .bind(&a.name) .bind(&a.path) .bind(&a.blob) .bind(a.size) .bind(&a.content) .bind(a.created_ms) .execute(pool) .await?; Ok(()) } /// Job `job_id`'s artifacts, by name. pub async fn artifacts(pool: &Pool, job_id: i64) -> sqlx::Result> { sqlx::query_as("SELECT * FROM flow_artifacts WHERE job_id = ? ORDER BY name").bind(job_id).fetch_all(pool).await } /// Where an artifact with given content came from: the run and job that /// built it, and the job's signed receipt when one was kept. #[derive(Clone, Debug, FromRow)] pub struct Provenance { pub run_number: i64, pub job: String, pub name: String, pub receipt_attest: String, pub receipt_signer: String, } /// The build of the bytes whose BLAKE3 is `content` in repository /// `repo_id`: a passed job, one with a kept signed receipt first, newest /// run first. pub async fn provenance(pool: &Pool, repo_id: i64, content: &str) -> sqlx::Result> { sqlx::query_as( "SELECT r.number AS run_number, j.name AS job, a.name AS name, j.receipt_attest, j.receipt_signer \ FROM flow_artifacts a JOIN flow_jobs j ON j.id = a.job_id JOIN flow_runs r ON r.id = j.run_id \ WHERE r.repo_id = ? AND a.content = ? AND j.status = 'passed' \ ORDER BY (j.receipt_attest != '') DESC, r.number DESC LIMIT 1", ) .bind(repo_id) .bind(content) .fetch_optional(pool) .await }