OXYGENHUB
# cc19697f8871 oxygenhub / crates / hub / src / db / flow.rs
14.8 KiB · Rust · last changed Verify and keep signed Flow receipts 1 month ago source blame history raw
1
//! OxygenFlow runs, jobs and logs (hub spec §13.2).
2
3
use super::{now_ms, Pool};
4
use sqlx::FromRow;
5
6
#[derive(Clone, Debug, FromRow)]
7
pub struct Run {
8
    pub id: i64,
9
    pub repo_id: i64,
10
    /// The run's number within its repository, from 1 (migration 0019).
11
    #[sqlx(default)]
12
    pub number: i64,
13
    /// `push`, `manual` or `rerun:N` (migration 0020).
14
    #[sqlx(default)]
15
    pub trigger: String,
16
    pub bookmark: String,
17
    pub state: String,
18
    pub heads: String,
19
    pub pipeline_hash: String,
20
    pub status: String,
21
    pub error: String,
22
    pub actor: String,
23
    pub created_ms: i64,
24
    pub finished_ms: Option<i64>,
25
}
26
27
impl Run {
28
    /// How the run page says what started it.
29
    pub fn started_by(&self) -> String {
30
        match self.trigger.as_str() {
31
            "manual" => format!("started by {}", self.actor),
32
            t if t.starts_with("rerun:") => format!("re-run of #{} by {}", &t[6..], self.actor),
33
            _ => format!("pushed by {}", self.actor),
34
        }
35
    }
36
37
    pub fn head_list(&self) -> Vec<&str> {
38
        self.heads.split('+').filter(|h| !h.is_empty()).collect()
39
    }
40
41
    pub fn live(&self) -> bool {
42
        matches!(self.status.as_str(), "queued" | "running")
43
    }
44
}
45
46
#[derive(Clone, Debug, FromRow)]
47
pub struct Job {
48
    pub id: i64,
49
    pub run_id: i64,
50
    pub name: String,
51
    pub depth: i64,
52
    pub depends: String,
53
    pub spec: String,
54
    pub sandbox: String,
55
    pub runtime: String,
56
    pub status: String,
57
    pub runner: String,
58
    pub token_hash: String,
59
    pub started_ms: Option<i64>,
60
    pub finished_ms: Option<i64>,
61
    pub exit_code: Option<i64>,
62
    pub error: String,
63
    pub receipt: String,
64
    pub receipt_hash: String,
65
    pub log_count: i64,
66
    /// Comma-separated ids of other repositories the job token may read.
67
    pub also_read: String,
68
    /// The receipt attestation kept in the repository (Oxygen spec §14.6).
69
    pub receipt_attest: String,
70
    /// Who signed it: the runner's identity name.
71
    pub receipt_signer: String,
72
    /// Why a signed receipt was not kept.
73
    pub receipt_note: String,
74
}
75
76
impl Job {
77
    pub fn depends_on(&self) -> Vec<String> {
78
        serde_json::from_str(&self.depends).unwrap_or_default()
79
    }
80
81
    pub fn done(&self) -> bool {
82
        matches!(self.status.as_str(), "passed" | "failed" | "skipped" | "error")
83
    }
84
}
85
86
#[derive(Clone, Debug, FromRow)]
87
pub struct LogLine {
88
    pub seq: i64,
89
    pub t: i64,
90
    pub stream: String,
91
    pub text: String,
92
}
93
94
/// A job as the trigger creates it.
95
pub struct NewJob {
96
    pub name: String,
97
    pub depth: i64,
98
    pub depends: Vec<String>,
99
    pub spec: String,
100
    pub sandbox: String,
101
    pub runtime: String,
102
}
103
104
pub struct NewRun<'a> {
105
    pub repo_id: i64,
106
    pub bookmark: &'a str,
107
    pub state: &'a str,
108
    pub heads: &'a str,
109
    pub pipeline_hash: &'a str,
110
    pub actor: &'a str,
111
    /// `push`, `manual` or `rerun:N`.
112
    pub trigger: &'a str,
113
    /// Problems when the pipeline is invalid: the run is recorded in error.
114
    pub error: Option<&'a str>,
115
}
116
117
pub async fn create_run(pool: &Pool, r: NewRun<'_>, jobs: &[NewJob]) -> sqlx::Result<i64> {
118
    let mut tx = super::begin_write(pool).await?;
119
    let now = now_ms();
120
    let status = if r.error.is_some() { "error" } else { "queued" };
121
    let id: i64 = sqlx::query_scalar(
122
        "INSERT INTO flow_runs (repo_id, number, bookmark, state, heads, pipeline_hash, status, error, actor, trigger, created_ms, finished_ms) \
123
         VALUES (?, (SELECT COALESCE(MAX(number), 0) + 1 FROM flow_runs WHERE repo_id = ?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id",
124
    )
125
    .bind(r.repo_id)
126
    .bind(r.repo_id)
127
    .bind(r.bookmark)
128
    .bind(r.state)
129
    .bind(r.heads)
130
    .bind(r.pipeline_hash)
131
    .bind(status)
132
    .bind(r.error.unwrap_or(""))
133
    .bind(r.actor)
134
    .bind(r.trigger)
135
    .bind(now)
136
    .bind(r.error.map(|_| now))
137
    .fetch_one(&mut *tx)
138
    .await?;
139
    for j in jobs {
140
        sqlx::query(
141
            "INSERT INTO flow_jobs (run_id, name, depth, depends, spec, sandbox, runtime, status) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued')",
142
        )
143
        .bind(id)
144
        .bind(&j.name)
145
        .bind(j.depth)
146
        .bind(serde_json::to_string(&j.depends).unwrap_or_else(|_| "[]".into()))
147
        .bind(&j.spec)
148
        .bind(&j.sandbox)
149
        .bind(&j.runtime)
150
        .execute(&mut *tx)
151
        .await?;
152
    }
153
    tx.commit().await?;
154
    Ok(id)
155
}
156
157
/// A repository's runs, newest first, numbered below `before`.
158
pub async fn runs_by_number(pool: &Pool, repo_id: i64, limit: i64, before: Option<i64>) -> sqlx::Result<Vec<Run>> {
159
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = ? AND number < ? ORDER BY number DESC LIMIT ?")
160
        .bind(repo_id)
161
        .bind(before.unwrap_or(i64::MAX))
162
        .bind(limit)
163
        .fetch_all(pool)
164
        .await
165
}
166
167
pub async fn runs(pool: &Pool, repo_id: i64, limit: i64, before: Option<i64>) -> sqlx::Result<Vec<Run>> {
168
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = ? AND id < ? ORDER BY id DESC LIMIT ?")
169
        .bind(repo_id)
170
        .bind(before.unwrap_or(i64::MAX))
171
        .bind(limit)
172
        .fetch_all(pool)
173
        .await
174
}
175
176
/// A repository's run by its number there.
177
pub async fn run(pool: &Pool, repo_id: i64, number: i64) -> sqlx::Result<Option<Run>> {
178
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = ? AND number = ?").bind(repo_id).bind(number).fetch_optional(pool).await
179
}
180
181
pub async fn run_by_id(pool: &Pool, id: i64) -> sqlx::Result<Option<Run>> {
182
    sqlx::query_as("SELECT * FROM flow_runs WHERE id = ?").bind(id).fetch_optional(pool).await
183
}
184
185
pub async fn jobs(pool: &Pool, run_id: i64) -> sqlx::Result<Vec<Job>> {
186
    sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = ? ORDER BY depth, name").bind(run_id).fetch_all(pool).await
187
}
188
189
pub async fn job(pool: &Pool, id: i64) -> sqlx::Result<Option<Job>> {
190
    sqlx::query_as("SELECT * FROM flow_jobs WHERE id = ?").bind(id).fetch_optional(pool).await
191
}
192
193
/// The running job a job token belongs to, while it may still be used.
194
pub async fn job_by_token(pool: &Pool, token_hash: &str, max_age_ms: i64) -> sqlx::Result<Option<Job>> {
195
    sqlx::query_as("SELECT * FROM flow_jobs WHERE token_hash = ? AND status = 'running' AND started_ms >= ?")
196
        .bind(token_hash)
197
        .bind(now_ms() - max_age_ms)
198
        .fetch_optional(pool)
199
        .await
200
}
201
202
/// Take the oldest queued job whose dependencies all passed and whose tier
203
/// the runner offers, and mark it running for `runner` with a token.
204
/// Let a job's token also read these repositories (its pinned subrepos).
205
pub async fn set_also_read(pool: &Pool, job_id: i64, repo_ids: &[i64]) -> sqlx::Result<()> {
206
    let list: Vec<String> = repo_ids.iter().map(|i| i.to_string()).collect();
207
    sqlx::query("UPDATE flow_jobs SET also_read = ? WHERE id = ?").bind(list.join(",")).bind(job_id).execute(pool).await?;
208
    Ok(())
209
}
210
211
impl Job {
212
    pub fn also_read_ids(&self) -> Vec<i64> {
213
        self.also_read.split(',').filter_map(|x| x.trim().parse().ok()).collect()
214
    }
215
}
216
217
/// What a runner can take on: free memory (MiB) and vCPUs, `None` for no limit.
218
#[derive(Clone, Copy, Debug, Default)]
219
pub struct Room {
220
    pub memory_mib: Option<u32>,
221
    pub cpus: Option<u32>,
222
}
223
224
impl Room {
225
    /// Whether a job's sandbox (its stored JSON) fits.
226
    pub fn fits(&self, sandbox_json: &str) -> bool {
227
        let v: serde_json::Value = serde_json::from_str(sandbox_json).unwrap_or_default();
228
        let need = |k: &str| v[k].as_u64().unwrap_or(0);
229
        self.memory_mib.is_none_or(|m| need("memory_mib") <= m as u64) && self.cpus.is_none_or(|c| need("cpus") <= c as u64)
230
    }
231
}
232
233
pub async fn claim(pool: &Pool, runtimes: &[String], runner: &str, token_hash: &str) -> sqlx::Result<Option<Job>> {
234
    claim_within(pool, runtimes, runner, token_hash, Room::default()).await
235
}
236
237
/// The oldest runnable job that fits in `room`.
238
pub async fn claim_within(pool: &Pool, runtimes: &[String], runner: &str, token_hash: &str, room: Room) -> sqlx::Result<Option<Job>> {
239
    let mut tx = super::begin_write(pool).await?;
240
    let queued: Vec<Job> = sqlx::query_as(
241
        "SELECT j.* FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id WHERE j.status = 'queued' AND r.status IN ('queued', 'running') ORDER BY j.run_id, j.depth, j.id LIMIT 200",
242
    )
243
    .fetch_all(&mut *tx)
244
    .await?;
245
    for j in queued {
246
        if !runtimes.iter().any(|r| r == &j.runtime) || !room.fits(&j.sandbox) {
247
            continue;
248
        }
249
        let deps = j.depends_on();
250
        if !deps.is_empty() {
251
            let mut passed = 0i64;
252
            for d in &deps {
253
                let st: Option<String> = sqlx::query_scalar("SELECT status FROM flow_jobs WHERE run_id = ? AND name = ?")
254
                    .bind(j.run_id)
255
                    .bind(d)
256
                    .fetch_optional(&mut *tx)
257
                    .await?;
258
                if st.as_deref() == Some("passed") {
259
                    passed += 1;
260
                }
261
            }
262
            if passed != deps.len() as i64 {
263
                continue;
264
            }
265
        }
266
        let now = now_ms();
267
        let done = sqlx::query(
268
            "UPDATE flow_jobs SET status = 'running', runner = ?, token_hash = ?, started_ms = ? WHERE id = ? AND status = 'queued'",
269
        )
270
        .bind(runner)
271
        .bind(token_hash)
272
        .bind(now)
273
        .bind(j.id)
274
        .execute(&mut *tx)
275
        .await?;
276
        if done.rows_affected() == 1 {
277
            sqlx::query("UPDATE flow_runs SET status = 'running' WHERE id = ? AND status = 'queued'")
278
                .bind(j.run_id)
279
                .execute(&mut *tx)
280
                .await?;
281
            tx.commit().await?;
282
            return job(pool, j.id).await;
283
        }
284
    }
285
    tx.commit().await?;
286
    Ok(None)
287
}
288
289
/// Append records; returns how many were kept (the rest exceed the cap).
290
pub async fn append_log(pool: &Pool, job_id: i64, records: &[flow_core::api::Record], cap: i64) -> sqlx::Result<usize> {
291
    let mut tx = super::begin_write(pool).await?;
292
    let count: i64 = sqlx::query_scalar("SELECT log_count FROM flow_jobs WHERE id = ?").bind(job_id).fetch_one(&mut *tx).await?;
293
    let room = (cap - count).max(0) as usize;
294
    let keep = &records[..records.len().min(room)];
295
    for (i, r) in keep.iter().enumerate() {
296
        sqlx::query("INSERT INTO flow_logs (job_id, seq, t, stream, text) VALUES (?, ?, ?, ?, ?)")
297
            .bind(job_id)
298
            .bind(count + i as i64 + 1)
299
            .bind(r.t as i64)
300
            .bind(&r.stream)
301
            .bind(&r.text)
302
            .execute(&mut *tx)
303
            .await?;
304
    }
305
    sqlx::query("UPDATE flow_jobs SET log_count = log_count + ? WHERE id = ?")
306
        .bind(keep.len() as i64)
307
        .bind(job_id)
308
        .execute(&mut *tx)
309
        .await?;
310
    tx.commit().await?;
311
    Ok(keep.len())
312
}
313
314
pub async fn log(pool: &Pool, job_id: i64, after: i64, limit: i64) -> sqlx::Result<Vec<LogLine>> {
315
    sqlx::query_as("SELECT seq, t, stream, text FROM flow_logs WHERE job_id = ? AND seq > ? ORDER BY seq LIMIT ?")
316
        .bind(job_id)
317
        .bind(after)
318
        .bind(limit)
319
        .fetch_all(pool)
320
        .await
321
}
322
323
pub struct Outcome<'a> {
324
    pub status: &'a str,
325
    pub exit_code: Option<i64>,
326
    pub error: &'a str,
327
    pub receipt: &'a str,
328
    pub receipt_hash: &'a str,
329
}
330
331
/// Record a job's end, skip the jobs that can no longer run, and settle the
332
/// run. Returns the jobs whose status changed (for their checks).
333
pub async fn finish(pool: &Pool, job_id: i64, o: Outcome<'_>) -> sqlx::Result<Vec<Job>> {
334
    let mut tx = super::begin_write(pool).await?;
335
    let now = now_ms();
336
    let j: Job = sqlx::query_as("SELECT * FROM flow_jobs WHERE id = ?").bind(job_id).fetch_one(&mut *tx).await?;
337
    sqlx::query("UPDATE flow_jobs SET status = ?, exit_code = ?, error = ?, receipt = ?, receipt_hash = ?, finished_ms = ?, token_hash = '' WHERE id = ?")
338
        .bind(o.status)
339
        .bind(o.exit_code)
340
        .bind(o.error)
341
        .bind(o.receipt)
342
        .bind(o.receipt_hash)
343
        .bind(now)
344
        .bind(job_id)
345
        .execute(&mut *tx)
346
        .await?;
347
    let mut changed = vec![job_id];
348
    // Dependents of anything that did not pass are skipped, transitively.
349
    loop {
350
        let all: Vec<Job> = sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = ?").bind(j.run_id).fetch_all(&mut *tx).await?;
351
        let blocked: Vec<i64> = all
352
            .iter()
353
            .filter(|x| x.status == "queued")
354
            .filter(|x| {
355
                x.depends_on()
356
                    .iter()
357
                    .any(|d| all.iter().any(|y| &y.name == d && matches!(y.status.as_str(), "failed" | "skipped" | "error")))
358
            })
359
            .map(|x| x.id)
360
            .collect();
361
        if blocked.is_empty() {
362
            break;
363
        }
364
        for id in blocked {
365
            sqlx::query("UPDATE flow_jobs SET status = 'skipped', finished_ms = ? WHERE id = ?")
366
                .bind(now)
367
                .bind(id)
368
                .execute(&mut *tx)
369
                .await?;
370
            changed.push(id);
371
        }
372
    }
373
    let all: Vec<Job> = sqlx::query_as("SELECT * FROM flow_jobs WHERE run_id = ?").bind(j.run_id).fetch_all(&mut *tx).await?;
374
    if all.iter().all(Job::done) {
375
        let status = if all.iter().all(|x| x.status == "passed") { "passed" } else { "failed" };
376
        sqlx::query("UPDATE flow_runs SET status = ?, finished_ms = ? WHERE id = ?")
377
            .bind(status)
378
            .bind(now)
379
            .bind(j.run_id)
380
            .execute(&mut *tx)
381
            .await?;
382
    }
383
    tx.commit().await?;
384
    let mut out = Vec::new();
385
    for id in changed {
386
        if let Some(x) = job(pool, id).await? {
387
            out.push(x);
388
        }
389
    }
390
    Ok(out)
391
}
392
393
/// The newest run of exactly this state (heads joined with `+`, sorted).
394
pub async fn latest_for_heads(pool: &Pool, repo_id: i64, heads: &str) -> sqlx::Result<Option<Run>> {
395
    sqlx::query_as("SELECT * FROM flow_runs WHERE repo_id = ? AND heads = ? ORDER BY id DESC LIMIT 1")
396
        .bind(repo_id)
397
        .bind(heads)
398
        .fetch_optional(pool)
399
        .await
400
}
401
402
/// The key runner token `token_hash` signs with: the one already bound, or
403
/// `key` (named `name`) when this is the first.
404
pub async fn bind_runner_key(pool: &Pool, token_hash: &str, key: &str, name: &str) -> sqlx::Result<String> {
405
    sqlx::query("INSERT OR IGNORE INTO flow_runner_keys (token_hash, key, name, first_ms) VALUES (?, ?, ?, ?)")
406
        .bind(token_hash)
407
        .bind(key)
408
        .bind(name)
409
        .bind(crate::db::now_ms())
410
        .execute(pool)
411
        .await?;
412
    sqlx::query_scalar("SELECT key FROM flow_runner_keys WHERE token_hash = ?").bind(token_hash).fetch_one(pool).await
413
}
414
415
/// Record what became of job `job_id`'s signed receipt.
416
pub async fn set_receipt(pool: &Pool, job_id: i64, attest: &str, signer: &str, note: &str) -> sqlx::Result<()> {
417
    sqlx::query("UPDATE flow_jobs SET receipt_attest = ?, receipt_signer = ?, receipt_note = ? WHERE id = ?")
418
        .bind(attest)
419
        .bind(signer)
420
        .bind(note)
421
        .bind(job_id)
422
        .execute(pool)
423
        .await?;
424
    Ok(())
425
}