Write transactions begin IMMEDIATE, so a concurrent writer is waited for (busy timeout) instead of failing at once with database is locked; a test of 16 concurrent log appends fails without it 941abdd6bb4c
closes #21
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-28 20:56 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
941abdd6bb4c4c2de60cefa9d636544366d4333ea220564292247fa1575b39e8- Depends on
- 19bb31ff1f5c 42fa95540468 62d6e969f180 76bc5a058351 d23464c0ac60 de505df5cf8d e1ec9fbf538e fd3db644df22
- Depended on by
- 107eaba52c9d 3b1bc27fe805 d1bf2d698777 5d90e0e045e2 501d8b268dd4 74be97962ed8 9008bf6c2b8c
- In bookmarks
- api-unknown-fields cancel-landed-runs clear-waiting close-proposal-140 diffusion-pin-bookmark draft-ready image-check job-waits main oh-comment-body-file proposal-list-fast run-one-job wiki-says-so
- In tags
- v0.1.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.5.0 v0.5.1 v0.5.2 v0.6.0 v0.6.1 v0.7.0 v0.8.0 v0.9.0 v0.9.1 v0.9.11 v0.9.15 v0.9.16 v0.9.19 v0.9.2 v0.9.20 v0.9.21 v0.9.22 v0.9.23 v0.9.24 v0.9.3 v0.9.4 v0.9.5 v0.9.6 v0.9.7 v0.9.8 v0.9.9
- Edits
- 16 atoms, 0 tree operations, 7 files
Browse the tree at this change
modified crates/hub/src/db/accounts.rs +1 −1
| … | 251 unchanged lines |
||
| 252 | 252 | } |
|
| 253 | 253 | ||
| 254 | 254 | pub async fn create_org(pool: &Pool, name: &str, display_name: &str, owner: i64) -> sqlx::Result<Organisation> { |
|
| 255 | - | let mut tx = pool.begin().await?; |
|
| 255 | + | let mut tx = super::begin_write(pool).await?; |
|
| 256 | 256 | let org: Organisation = sqlx::query_as("INSERT INTO organisations (name, display_name, created_ms) VALUES (?, ?, ?) RETURNING *") |
|
| 257 | 257 | .bind(name) |
|
| 258 | 258 | .bind(display_name) |
|
| … | 467 unchanged lines |
modified crates/hub/src/db/flow.rs +4 −4
| … | 91 unchanged lines |
||
| 92 | 92 | } |
|
| 93 | 93 | ||
| 94 | 94 | pub async fn create_run(pool: &Pool, r: NewRun<'_>, jobs: &[NewJob]) -> sqlx::Result<i64> { |
|
| 95 | - | let mut tx = pool.begin().await?; |
|
| 95 | + | let mut tx = super::begin_write(pool).await?; |
|
| 96 | 96 | let now = now_ms(); |
|
| 97 | 97 | let status = if r.error.is_some() { "error" } else { "queued" }; |
|
| 98 | 98 | let id: i64 = sqlx::query_scalar( |
|
| … | 100 unchanged lines |
||
| 199 | 199 | ||
| 200 | 200 | /// The oldest runnable job that fits in `room`. |
|
| 201 | 201 | pub async fn claim_within(pool: &Pool, runtimes: &[String], runner: &str, token_hash: &str, room: Room) -> sqlx::Result<Option<Job>> { |
|
| 202 | - | let mut tx = pool.begin().await?; |
|
| 202 | + | let mut tx = super::begin_write(pool).await?; |
|
| 203 | 203 | let queued: Vec<Job> = sqlx::query_as( |
|
| 204 | 204 | "SELECT j.* FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id WHERE j.status = 'queued' AND r.status IN ('queued', 'running') ORDER BY j.run_id, j.depth, j.id LIMIT 200", |
|
| 205 | 205 | ) |
|
| … | 45 unchanged lines |
||
| 251 | 251 | ||
| 252 | 252 | /// Append records; returns how many were kept (the rest exceed the cap). |
|
| 253 | 253 | pub async fn append_log(pool: &Pool, job_id: i64, records: &[flow_core::api::Record], cap: i64) -> sqlx::Result<usize> { |
|
| 254 | - | let mut tx = pool.begin().await?; |
|
| 254 | + | let mut tx = super::begin_write(pool).await?; |
|
| 255 | 255 | let count: i64 = sqlx::query_scalar("SELECT log_count FROM flow_jobs WHERE id = ?").bind(job_id).fetch_one(&mut *tx).await?; |
|
| 256 | 256 | let room = (cap - count).max(0) as usize; |
|
| 257 | 257 | let keep = &records[..records.len().min(room)]; |
|
| … | 36 unchanged lines |
||
| 294 | 294 | /// Record a job's end, skip the jobs that can no longer run, and settle the |
|
| 295 | 295 | /// run. Returns the jobs whose status changed (for their checks). |
|
| 296 | 296 | pub async fn finish(pool: &Pool, job_id: i64, o: Outcome<'_>) -> sqlx::Result<Vec<Job>> { |
|
| 297 | - | let mut tx = pool.begin().await?; |
|
| 297 | + | let mut tx = super::begin_write(pool).await?; |
|
| 298 | 298 | let now = now_ms(); |
|
| 299 | 299 | let j: Job = sqlx::query_as("SELECT * FROM flow_jobs WHERE id = ?").bind(job_id).fetch_one(&mut *tx).await?; |
|
| 300 | 300 | sqlx::query("UPDATE flow_jobs SET status = ?, exit_code = ?, error = ?, receipt = ?, receipt_hash = ?, finished_ms = ?, token_hash = '' WHERE id = ?") |
|
| … | 54 unchanged lines |
modified crates/hub/src/db/mod.rs +8 −0
| … | 35 unchanged lines |
||
| 36 | 36 | pub fn now_ms() -> i64 { |
|
| 37 | 37 | std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as i64).unwrap_or(0) |
|
| 38 | 38 | } |
|
| 39 | + | ||
| 40 | + | /// A write transaction that takes SQLite's write lock at `BEGIN` |
|
| 41 | + | /// (`BEGIN IMMEDIATE`). A deferred transaction that reads and then writes |
|
| 42 | + | /// fails at once with "database is locked" when another writer holds the |
|
| 43 | + | /// lock, whatever the busy timeout; an immediate one waits its turn. |
|
| 44 | + | pub async fn begin_write(pool: &Pool) -> sqlx::Result<sqlx::Transaction<'static, sqlx::Sqlite>> { |
|
| 45 | + | pool.begin_with("BEGIN IMMEDIATE").await |
|
| 46 | + | } |
modified crates/hub/src/db/projects.rs +2 −2
| … | 40 unchanged lines |
||
| 41 | 41 | pub const DEFAULT_COLUMNS: [&str; 3] = ["Backlog", "Doing", "Done"]; |
|
| 42 | 42 | ||
| 43 | 43 | pub async fn create(pool: &Pool, owner: &str, title: &str, description: &str, public: bool) -> sqlx::Result<Project> { |
|
| 44 | - | let mut tx = pool.begin().await?; |
|
| 44 | + | let mut tx = super::begin_write(pool).await?; |
|
| 45 | 45 | let number: i64 = |
|
| 46 | 46 | sqlx::query_scalar("SELECT COALESCE(MAX(number), 0) + 1 FROM projects WHERE owner = ?").bind(owner).fetch_one(&mut *tx).await?; |
|
| 47 | 47 | let now = now_ms(); |
|
| … | 107 unchanged lines |
||
| 155 | 155 | if !cols.iter().any(|c| c.id == column_id) { |
|
| 156 | 156 | return Ok(false); |
|
| 157 | 157 | } |
|
| 158 | - | let mut tx = pool.begin().await?; |
|
| 158 | + | let mut tx = super::begin_write(pool).await?; |
|
| 159 | 159 | let base: i64 = sqlx::query_scalar("SELECT COALESCE(MAX(position), -1) + 1 FROM project_cards WHERE column_id = ?") |
|
| 160 | 160 | .bind(to.id) |
|
| 161 | 161 | .fetch_one(&mut *tx) |
|
| … | 120 unchanged lines |
modified crates/hub/src/db/proposals.rs +1 −1
| … | 82 unchanged lines |
||
| 83 | 83 | ||
| 84 | 84 | /// Take the next number of a repository's shared number space. |
|
| 85 | 85 | pub async fn next_number(pool: &Pool, repo_id: i64) -> sqlx::Result<i64> { |
|
| 86 | - | let mut tx = pool.begin().await?; |
|
| 86 | + | let mut tx = super::begin_write(pool).await?; |
|
| 87 | 87 | sqlx::query("INSERT OR IGNORE INTO numbers (repo_id, next) VALUES (?, 1)").bind(repo_id).execute(&mut *tx).await?; |
|
| 88 | 88 | let n: i64 = sqlx::query_scalar("SELECT next FROM numbers WHERE repo_id = ?").bind(repo_id).fetch_one(&mut *tx).await?; |
|
| 89 | 89 | sqlx::query("UPDATE numbers SET next = next + 1 WHERE repo_id = ?").bind(repo_id).execute(&mut *tx).await?; |
|
| … | 209 unchanged lines |
modified crates/hub/src/db/repos.rs +1 −1
| … | 134 unchanged lines |
||
| 135 | 135 | /// Record a push's per-reference results and bump the repository's activity. |
|
| 136 | 136 | pub async fn record_push(pool: &Pool, id: i64, pusher: Option<(i64, &str)>, results: &[PushOutcome]) -> sqlx::Result<()> { |
|
| 137 | 137 | let now = now_ms(); |
|
| 138 | - | let mut tx = pool.begin().await?; |
|
| 138 | + | let mut tx = super::begin_write(pool).await?; |
|
| 139 | 139 | for r in results { |
|
| 140 | 140 | sqlx::query("INSERT INTO pushes (repo_id, at_ms, reference, ok, reason, old_state, new_state, heads, changes, account_id, pusher) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") |
|
| 141 | 141 | .bind(id) |
|
| … | 123 unchanged lines |
modified crates/hub/tests/flow.rs +33 −0
| … | 409 unchanged lines |
||
| 410 | 410 | assert!(!read("vault"), "not pinned for this pusher"); |
|
| 411 | 411 | assert!(!read("other"), "public, but not the job's to read"); |
|
| 412 | 412 | } |
|
| 413 | + | ||
| 414 | + | #[tokio::test(flavor = "multi_thread", worker_threads = 8)] |
|
| 415 | + | async fn concurrent_log_appends_never_fail_on_a_locked_database() { |
|
| 416 | + | let h = hub().await; |
|
| 417 | + | create(&h.app, "andy", "demo").await; |
|
| 418 | + | let work = tempfile::tempdir().unwrap(); |
|
| 419 | + | let c = client(&format!("{}/andy/demo", h.base), work.path()); |
|
| 420 | + | write(work.path(), ".flow/pipeline.kdl", &pipeline("")); |
|
| 421 | + | write(work.path(), "Makefile", "all:\n"); |
|
| 422 | + | c.snapshot("t").unwrap(); |
|
| 423 | + | c.commit("pipeline", &[]).unwrap(); |
|
| 424 | + | assert!(c.push("origin", false).unwrap().iter().all(|r| r.ok)); |
|
| 425 | + | let run = runs_of(&h.app, "andy", "demo").await.remove(0); |
|
| 426 | + | let job = db::flow::jobs(&h.app.db, run.id).await.unwrap().remove(0); |
|
| 427 | + | // Many writers at once, each reading the count and then inserting: a |
|
| 428 | + | // deferred transaction would fail with "database is locked". |
|
| 429 | + | let mut tasks = Vec::new(); |
|
| 430 | + | for w in 0..16 { |
|
| 431 | + | let pool = h.app.db.clone(); |
|
| 432 | + | tasks.push(tokio::spawn(async move { |
|
| 433 | + | let records: Vec<flow_core::api::Record> = |
|
| 434 | + | (0..20).map(|i| flow_core::api::Record { t: i, stream: "out".into(), text: format!("writer {w} line {i}") }).collect(); |
|
| 435 | + | db::flow::append_log(&pool, job.id, &records, 100_000).await |
|
| 436 | + | })); |
|
| 437 | + | } |
|
| 438 | + | for t in tasks { |
|
| 439 | + | assert_eq!(t.await.unwrap().unwrap(), 20, "every append succeeds"); |
|
| 440 | + | } |
|
| 441 | + | let lines = db::flow::log(&h.app.db, job.id, 0, 1_000).await.unwrap(); |
|
| 442 | + | assert_eq!(lines.len(), 320); |
|
| 443 | + | let seqs: std::collections::BTreeSet<i64> = lines.iter().map(|l| l.seq).collect(); |
|
| 444 | + | assert_eq!(seqs.len(), 320, "sequence numbers are unique"); |
|
| 445 | + | } |