Scheduled runs: on { schedule } jobs start on the default bookmark when due 348f8c4baccf
A ticker checks each minute, on every repository that runs Flow, which schedule lines of the default bookmark's pipeline are due (UTC) and starts a run of those jobs (trigger schedule). Test. closes oxygenflow#24
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-10-02 22:08 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
348f8c4baccfc67050ed97ac93d9506d965e97e6e8b85543365ea1a4793848a7- Depends on
- 070a7b64b368 18e381ea88c2 1edac4b68c6d 380996db84e5 42fa95540468 4f6925337848 749d9a5fcd9f b5668108487f c889911bec12 d23464c0ac60 d7ce41acf22c eef677c48cd1
- Depended on by
- 5f3fa151688e e57765dee61a 0650d94ede82 4d2e44d79b03 a6c241e8eba0 8a3334b7a364
- In bookmarks
- api-unknown-fields cancel-landed-runs clear-waiting close-proposal-140 diffusion-pin-bookmark draft-ready image-check job-waits main oh-comment-body-file proposal-list-fast run-one-job wiki-says-so
- In tags
- v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.5.0 v0.5.1 v0.5.2 v0.6.0 v0.6.1 v0.7.0 v0.8.0 v0.9.0 v0.9.1 v0.9.11 v0.9.15 v0.9.16 v0.9.19 v0.9.2 v0.9.20 v0.9.21 v0.9.22 v0.9.23 v0.9.24 v0.9.3 v0.9.4 v0.9.5 v0.9.6 v0.9.7 v0.9.8 v0.9.9
- Edits
- 11 atoms, 0 tree operations, 5 files
Browse the tree at this change
modified CHANGELOG.md +1 −1
| … | 3 unchanged lines |
||
| 4 | 4 | ||
| 5 | 5 | ## Unreleased |
|
| 6 | 6 | ||
| 7 | - | - **What starts a job: `on` (oxygenflow#24).** A pipeline's or a job's `on { push "main" "feature/*"; tag "v*"; manual }` decides which events run it: pushing a tag now runs the jobs that name it, on the tagged state; a run has only the jobs its event starts (and what they need), and an event that starts none starts no run. A rerun repeats its run's event. A bookmark sent again unchanged starts no run. |
|
| 7 | + | - **What starts a job: `on` (oxygenflow#24).** A pipeline's or a job's `on { push "main" "feature/*"; tag "v*"; manual }` decides which events run it: pushing a tag now runs the jobs that name it, on the tagged state; a run has only the jobs its event starts (and what they need), and an event that starts none starts no run. A rerun repeats its run's event. A bookmark sent again unchanged starts no run. `schedule "0 3 * * *"` lines run their jobs on the default bookmark when due (UTC), checked each minute. |
|
| 8 | 8 | ||
| 9 | 9 | ## 0.3.1 (2026-10-02) |
|
| 10 | 10 | ||
| … | 56 unchanged lines |
modified crates/hub/src/flow.rs +78 −1
| … | 237 unchanged lines |
||
| 238 | 238 | trigger: &str, |
|
| 239 | 239 | touched: Option<&[String]>, |
|
| 240 | 240 | ) -> Result<Option<i64>, String> { |
|
| 241 | + | start_run_at(app, row, bookmark, state, heads, actor, trigger, touched, None).await |
|
| 242 | + | } |
|
| 243 | + | ||
| 244 | + | /// [`start_run_with`], for a scheduled run due at `at` on the hub's clock. |
|
| 245 | + | #[allow(clippy::too_many_arguments)] |
|
| 246 | + | async fn start_run_at( |
|
| 247 | + | app: &Shared, |
|
| 248 | + | row: &db::repos::RepoRow, |
|
| 249 | + | bookmark: &str, |
|
| 250 | + | state: &str, |
|
| 251 | + | heads: &[ChangeHash], |
|
| 252 | + | actor: &str, |
|
| 253 | + | trigger: &str, |
|
| 254 | + | touched: Option<&[String]>, |
|
| 255 | + | at: Option<flow_core::pipeline::When>, |
|
| 256 | + | ) -> Result<Option<i64>, String> { |
|
| 241 | 257 | let repo = RepoName::new(&row.owner, &row.name).ok_or("bad repository name")?; |
|
| 242 | 258 | let h = heads.to_vec(); |
|
| 243 | 259 | let text = app |
|
| … | 23 unchanged lines |
||
| 267 | 283 | // needs them. |
|
| 268 | 284 | // The jobs this run's event starts (`on`), out of those its |
|
| 269 | 285 | // push touched (`scope`); then what they depend on. |
|
| 270 | - | let event = event_of(trigger, bookmark); |
|
| 286 | + | let event = match at { |
|
| 287 | + | Some(w) => flow_core::pipeline::Event::ScheduleAt(w), |
|
| 288 | + | None => event_of(trigger, bookmark), |
|
| 289 | + | }; |
|
| 271 | 290 | let mut keep: std::collections::HashSet<String> = p |
|
| 272 | 291 | .order() |
|
| 273 | 292 | .into_iter() |
|
| … | 478 unchanged lines |
||
| 752 | 771 | report_changed(app, job.run_id, changed).await?; |
|
| 753 | 772 | } |
|
| 754 | 773 | Ok(()) |
|
| 774 | + | } |
|
| 775 | + | ||
| 776 | + | /// Start the scheduled runs due at `when` (oxygenflow#24): on each |
|
| 777 | + | /// repository that runs Flow, the jobs of its default bookmark's pipeline |
|
| 778 | + | /// with a `schedule` line due then. Returns how many runs started. |
|
| 779 | + | pub async fn run_due(app: &Shared, when: flow_core::pipeline::When) -> usize { |
|
| 780 | + | let Ok(rows) = db::repos::list(&app.db).await else { return 0 }; |
|
| 781 | + | let mut started = 0; |
|
| 782 | + | for row in rows.into_iter().filter(|r| enabled(app, r)) { |
|
| 783 | + | let Some(repo) = RepoName::new(&row.owner, &row.name) else { continue }; |
|
| 784 | + | let b = row.default_bookmark.clone(); |
|
| 785 | + | let found = app |
|
| 786 | + | .repos |
|
| 787 | + | .read(&repo, move |r, _| { |
|
| 788 | + | let Some(st) = r.view()?.bookmarks.get(&b).copied() else { return Ok(None) }; |
|
| 789 | + | Ok(Some((st, r.store().get_state(&st)?.heads))) |
|
| 790 | + | }) |
|
| 791 | + | .await; |
|
| 792 | + | let Ok(Some((state, heads))) = found else { continue }; |
|
| 793 | + | if heads.is_empty() { |
|
| 794 | + | continue; |
|
| 795 | + | } |
|
| 796 | + | match start_run_at(app, &row, &row.default_bookmark, &state.to_hex(), &heads, &row.owner, "schedule", None, Some(when)).await { |
|
| 797 | + | Ok(Some(_)) => started += 1, |
|
| 798 | + | Ok(None) => {} |
|
| 799 | + | Err(e) => tracing::warn!("flow: scheduled run of {}/{}: {e}", row.owner, row.name), |
|
| 800 | + | } |
|
| 801 | + | } |
|
| 802 | + | started |
|
| 803 | + | } |
|
| 804 | + | ||
| 805 | + | /// The hub's clock as a [`When`](flow_core::pipeline::When). |
|
| 806 | + | pub fn when_now() -> flow_core::pipeline::When { |
|
| 807 | + | let now = time::OffsetDateTime::now_utc(); |
|
| 808 | + | flow_core::pipeline::When { |
|
| 809 | + | minute: now.minute() as u32, |
|
| 810 | + | hour: now.hour() as u32, |
|
| 811 | + | day: now.day() as u32, |
|
| 812 | + | month: u8::from(now.month()) as u32, |
|
| 813 | + | weekday: now.weekday().number_days_from_sunday() as u32, |
|
| 814 | + | } |
|
| 815 | + | } |
|
| 816 | + | ||
| 817 | + | /// Start the scheduled runs due each minute, once a minute. |
|
| 818 | + | pub async fn scheduler(app: Shared) { |
|
| 819 | + | let mut last = None; |
|
| 820 | + | loop { |
|
| 821 | + | tokio::time::sleep(std::time::Duration::from_secs(15)).await; |
|
| 822 | + | let minute = time::OffsetDateTime::now_utc().unix_timestamp() / 60; |
|
| 823 | + | if last == Some(minute) { |
|
| 824 | + | continue; |
|
| 825 | + | } |
|
| 826 | + | last = Some(minute); |
|
| 827 | + | let n = run_due(&app, when_now()).await; |
|
| 828 | + | if n > 0 { |
|
| 829 | + | tracing::info!("flow: {n} scheduled run(s) started"); |
|
| 830 | + | } |
|
| 831 | + | } |
|
| 755 | 832 | } |
|
| 756 | 833 | ||
| 757 | 834 | /// Look for lost jobs once a minute. |
|
| … | 178 unchanged lines |
modified crates/hub/src/lib.rs +1 −0
| … | 65 unchanged lines |
||
| 66 | 66 | *slot.lock().unwrap() = Some(Arc::downgrade(&app)); |
|
| 67 | 67 | tokio::spawn(hooks::worker(app.clone())); |
|
| 68 | 68 | tokio::spawn(flow::reaper(app.clone())); |
|
| 69 | + | tokio::spawn(flow::scheduler(app.clone())); |
|
| 69 | 70 | Ok(app) |
|
| 70 | 71 | } |
|
| 71 | 72 | ||
| … | 41 unchanged lines |
modified crates/hub/tests/flow.rs +37 −1
| … | 646 unchanged lines |
||
| 647 | 647 | let (st, body) = call(&h.app, "POST", "/api/v1/repos/andy/demo/flow/runs/1/rerun", Some("site"), None).await; |
|
| 648 | 648 | assert_eq!(st, StatusCode::CREATED, "{body}"); |
|
| 649 | 649 | let third = db::flow::run(&h.app.db, row.id, 4).await.unwrap().unwrap(); |
|
| 650 | - | assert_eq!((third.trigger_label().as_str(), third.status.as_str(), third.state.as_str()), ("rerun:1", "queued", first[0].state.as_str())); |
|
| 650 | + | assert_eq!( |
|
| 651 | + | (third.trigger_label().as_str(), third.status.as_str(), third.state.as_str()), |
|
| 652 | + | ("rerun:1", "queued", first[0].state.as_str()) |
|
| 653 | + | ); |
|
| 651 | 654 | assert_eq!(third.trigger, "rerun:1:push", "a rerun keeps the event it repeats"); |
|
| 652 | 655 | let checks = db::hooks::checks_for_state(&h.app.db, row.id, &third.state).await.unwrap(); |
|
| 653 | 656 | assert!( |
|
| … | 940 unchanged lines |
||
| 1594 | 1597 | assert!(c.push_bookmark("origin", "develop", false).unwrap().iter().all(|r| r.ok)); |
|
| 1595 | 1598 | tokio::time::sleep(std::time::Duration::from_millis(300)).await; |
|
| 1596 | 1599 | assert_eq!(runs_of(&h.app, "andy", "events").await.len(), 3, "no run for develop"); |
|
| 1600 | + | } |
|
| 1601 | + | ||
| 1602 | + | /// oxygenflow#24: a job with a `schedule` line runs on the default |
|
| 1603 | + | /// bookmark when the line comes due, and only then; other jobs stay out. |
|
| 1604 | + | #[tokio::test(flavor = "multi_thread")] |
|
| 1605 | + | async fn scheduled_jobs_run_when_due() { |
|
| 1606 | + | let h = hub().await; |
|
| 1607 | + | db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", true).await.unwrap(); |
|
| 1608 | + | create(&h.app, "andy", "nightly").await; |
|
| 1609 | + | let url = format!("{}/andy/nightly", h.base); |
|
| 1610 | + | let work = tempfile::tempdir().unwrap(); |
|
| 1611 | + | let c = client(&url, work.path()); |
|
| 1612 | + | // An odd minute, so the hub's own ticker cannot fire it during the test. |
|
| 1613 | + | let extra = "job \"nightly\" {\n sandbox \"alpine\"\n on {\n schedule \"17 4 29 2 *\"\n }\n exec \"true\"\n}\n"; |
|
| 1614 | + | write(work.path(), ".flow/pipeline.kdl", &pipeline(extra)); |
|
| 1615 | + | write(work.path(), "test.sh", "echo ok\n"); |
|
| 1616 | + | c.commit("Start", &[]).unwrap(); |
|
| 1617 | + | assert!(c.push("origin", false).unwrap().iter().all(|r| r.ok)); |
|
| 1618 | + | assert_eq!(runs_of(&h.app, "andy", "nightly").await.len(), 1); |
|
| 1619 | + | let first = runs_of(&h.app, "andy", "nightly").await; |
|
| 1620 | + | let names: Vec<String> = db::flow::jobs(&h.app.db, first[0].id).await.unwrap().into_iter().map(|j| j.name).collect(); |
|
| 1621 | + | assert!(!names.contains(&"nightly".to_string()), "a push does not run a scheduled job"); |
|
| 1622 | + | use flow_core::pipeline::When; |
|
| 1623 | + | let not_yet = When { minute: 16, hour: 4, day: 29, month: 2, weekday: 4 }; |
|
| 1624 | + | assert_eq!(oxygenhub::flow::run_due(&h.app, not_yet).await, 0); |
|
| 1625 | + | let due = When { minute: 17, ..not_yet }; |
|
| 1626 | + | assert_eq!(oxygenhub::flow::run_due(&h.app, due).await, 1); |
|
| 1627 | + | let runs = runs_of(&h.app, "andy", "nightly").await; |
|
| 1628 | + | assert_eq!(runs.len(), 2); |
|
| 1629 | + | assert_eq!(runs[0].trigger, "schedule"); |
|
| 1630 | + | let names: Vec<String> = db::flow::jobs(&h.app.db, runs[0].id).await.unwrap().into_iter().map(|j| j.name).collect(); |
|
| 1631 | + | assert_eq!(names, ["nightly"]); |
|
| 1632 | + | assert_eq!(runs[0].bookmark, "main", "on the default bookmark"); |
|
| 1597 | 1633 | } |
replaced deps/oxygenflow +0 −0
No line changes.