Enforce the Content Security Policy (closes #227); show where a job's time went (refs oxygenflow#35); a Diffusion test waits up to 30 s 0468394734e9
The policy allows Cloudflare's Web Analytics beacon, the only thing a week of reports said it would block. Phases are stored per job (migration 0099), shown on the run page and exported as oxyhub_flow_job_phase_seconds. Pins OxygenFlow main.
- Author
- Andy McCall @andymccall <[email protected]>
- via
- agent
agent:claude@andymccall <[email protected]> · keyless, under your grant for changes,refs,seal · key 6848be981fe9
- When
- · 2026-10-27 03:59 UTC
- Signature
- ◈ sealed by agent:claude@andymccall
- Hash
0468394734e96d5db008f4e36827ed560766ae8742e0e42ed4ac7184f49242e5- Depends on
- 19bb31ff1f5c 380996db84e5 3b1bc27fe805 417184c73f4e 42fa95540468 44a3d2ebaddd 501d8b268dd4 5588bb3631d2 8a48dd830178 951aed50c2f5 9e8cbfc3ad00 a7e6e8e6094c ab4df6aae179 b5668108487f c4bffee5b0d4 d23464c0ac60 d29070faef66 d74ce22fa80e e8fa53694f68
- Depended on by
- 42cd46c9b09e 12325e1b1020 5d66f27cebe2
- In bookmarks
- main
- In tags
- v0.9.11 v0.9.15 v0.9.16 v0.9.19 v0.9.20 v0.9.21 v0.9.22 v0.9.23 v0.9.24 v0.9.9
- Edits
- 27 atoms, 2 tree operations, 11 files
Browse the tree at this change
added crates/hub/migrations-pg/0099_job_phases.sql +4 −0
| 1 | + | -- Where a finished job's time went, as its runner reported (oxygenflow#35): |
|
| 2 | + | -- JSON of phase to milliseconds (image, prepare, run, finish); empty when |
|
| 3 | + | -- the runner predates it. |
|
| 4 | + | ALTER TABLE flow_jobs ADD COLUMN phases TEXT NOT NULL DEFAULT ''; |
added crates/hub/migrations/0099_job_phases.sql +4 −0
| 1 | + | -- Where a finished job's time went, as its runner reported (oxygenflow#35): |
|
| 2 | + | -- JSON of phase to milliseconds (image, prepare, run, finish); empty when |
|
| 3 | + | -- the runner predates it. |
|
| 4 | + | ALTER TABLE flow_jobs ADD COLUMN phases TEXT NOT NULL DEFAULT ''; |
modified CHANGELOG.md +3 −0
| … | 3 unchanged lines |
||
| 4 | 4 | ||
| 5 | 5 | ## Unreleased |
|
| 6 | 6 | ||
| 7 | + | - **The Content Security Policy is enforced (#227).** The pages carry no inline scripts or handlers, and a week of reports showed it would block only Cloudflare's Web Analytics beacon, which the policy now allows (`static.cloudflareinsights.com`, reports to `cloudflareinsights.com`). Violations are still reported and logged. |
|
| 8 | + | - **Where a job's time went (oxygenflow#35).** A runner's finish report carries its phases (image, prepare, run, finish); the run page shows them under each job, and `/metrics` has `oxyhub_flow_job_phase_seconds` by phase. Migration 0099 (`flow_jobs.phases`). |
|
| 9 | + | ||
| 7 | 10 | ## 0.9.8 (2026-10-27) |
|
| 8 | 11 | ||
| 9 | 12 | - **Images only runners hold (#342, oxygenflow#94).** A runner that serves its peers sends the manifest of each image it holds that the hub lacks (`POST /flow/images/held`, `PUT /flow/images/{hash}/held`, runner tokens only); the hub keeps it as held by runners and offers the image's jobs to runners of the same swarm site, which fetch it from their peers (the announce says the hub is no seed for it). A pushed image replaces such a record. On 2026-10-25 a re-pinned image that was never pushed kept every Rust job on one runner for a day. Migration 0098 (`jet_images.source`, `holder`). |
|
| … | 360 unchanged lines |
modified crates/hub/src/db/flow.rs +18 −1
| … | 146 unchanged lines |
||
| 147 | 147 | /// What it took, as its runner reported (oxygenflow#41), in MiB. |
|
| 148 | 148 | pub peak_disk_mib: Option<i64>, |
|
| 149 | 149 | pub peak_memory_mib: Option<i64>, |
|
| 150 | + | /// Where its time went, JSON of phase to milliseconds (oxygenflow#35). |
|
| 151 | + | #[sqlx(default)] |
|
| 152 | + | pub phases: String, |
|
| 150 | 153 | /// The free disk it needs, learned from past runs of the same job. |
|
| 151 | 154 | pub need_disk_mib: Option<i64>, |
|
| 152 | 155 | /// Why it ran on its runner (oxygenflow#42). |
|
| … | 26 unchanged lines |
||
| 179 | 182 | } |
|
| 180 | 183 | ||
| 181 | 184 | impl Job { |
|
| 185 | + | /// Where its time went, as its runner reported (oxygenflow#35): each |
|
| 186 | + | /// phase it took, in the order it took them, with its milliseconds. |
|
| 187 | + | pub fn phase_list(&self) -> Vec<(String, u64)> { |
|
| 188 | + | let m: std::collections::BTreeMap<String, u64> = serde_json::from_str(&self.phases).unwrap_or_default(); |
|
| 189 | + | ["image", "prepare", "run", "finish"].iter().filter_map(|p| m.get(*p).map(|ms| (p.to_string(), *ms))).collect() |
|
| 190 | + | } |
|
| 191 | + | ||
| 182 | 192 | /// The run whose pass this job reused (#265), if it did. |
|
| 183 | 193 | pub fn reused_from(&self) -> Option<i64> { |
|
| 184 | 194 | self.error.strip_prefix("reused from run #")?.parse().ok() |
|
| … | 116 unchanged lines |
||
| 301 | 311 | .bind(job_id) |
|
| 302 | 312 | .execute(pool) |
|
| 303 | 313 | .await?; |
|
| 314 | + | Ok(()) |
|
| 315 | + | } |
|
| 316 | + | ||
| 317 | + | /// Where a finished job's time went, as its runner reported (oxygenflow#35). |
|
| 318 | + | pub async fn set_phases(pool: &Pool, job_id: i64, phases: &std::collections::BTreeMap<String, u64>) -> sqlx::Result<()> { |
|
| 319 | + | let json = if phases.is_empty() { String::new() } else { serde_json::to_string(phases).unwrap_or_default() }; |
|
| 320 | + | sqlx::query("UPDATE flow_jobs SET phases = $1 WHERE id = $2").bind(json).bind(job_id).execute(pool).await?; |
|
| 304 | 321 | Ok(()) |
|
| 305 | 322 | } |
|
| 306 | 323 | ||
| … | 959 unchanged lines |
||
| 1266 | 1283 | "UPDATE flow_jobs SET status = 'queued', runner = '', token_hash = '', started_ms = NULL, finished_ms = NULL, |
|
| 1267 | 1284 | exit_code = NULL, error = '', receipt = '', receipt_hash = '', log_count = 0, also_read = '', receipt_attest = '', |
|
| 1268 | 1285 | receipt_signer = '', receipt_note = '', runner_id = NULL, waiting = '', placed = '', peak_disk_mib = NULL, |
|
| 1269 | - | peak_memory_mib = NULL, approval = CASE WHEN approval = '' THEN '' ELSE 'waiting' END, approved_by = '', |
|
| 1286 | + | peak_memory_mib = NULL, phases = '', approval = CASE WHEN approval = '' THEN '' ELSE 'waiting' END, approved_by = '', |
|
| 1270 | 1287 | approved_ms = NULL, approval_seal = '', approved_via = '', queued_ms = $2, ready_ms = NULL, handed_back_ms = NULL, |
|
| 1271 | 1288 | handbacks = 0 WHERE id = $1", |
|
| 1272 | 1289 | ) |
|
| … | 249 unchanged lines |
modified crates/hub/src/flow.rs +1 −0
| … | 2439 unchanged lines |
||
| 2440 | 2440 | }; |
|
| 2441 | 2441 | let error = f.error.clone().unwrap_or_default(); |
|
| 2442 | 2442 | fdb::set_peaks(&app.db, job.id, f.peak_disk_mib, f.peak_memory_mib).await?; |
|
| 2443 | + | fdb::set_phases(&app.db, job.id, &f.phases).await?; |
|
| 2443 | 2444 | let changed = fdb::finish( |
|
| 2444 | 2445 | &app.db, |
|
| 2445 | 2446 | job.id, |
|
| … | 612 unchanged lines |
modified crates/hub/src/headers.rs +12 −8
| 1 | 1 | //! Security headers on every response (oxygenhub#222), and the endpoint |
|
| 2 | 2 | //! the browser reports Content-Security-Policy violations to. |
|
| 3 | 3 | //! |
|
| 4 | - | //! The policy is sent as `Content-Security-Policy-Report-Only` while the |
|
| 5 | - | //! pages still carry inline scripts and handlers: browsers report what the |
|
| 6 | - | //! policy would block, the hub logs it, and nothing breaks. Switching to |
|
| 7 | - | //! enforcing is one constant ([`ENFORCE`]) once the reports are quiet. |
|
| 4 | + | //! The policy is enforced (#227): the pages carry no inline scripts or |
|
| 5 | + | //! handlers, and in the week of reports before it was switched on the only |
|
| 6 | + | //! thing it would block was Cloudflare's analytics beacon, which the |
|
| 7 | + | //! policy now allows. Violations are still reported and logged; going back |
|
| 8 | + | //! to report-only is one constant ([`ENFORCE`]). |
|
| 8 | 9 | //! Responses that set their own policy (report frames, #127; a job's |
|
| 9 | 10 | //! HTML artifact) keep theirs, and may be framed as they say. |
|
| 10 | 11 | //! Strict-Transport-Security is sent only when `[server] hsts_max_age` is |
|
| … | 8 unchanged lines |
||
| 19 | 20 | pub const REPORT_PATH: &str = "/csp-report"; |
|
| 20 | 21 | ||
| 21 | 22 | /// Enforce the policy rather than only report it. |
|
| 22 | - | const ENFORCE: bool = false; |
|
| 23 | + | const ENFORCE: bool = true; |
|
| 23 | 24 | ||
| 24 | 25 | /// What the pages may load: their own scripts (and WebAssembly, for the |
|
| 25 | 26 | /// backdrop), styles (inline too: style attributes are everywhere), images |
|
| 26 | 27 | /// from anywhere over https (READMEs show badges), and forms posted only to |
|
| 27 | - | /// the hub (sign-in providers are reached by links, not forms). |
|
| 28 | - | const POLICY: &str = "default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; \ |
|
| 29 | - | img-src 'self' data: https:; font-src 'self'; connect-src 'self'; media-src 'self'; object-src 'none'; \ |
|
| 28 | + | /// the hub (sign-in providers are reached by links, not forms). Cloudflare, |
|
| 29 | + | /// in front of oxygenhub.io, adds its Web Analytics beacon to pages: its |
|
| 30 | + | /// script and where it reports are allowed, nothing else of theirs. |
|
| 31 | + | const POLICY: &str = "default-src 'self'; script-src 'self' 'wasm-unsafe-eval' https://static.cloudflareinsights.com; \ |
|
| 32 | + | style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self'; \ |
|
| 33 | + | connect-src 'self' https://cloudflareinsights.com; media-src 'self'; object-src 'none'; \ |
|
| 30 | 34 | base-uri 'self'; form-action 'self'; frame-ancestors 'none'; report-uri /csp-report; report-to csp"; |
|
| 31 | 35 | ||
| 32 | 36 | const PERMISSIONS: &str = "camera=(), microphone=(), geolocation=(), payment=(), usb=(), serial=(), bluetooth=(), \ |
|
| … | 147 unchanged lines |
modified crates/hub/src/metrics.rs +27 −2
| … | 397 unchanged lines |
||
| 398 | 398 | &queue, |
|
| 399 | 399 | ); |
|
| 400 | 400 | histogram(out, "oxyhub_flow_job_run_seconds", "How long finished jobs of the last day ran, by runner.", &run); |
|
| 401 | + | // Where their time went, as their runners reported (oxygenflow#35). |
|
| 402 | + | let phases: Vec<String> = sqlx::query_scalar("SELECT phases FROM flow_jobs WHERE finished_ms >= $1 AND phases != ''") |
|
| 403 | + | .bind(now - 86_400_000) |
|
| 404 | + | .fetch_all(&app.db) |
|
| 405 | + | .await |
|
| 406 | + | .unwrap_or_default(); |
|
| 407 | + | let mut by_phase: std::collections::BTreeMap<String, Vec<f64>> = Default::default(); |
|
| 408 | + | for json in &phases { |
|
| 409 | + | let m: std::collections::BTreeMap<String, u64> = serde_json::from_str(json).unwrap_or_default(); |
|
| 410 | + | for (phase, ms) in m { |
|
| 411 | + | by_phase.entry(phase).or_default().push(ms as f64 / 1000.0); |
|
| 412 | + | } |
|
| 413 | + | } |
|
| 414 | + | histogram_by( |
|
| 415 | + | out, |
|
| 416 | + | "oxyhub_flow_job_phase_seconds", |
|
| 417 | + | "Where finished jobs of the last day spent their time, by phase: image (fetching it), prepare, run, finish.", |
|
| 418 | + | "phase", |
|
| 419 | + | &by_phase, |
|
| 420 | + | ); |
|
| 401 | 421 | family( |
|
| 402 | 422 | out, |
|
| 403 | 423 | "oxyhub_flow_jobs_finished_last_day", |
|
| … | 12 unchanged lines |
||
| 416 | 436 | ||
| 417 | 437 | /// A histogram family from samples (seconds) per runner. |
|
| 418 | 438 | fn histogram(out: &mut String, name: &str, help: &str, by_runner: &std::collections::BTreeMap<String, Vec<f64>>) { |
|
| 439 | + | histogram_by(out, name, help, "runner", by_runner) |
|
| 440 | + | } |
|
| 441 | + | ||
| 442 | + | /// A histogram of job seconds, one series per value of label `key`. |
|
| 443 | + | fn histogram_by(out: &mut String, name: &str, help: &str, key: &str, by: &std::collections::BTreeMap<String, Vec<f64>>) { |
|
| 419 | 444 | let _ = writeln!(out, "# HELP {name} {help}"); |
|
| 420 | 445 | let _ = writeln!(out, "# TYPE {name} histogram"); |
|
| 421 | - | for (runner, samples) in by_runner { |
|
| 422 | - | let l = format!("runner=\"{}\"", label(runner)); |
|
| 446 | + | for (runner, samples) in by { |
|
| 447 | + | let l = format!("{key}=\"{}\"", label(runner)); |
|
| 423 | 448 | for b in JOB_SECONDS { |
|
| 424 | 449 | let n = samples.iter().filter(|s| **s <= b).count(); |
|
| 425 | 450 | let _ = writeln!(out, "{name}_bucket{{{l},le=\"{b}\"}} {n}"); |
|
| … | 56 unchanged lines |
modified crates/hub/tests/accounts.rs +4 −3
| … | 1564 unchanged lines |
||
| 1565 | 1565 | assert_eq!(hd[header::REFERRER_POLICY], "strict-origin-when-cross-origin"); |
|
| 1566 | 1566 | assert_eq!(hd[header::X_FRAME_OPTIONS], "DENY"); |
|
| 1567 | 1567 | assert!(hd["permissions-policy"].to_str().unwrap().contains("camera=()")); |
|
| 1568 | - | // Reported, not yet enforced; and no HSTS until the site owner says. |
|
| 1569 | - | let csp = hd[header::CONTENT_SECURITY_POLICY_REPORT_ONLY].to_str().unwrap(); |
|
| 1568 | + | // Enforced (#227), and still reported; no HSTS until the site owner says. |
|
| 1569 | + | let csp = hd[header::CONTENT_SECURITY_POLICY].to_str().unwrap(); |
|
| 1570 | 1570 | for part in [ |
|
| 1571 | 1571 | "default-src 'self'", |
|
| 1572 | 1572 | "object-src 'none'", |
|
| … | 1 unchanged lines |
||
| 1574 | 1574 | "form-action 'self'", |
|
| 1575 | 1575 | "'wasm-unsafe-eval'", |
|
| 1576 | 1576 | "report-uri /csp-report", |
|
| 1577 | + | "script-src 'self' 'wasm-unsafe-eval' https://static.cloudflareinsights.com", |
|
| 1577 | 1578 | ] { |
|
| 1578 | 1579 | assert!(csp.contains(part), "{part} in {csp}"); |
|
| 1579 | 1580 | } |
|
| 1580 | - | assert!(hd.get(header::CONTENT_SECURITY_POLICY).is_none() && hd.get(header::STRICT_TRANSPORT_SECURITY).is_none()); |
|
| 1581 | + | assert!(hd.get(header::CONTENT_SECURITY_POLICY_REPORT_ONLY).is_none() && hd.get(header::STRICT_TRANSPORT_SECURITY).is_none()); |
|
| 1581 | 1582 | // Static files and errors carry them too. |
|
| 1582 | 1583 | let res = oxygenhub::router(h.app.clone()).oneshot(Request::get("/no/such/page").body(Body::empty()).unwrap()).await.unwrap(); |
|
| 1583 | 1584 | assert_eq!(res.headers()[header::X_CONTENT_TYPE_OPTIONS], "nosniff"); |
|
| … | 216 unchanged lines |
modified crates/hub/tests/flow.rs +12 −2
| … | 1210 unchanged lines |
||
| 1211 | 1211 | "a failing receipt names no outputs" |
|
| 1212 | 1212 | ); |
|
| 1213 | 1213 | r["outputs"] = serde_json::json!([]); |
|
| 1214 | - | assert_eq!(finish(&h.app, id, serde_json::json!({ "exit_code": 1, "duration_ms": 700, "receipt": r })).await, StatusCode::NO_CONTENT); |
|
| 1214 | + | // oxygenflow#35: where its time went, as the runner reports it. |
|
| 1215 | + | let phases = serde_json::json!({ "image": 9000, "prepare": 1200, "run": 3400, "finish": 500 }); |
|
| 1216 | + | assert_eq!( |
|
| 1217 | + | finish(&h.app, id, serde_json::json!({ "exit_code": 1, "duration_ms": 700, "receipt": r, "phases": phases })).await, |
|
| 1218 | + | StatusCode::NO_CONTENT |
|
| 1219 | + | ); |
|
| 1215 | 1220 | let (_, page) = call(&h.app, "GET", &format!("/andy/demo/flow/{}", run.number), None, None).await; |
|
| 1221 | + | let at = page.find("Where its time went: ").expect("the job's phases"); |
|
| 1222 | + | let line = &page[at..at + 120]; |
|
| 1223 | + | let order: Vec<usize> = ["image", "prepare", "run", "finish"].iter().map(|p| line.find(p).expect(p)).collect(); |
|
| 1224 | + | assert!(order.windows(2).all(|w| w[0] < w[1]), "in the order they happened: {line}"); |
|
| 1216 | 1225 | assert!(page.contains(&format!("/andy/demo/flow/{}/artifacts/test/junit", run.number)), "listed"); |
|
| 1217 | 1226 | assert!(page.contains("not in its receipt"), "marked"); |
|
| 1218 | 1227 | assert!(page.contains("2 tests: 1 failed"), "counts beside the job"); |
|
| … | 2839 unchanged lines |
||
| 4058 | 4067 | assert!(!bump().await.unwrap().body.contains("api v4"), "not before its run passes"); |
|
| 4059 | 4068 | pass_all().await; |
|
| 4060 | 4069 | let mut updated = None; |
|
| 4061 | - | for _ in 0..100 { |
|
| 4070 | + | // Up to 30 s: under the whole Postgres suite 5 s was not always enough. |
|
| 4071 | + | for _ in 0..600 { |
|
| 4062 | 4072 | let p2 = bump().await.unwrap(); |
|
| 4063 | 4073 | if p2.body.contains("api v4") { |
|
| 4064 | 4074 | updated = Some(p2); |
|
| … | 2910 unchanged lines |
replaced deps/oxygenflow +0 −0
No line changes.
modified templates/flow_run.html +1 −0
| … | 62 unchanged lines |
||
| 63 | 63 | <pre class="flow-log" data-log data-seq="{{ l.last_seq }}">{% for line in l.lines %}<span class="ln s-{{ line.stream }}"><span class="t">{{ line.t / 1000 }}.{{ "{:01}"|format(line.t % 1000 / 100) }}s</span>{{ line.text }}</span> |
|
| 64 | 64 | {% endfor %}</pre> |
|
| 65 | 65 | {% if !l.j.placed.is_empty() %}<p class="small mute pad">Ran on {{ l.j.runner }}: {{ l.j.placed }}.</p>{% endif %} |
|
| 66 | + | {% let phases = l.j.phase_list() %}{% if !phases.is_empty() %}<p class="small mute pad">Where its time went: {% for (p, ms) in phases %}{{ p }} {{ crate::web::flow::secs(ms.clone() as i64) }}{% if !loop.last %}, {% endif %}{% endfor %}.</p>{% endif %} |
|
| 66 | 67 | {% if l.j.status == "passed" %}{% if let Some(n) = l.j.reused_from() %}<p class="small mute pad" data-reused>Not run again: this job, with the same spec and sandbox, passed on this very state in <a href="{{ repo.url }}/flow/{{ n }}">run #{{ n }}</a>, whose log it would repeat; that pass's receipt stands for it here.</p>{% endif %}{% endif %} |
|
| 67 | 68 | {% if l.j.status == "queued" && !l.j.waiting.is_empty() %}<p class="small warn pad" data-waiting data-queued-note>Waiting: {{ l.j.waiting }}.</p>{% else if l.lines.is_empty() && l.j.status == "queued" %}<p class="small mute pad" data-queued-note>Waiting for a runner{% if !l.j.depends_on().is_empty() %} and for {{ l.j.depends_on().join(", ") }}{% endif %}.</p>{% endif %} |
|
| 68 | 69 | {% if l.lines.is_empty() && l.j.status == "skipped" %}<p class="small mute pad">Skipped: a job it depends on did not pass.</p>{% endif %} |
|
| … | 4 unchanged lines |