OXYGENHUB

Enforce the Content Security Policy (closes #227); show where a job's time went (refs oxygenflow#35); a Diffusion test waits up to 30 s 0468394734e9

The policy allows Cloudflare's Web Analytics beacon, the only thing a week of reports said it would block. Phases are stored per job (migration 0099), shown on the run page and exported as oxyhub_flow_job_phase_seconds. Pins OxygenFlow main.
Author
Andy McCall @andymccall <[email protected]>
via
agent agent:claude@andymccall <[email protected]> · keyless, under your grant for changes,refs,seal · key 6848be981fe9
When
· 2026-10-27 03:59 UTC
Signature
◈ sealed by agent:claude@andymccall
Hash
0468394734e96d5db008f4e36827ed560766ae8742e0e42ed4ac7184f49242e5
Depends on
19bb31ff1f5c 380996db84e5 3b1bc27fe805 417184c73f4e 42fa95540468 44a3d2ebaddd 501d8b268dd4 5588bb3631d2 8a48dd830178 951aed50c2f5 9e8cbfc3ad00 a7e6e8e6094c ab4df6aae179 b5668108487f c4bffee5b0d4 d23464c0ac60 d29070faef66 d74ce22fa80e e8fa53694f68
Depended on by
42cd46c9b09e 12325e1b1020 5d66f27cebe2
In bookmarks
main
In tags
v0.9.11 v0.9.15 v0.9.16 v0.9.19 v0.9.20 v0.9.21 v0.9.22 v0.9.23 v0.9.24 v0.9.9
Edits
27 atoms, 2 tree operations, 11 files

Browse the tree at this change

added crates/hub/migrations-pg/0099_job_phases.sql +4 −0
1 +
-- Where a finished job's time went, as its runner reported (oxygenflow#35):
2 +
-- JSON of phase to milliseconds (image, prepare, run, finish); empty when
3 +
-- the runner predates it.
4 +
ALTER TABLE flow_jobs ADD COLUMN phases TEXT NOT NULL DEFAULT '';
added crates/hub/migrations/0099_job_phases.sql +4 −0
1 +
-- Where a finished job's time went, as its runner reported (oxygenflow#35):
2 +
-- JSON of phase to milliseconds (image, prepare, run, finish); empty when
3 +
-- the runner predates it.
4 +
ALTER TABLE flow_jobs ADD COLUMN phases TEXT NOT NULL DEFAULT '';
modified CHANGELOG.md +3 −0
…
3 unchanged lines
4 4
5 5
## Unreleased
6 6
7 +
- **The Content Security Policy is enforced (#227).** The pages carry no inline scripts or handlers, and a week of reports showed it would block only Cloudflare's Web Analytics beacon, which the policy now allows (`static.cloudflareinsights.com`, reports to `cloudflareinsights.com`). Violations are still reported and logged.
8 +
- **Where a job's time went (oxygenflow#35).** A runner's finish report carries its phases (image, prepare, run, finish); the run page shows them under each job, and `/metrics` has `oxyhub_flow_job_phase_seconds` by phase. Migration 0099 (`flow_jobs.phases`).
9 +
7 10
## 0.9.8 (2026-10-27)
8 11
9 12
- **Images only runners hold (#342, oxygenflow#94).** A runner that serves its peers sends the manifest of each image it holds that the hub lacks (`POST /flow/images/held`, `PUT /flow/images/{hash}/held`, runner tokens only); the hub keeps it as held by runners and offers the image's jobs to runners of the same swarm site, which fetch it from their peers (the announce says the hub is no seed for it). A pushed image replaces such a record. On 2026-10-25 a re-pinned image that was never pushed kept every Rust job on one runner for a day. Migration 0098 (`jet_images.source`, `holder`).
…
360 unchanged lines
modified crates/hub/src/db/flow.rs +18 −1
…
146 unchanged lines
147 147
    /// What it took, as its runner reported (oxygenflow#41), in MiB.
148 148
    pub peak_disk_mib: Option<i64>,
149 149
    pub peak_memory_mib: Option<i64>,
150 +
    /// Where its time went, JSON of phase to milliseconds (oxygenflow#35).
151 +
    #[sqlx(default)]
152 +
    pub phases: String,
150 153
    /// The free disk it needs, learned from past runs of the same job.
151 154
    pub need_disk_mib: Option<i64>,
152 155
    /// Why it ran on its runner (oxygenflow#42).
…
26 unchanged lines
179 182
}
180 183
181 184
impl Job {
185 +
    /// Where its time went, as its runner reported (oxygenflow#35): each
186 +
    /// phase it took, in the order it took them, with its milliseconds.
187 +
    pub fn phase_list(&self) -> Vec<(String, u64)> {
188 +
        let m: std::collections::BTreeMap<String, u64> = serde_json::from_str(&self.phases).unwrap_or_default();
189 +
        ["image", "prepare", "run", "finish"].iter().filter_map(|p| m.get(*p).map(|ms| (p.to_string(), *ms))).collect()
190 +
    }
191 +
182 192
    /// The run whose pass this job reused (#265), if it did.
183 193
    pub fn reused_from(&self) -> Option<i64> {
184 194
        self.error.strip_prefix("reused from run #")?.parse().ok()
…
116 unchanged lines
301 311
        .bind(job_id)
302 312
        .execute(pool)
303 313
        .await?;
314 +
    Ok(())
315 +
}
316 +
317 +
/// Where a finished job's time went, as its runner reported (oxygenflow#35).
318 +
pub async fn set_phases(pool: &Pool, job_id: i64, phases: &std::collections::BTreeMap<String, u64>) -> sqlx::Result<()> {
319 +
    let json = if phases.is_empty() { String::new() } else { serde_json::to_string(phases).unwrap_or_default() };
320 +
    sqlx::query("UPDATE flow_jobs SET phases = $1 WHERE id = $2").bind(json).bind(job_id).execute(pool).await?;
304 321
    Ok(())
305 322
}
306 323
…
959 unchanged lines
1266 1283
            "UPDATE flow_jobs SET status = 'queued', runner = '', token_hash = '', started_ms = NULL, finished_ms = NULL,
1267 1284
             exit_code = NULL, error = '', receipt = '', receipt_hash = '', log_count = 0, also_read = '', receipt_attest = '',
1268 1285
             receipt_signer = '', receipt_note = '', runner_id = NULL, waiting = '', placed = '', peak_disk_mib = NULL,
1269 -
             peak_memory_mib = NULL, approval = CASE WHEN approval = '' THEN '' ELSE 'waiting' END, approved_by = '',
1286 +
             peak_memory_mib = NULL, phases = '', approval = CASE WHEN approval = '' THEN '' ELSE 'waiting' END, approved_by = '',
1270 1287
             approved_ms = NULL, approval_seal = '', approved_via = '', queued_ms = $2, ready_ms = NULL, handed_back_ms = NULL,
1271 1288
             handbacks = 0 WHERE id = $1",
1272 1289
        )
…
249 unchanged lines
modified crates/hub/src/flow.rs +1 −0
…
2439 unchanged lines
2440 2440
    };
2441 2441
    let error = f.error.clone().unwrap_or_default();
2442 2442
    fdb::set_peaks(&app.db, job.id, f.peak_disk_mib, f.peak_memory_mib).await?;
2443 +
    fdb::set_phases(&app.db, job.id, &f.phases).await?;
2443 2444
    let changed = fdb::finish(
2444 2445
        &app.db,
2445 2446
        job.id,
…
612 unchanged lines
modified crates/hub/src/headers.rs +12 −8
1 1
//! Security headers on every response (oxygenhub#222), and the endpoint
2 2
//! the browser reports Content-Security-Policy violations to.
3 3
//!
4 -
//! The policy is sent as `Content-Security-Policy-Report-Only` while the
5 -
//! pages still carry inline scripts and handlers: browsers report what the
6 -
//! policy would block, the hub logs it, and nothing breaks. Switching to
7 -
//! enforcing is one constant ([`ENFORCE`]) once the reports are quiet.
4 +
//! The policy is enforced (#227): the pages carry no inline scripts or
5 +
//! handlers, and in the week of reports before it was switched on the only
6 +
//! thing it would block was Cloudflare's analytics beacon, which the
7 +
//! policy now allows. Violations are still reported and logged; going back
8 +
//! to report-only is one constant ([`ENFORCE`]).
8 9
//! Responses that set their own policy (report frames, #127; a job's
9 10
//! HTML artifact) keep theirs, and may be framed as they say.
10 11
//! Strict-Transport-Security is sent only when `[server] hsts_max_age` is
…
8 unchanged lines
19 20
pub const REPORT_PATH: &str = "/csp-report";
20 21
21 22
/// Enforce the policy rather than only report it.
22 -
const ENFORCE: bool = false;
23 +
const ENFORCE: bool = true;
23 24
24 25
/// What the pages may load: their own scripts (and WebAssembly, for the
25 26
/// backdrop), styles (inline too: style attributes are everywhere), images
26 27
/// from anywhere over https (READMEs show badges), and forms posted only to
27 -
/// the hub (sign-in providers are reached by links, not forms).
28 -
const POLICY: &str = "default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; \
29 -
img-src 'self' data: https:; font-src 'self'; connect-src 'self'; media-src 'self'; object-src 'none'; \
28 +
/// the hub (sign-in providers are reached by links, not forms). Cloudflare,
29 +
/// in front of oxygenhub.io, adds its Web Analytics beacon to pages: its
30 +
/// script and where it reports are allowed, nothing else of theirs.
31 +
const POLICY: &str = "default-src 'self'; script-src 'self' 'wasm-unsafe-eval' https://static.cloudflareinsights.com; \
32 +
style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self'; \
33 +
connect-src 'self' https://cloudflareinsights.com; media-src 'self'; object-src 'none'; \
30 34
base-uri 'self'; form-action 'self'; frame-ancestors 'none'; report-uri /csp-report; report-to csp";
31 35
32 36
const PERMISSIONS: &str = "camera=(), microphone=(), geolocation=(), payment=(), usb=(), serial=(), bluetooth=(), \
…
147 unchanged lines
modified crates/hub/src/metrics.rs +27 −2
…
397 unchanged lines
398 398
        &queue,
399 399
    );
400 400
    histogram(out, "oxyhub_flow_job_run_seconds", "How long finished jobs of the last day ran, by runner.", &run);
401 +
    // Where their time went, as their runners reported (oxygenflow#35).
402 +
    let phases: Vec<String> = sqlx::query_scalar("SELECT phases FROM flow_jobs WHERE finished_ms >= $1 AND phases != ''")
403 +
        .bind(now - 86_400_000)
404 +
        .fetch_all(&app.db)
405 +
        .await
406 +
        .unwrap_or_default();
407 +
    let mut by_phase: std::collections::BTreeMap<String, Vec<f64>> = Default::default();
408 +
    for json in &phases {
409 +
        let m: std::collections::BTreeMap<String, u64> = serde_json::from_str(json).unwrap_or_default();
410 +
        for (phase, ms) in m {
411 +
            by_phase.entry(phase).or_default().push(ms as f64 / 1000.0);
412 +
        }
413 +
    }
414 +
    histogram_by(
415 +
        out,
416 +
        "oxyhub_flow_job_phase_seconds",
417 +
        "Where finished jobs of the last day spent their time, by phase: image (fetching it), prepare, run, finish.",
418 +
        "phase",
419 +
        &by_phase,
420 +
    );
401 421
    family(
402 422
        out,
403 423
        "oxyhub_flow_jobs_finished_last_day",
…
12 unchanged lines
416 436
417 437
/// A histogram family from samples (seconds) per runner.
418 438
fn histogram(out: &mut String, name: &str, help: &str, by_runner: &std::collections::BTreeMap<String, Vec<f64>>) {
439 +
    histogram_by(out, name, help, "runner", by_runner)
440 +
}
441 +
442 +
/// A histogram of job seconds, one series per value of label `key`.
443 +
fn histogram_by(out: &mut String, name: &str, help: &str, key: &str, by: &std::collections::BTreeMap<String, Vec<f64>>) {
419 444
    let _ = writeln!(out, "# HELP {name} {help}");
420 445
    let _ = writeln!(out, "# TYPE {name} histogram");
421 -
    for (runner, samples) in by_runner {
422 -
        let l = format!("runner=\"{}\"", label(runner));
446 +
    for (runner, samples) in by {
447 +
        let l = format!("{key}=\"{}\"", label(runner));
423 448
        for b in JOB_SECONDS {
424 449
            let n = samples.iter().filter(|s| **s <= b).count();
425 450
            let _ = writeln!(out, "{name}_bucket{{{l},le=\"{b}\"}} {n}");
…
56 unchanged lines
modified crates/hub/tests/accounts.rs +4 −3
…
1564 unchanged lines
1565 1565
    assert_eq!(hd[header::REFERRER_POLICY], "strict-origin-when-cross-origin");
1566 1566
    assert_eq!(hd[header::X_FRAME_OPTIONS], "DENY");
1567 1567
    assert!(hd["permissions-policy"].to_str().unwrap().contains("camera=()"));
1568 -
    // Reported, not yet enforced; and no HSTS until the site owner says.
1569 -
    let csp = hd[header::CONTENT_SECURITY_POLICY_REPORT_ONLY].to_str().unwrap();
1568 +
    // Enforced (#227), and still reported; no HSTS until the site owner says.
1569 +
    let csp = hd[header::CONTENT_SECURITY_POLICY].to_str().unwrap();
1570 1570
    for part in [
1571 1571
        "default-src 'self'",
1572 1572
        "object-src 'none'",
…
1 unchanged lines
1574 1574
        "form-action 'self'",
1575 1575
        "'wasm-unsafe-eval'",
1576 1576
        "report-uri /csp-report",
1577 +
        "script-src 'self' 'wasm-unsafe-eval' https://static.cloudflareinsights.com",
1577 1578
    ] {
1578 1579
        assert!(csp.contains(part), "{part} in {csp}");
1579 1580
    }
1580 -
    assert!(hd.get(header::CONTENT_SECURITY_POLICY).is_none() && hd.get(header::STRICT_TRANSPORT_SECURITY).is_none());
1581 +
    assert!(hd.get(header::CONTENT_SECURITY_POLICY_REPORT_ONLY).is_none() && hd.get(header::STRICT_TRANSPORT_SECURITY).is_none());
1581 1582
    // Static files and errors carry them too.
1582 1583
    let res = oxygenhub::router(h.app.clone()).oneshot(Request::get("/no/such/page").body(Body::empty()).unwrap()).await.unwrap();
1583 1584
    assert_eq!(res.headers()[header::X_CONTENT_TYPE_OPTIONS], "nosniff");
…
216 unchanged lines
modified crates/hub/tests/flow.rs +12 −2
…
1210 unchanged lines
1211 1211
        "a failing receipt names no outputs"
1212 1212
    );
1213 1213
    r["outputs"] = serde_json::json!([]);
1214 -
    assert_eq!(finish(&h.app, id, serde_json::json!({ "exit_code": 1, "duration_ms": 700, "receipt": r })).await, StatusCode::NO_CONTENT);
1214 +
    // oxygenflow#35: where its time went, as the runner reports it.
1215 +
    let phases = serde_json::json!({ "image": 9000, "prepare": 1200, "run": 3400, "finish": 500 });
1216 +
    assert_eq!(
1217 +
        finish(&h.app, id, serde_json::json!({ "exit_code": 1, "duration_ms": 700, "receipt": r, "phases": phases })).await,
1218 +
        StatusCode::NO_CONTENT
1219 +
    );
1215 1220
    let (_, page) = call(&h.app, "GET", &format!("/andy/demo/flow/{}", run.number), None, None).await;
1221 +
    let at = page.find("Where its time went: ").expect("the job's phases");
1222 +
    let line = &page[at..at + 120];
1223 +
    let order: Vec<usize> = ["image", "prepare", "run", "finish"].iter().map(|p| line.find(p).expect(p)).collect();
1224 +
    assert!(order.windows(2).all(|w| w[0] < w[1]), "in the order they happened: {line}");
1216 1225
    assert!(page.contains(&format!("/andy/demo/flow/{}/artifacts/test/junit", run.number)), "listed");
1217 1226
    assert!(page.contains("not in its receipt"), "marked");
1218 1227
    assert!(page.contains("2 tests: 1 failed"), "counts beside the job");
…
2839 unchanged lines
4058 4067
    assert!(!bump().await.unwrap().body.contains("api v4"), "not before its run passes");
4059 4068
    pass_all().await;
4060 4069
    let mut updated = None;
4061 -
    for _ in 0..100 {
4070 +
    // Up to 30 s: under the whole Postgres suite 5 s was not always enough.
4071 +
    for _ in 0..600 {
4062 4072
        let p2 = bump().await.unwrap();
4063 4073
        if p2.body.contains("api v4") {
4064 4074
            updated = Some(p2);
…
2910 unchanged lines
replaced deps/oxygenflow +0 −0

No line changes.

modified templates/flow_run.html +1 −0
…
62 unchanged lines
63 63
  <pre class="flow-log" data-log data-seq="{{ l.last_seq }}">{% for line in l.lines %}<span class="ln s-{{ line.stream }}"><span class="t">{{ line.t / 1000 }}.{{ "{:01}"|format(line.t % 1000 / 100) }}s</span>{{ line.text }}</span>
64 64
{% endfor %}</pre>
65 65
  {% if !l.j.placed.is_empty() %}<p class="small mute pad">Ran on {{ l.j.runner }}: {{ l.j.placed }}.</p>{% endif %}
66 +
  {% let phases = l.j.phase_list() %}{% if !phases.is_empty() %}<p class="small mute pad">Where its time went: {% for (p, ms) in phases %}{{ p }} {{ crate::web::flow::secs(ms.clone() as i64) }}{% if !loop.last %}, {% endif %}{% endfor %}.</p>{% endif %}
66 67
  {% if l.j.status == "passed" %}{% if let Some(n) = l.j.reused_from() %}<p class="small mute pad" data-reused>Not run again: this job, with the same spec and sandbox, passed on this very state in <a href="{{ repo.url }}/flow/{{ n }}">run #{{ n }}</a>, whose log it would repeat; that pass's receipt stands for it here.</p>{% endif %}{% endif %}
67 68
  {% if l.j.status == "queued" && !l.j.waiting.is_empty() %}<p class="small warn pad" data-waiting data-queued-note>Waiting: {{ l.j.waiting }}.</p>{% else if l.lines.is_empty() && l.j.status == "queued" %}<p class="small mute pad" data-queued-note>Waiting for a runner{% if !l.j.depends_on().is_empty() %} and for {{ l.j.depends_on().join(", ") }}{% endif %}.</p>{% endif %}
68 69
  {% if l.lines.is_empty() && l.j.status == "skipped" %}<p class="small mute pad">Skipped: a job it depends on did not pass.</p>{% endif %}
…
4 unchanged lines