Insights: OxygenFlow's standard charts, for the site, an owner and a repository e97500b70f9a

-- When a job entered the queue and when it became ready to run

-- (oxygenhub#257): its pickup time is started_ms - ready_ms. queued_ms is

-- set when the job is made and when it is run again; ready_ms when a

-- runner claims it: the latest of queued_ms, its dependencies finishing

-- and its approval.

ALTER TABLE flow_jobs ADD COLUMN queued_ms BIGINT;

ALTER TABLE flow_jobs ADD COLUMN ready_ms BIGINT;

-- Earlier jobs: queued with their run, ready once the stages before

-- theirs had finished (a job runs again keeps its run's time: an estimate).

UPDATE flow_jobs SET queued_ms = (SELECT r.created_ms FROM flow_runs r WHERE r.id = flow_jobs.run_id);

UPDATE flow_jobs SET ready_ms = LEAST(started_ms, GREATEST(queued_ms, COALESCE(approved_ms, 0),

    COALESCE((SELECT MAX(x.finished_ms) FROM flow_jobs x WHERE x.run_id = flow_jobs.run_id AND x.depth < flow_jobs.depth), 0)))

  WHERE started_ms IS NOT NULL;

CREATE INDEX flow_jobs_started ON flow_jobs (started_ms);

CREATE INDEX flow_jobs_finished ON flow_jobs (finished_ms);

-- When a job entered the queue and when it became ready to run

-- (oxygenhub#257): its pickup time is started_ms - ready_ms. queued_ms is

-- set when the job is made and when it is run again; ready_ms when a

-- runner claims it: the latest of queued_ms, its dependencies finishing

-- and its approval.

ALTER TABLE flow_jobs ADD COLUMN queued_ms INTEGER;

ALTER TABLE flow_jobs ADD COLUMN ready_ms INTEGER;

-- Earlier jobs: queued with their run, ready once the stages before

-- theirs had finished (a job runs again keeps its run's time: an estimate).

UPDATE flow_jobs SET queued_ms = (SELECT r.created_ms FROM flow_runs r WHERE r.id = flow_jobs.run_id);

UPDATE flow_jobs SET ready_ms = MIN(started_ms, MAX(queued_ms, COALESCE(approved_ms, 0),

    COALESCE((SELECT MAX(x.finished_ms) FROM flow_jobs x WHERE x.run_id = flow_jobs.run_id AND x.depth < flow_jobs.depth), 0)))

  WHERE started_ms IS NOT NULL;

CREATE INDEX flow_jobs_started ON flow_jobs (started_ms);

CREATE INDEX flow_jobs_finished ON flow_jobs (finished_ms);

//! Insights (oxygenhub#257): OxygenFlow's standard numbers over time, for

//! the site, an owner or a repository: how long jobs wait for a runner, how

//! many run and how they end, how long they take, how deep the queue is and

//! how busy the runners are. All from `flow_jobs` (migration 0080's queue

//! and ready times) and the runners' rows, so nothing new is collected;

//! power users point Grafana at `/metrics` for anything more.

use crate::db::{self, Pool};

use sqlx::FromRow;

/// The span a page shows.

#[derive(Clone, Copy, Debug, PartialEq, Eq)]

pub enum Range {

    Hour,

    Day,

    Week,

}

impl Range {

    pub fn parse(s: &str) -> Range {

        match s {

            "1h" => Range::Hour,

            "7d" => Range::Week,

            _ => Range::Day,

        }

    }

    pub fn key(self) -> &'static str {

        match self {

            Range::Hour => "1h",

            Range::Day => "24h",

            Range::Week => "7d",

        }

    }

    pub fn ms(self) -> i64 {

        match self {

            Range::Hour => 3_600_000,

            Range::Day => 86_400_000,

            Range::Week => 7 * 86_400_000,

        }

    }

    /// How many buckets the charts have.

    pub fn buckets(self) -> usize {

        match self {

            Range::Hour => 60,

            Range::Day => 48,

            Range::Week => 56,

        }

    }

    pub fn bucket_ms(self) -> i64 {

        self.ms() / self.buckets() as i64

    }

}

/// Which jobs count.

#[derive(Clone, Debug, Default)]

pub struct Scope {

    /// An owner's repositories.

    pub owner: Option<String>,

    /// One repository.

    pub repo_id: Option<i64>,

    /// One runner's jobs.

    pub runner: Option<String>,

}

#[derive(Clone, Debug, FromRow)]

struct JobRow {

    status: String,

    runner: String,

    owner: String,

    repo: String,

    queued_ms: Option<i64>,

    ready_ms: Option<i64>,

    started_ms: Option<i64>,

    finished_ms: Option<i64>,

}

impl JobRow {

    /// Seconds from ready to started, when it started.

    fn pickup(&self) -> Option<f64> {

        Some((self.started_ms? - self.ready_ms.or(self.queued_ms)?).max(0) as f64 / 1000.0)

    }

    fn duration(&self) -> Option<f64> {

        Some((self.finished_ms? - self.started_ms?).max(0) as f64 / 1000.0)

    }

    /// When it left the queue: started, or ended without starting.

    fn left_queue(&self) -> Option<i64> {

        self.started_ms.or(self.finished_ms)

    }

    fn ended(&self) -> bool {

        !matches!(self.status.as_str(), "queued" | "running")

    }

}

/// One chart's line or stack: a value per bucket.

#[derive(Clone, Debug)]

pub struct Series {

    pub name: &'static str,

    /// The CSS class that colours it (`s-ok`, `s-bad`, …).

    pub class: &'static str,

    pub values: Vec<Option<f64>>,

}

/// One runner's row.

#[derive(Clone, Debug)]

pub struct RunnerRow {

    pub name: String,

    pub status: &'static str,

    pub running: i64,

    pub max_jobs: i64,

    pub jobs: i64,

    pub failed: i64,

    pub pickup_median: Option<f64>,

    pub duration_median: Option<f64>,

    pub last_seen_ms: Option<i64>,

}

/// One repository's row.

#[derive(Clone, Debug)]

pub struct RepoRow {

    pub slug: String,

    pub jobs: i64,

    pub failed: i64,

    pub pickup_median: Option<f64>,

    pub pickup_p95: Option<f64>,

    pub queued: i64,

}

/// Everything a page draws.

#[derive(Clone, Debug)]

pub struct Insights {

    pub range: Range,

    /// The first bucket's start; buckets follow every `range.bucket_ms()`.

    pub start_ms: i64,

    pub now_ms: i64,

    /// Seconds: median and 95th percentile per bucket, by the job's start.

    pub pickup: Vec<Series>,

    /// Jobs that ended per bucket, by result.

    pub results: Vec<Series>,

    /// Seconds: median and 95th percentile per bucket, by the job's end.

    pub durations: Vec<Series>,

    /// Jobs in the queue and running, at each bucket's end.

    pub load: Vec<Series>,

    /// Running jobs over the online runners' slots, percent, per bucket.

    pub utilisation: Vec<Series>,

    pub ended: i64,

    pub passed: i64,

    pub failed: i64,

    pub pickup_median: Option<f64>,

    pub pickup_p95: Option<f64>,

    pub duration_median: Option<f64>,

    pub queued_now: i64,

    pub running_now: i64,

    pub runners_online: i64,

    pub runners_total: i64,

    pub slots_busy: i64,

    pub slots_total: i64,

    pub runners: Vec<RunnerRow>,

    pub repos: Vec<RepoRow>,

}

/// The `p`-th percentile (0..=1) of `v`, nearest rank.

pub fn percentile(v: &mut [f64], p: f64) -> Option<f64> {

    if v.is_empty() {

        return None;

    }

    v.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));

    let rank = ((p * v.len() as f64).ceil() as usize).clamp(1, v.len());

    Some(v[rank - 1])

}

async fn rows(pool: &Pool, scope: &Scope, since: i64) -> sqlx::Result<Vec<JobRow>> {

    sqlx::query_as(

        "SELECT j.status, j.runner, p.owner, p.name AS repo, COALESCE(j.queued_ms, r.created_ms) AS queued_ms, j.ready_ms,

                j.started_ms, j.finished_ms

         FROM flow_jobs j JOIN flow_runs r ON r.id = j.run_id JOIN repositories p ON p.id = r.repo_id

         WHERE (j.finished_ms >= $1 OR j.status IN ('queued', 'running'))

           AND ($2 = '' OR p.owner = $3) AND ($4 IS NULL OR p.id = $5) AND ($6 = '' OR j.runner = $7)",

    )

    .bind(since)

    .bind(scope.owner.as_deref().unwrap_or(""))

    .bind(scope.owner.as_deref().unwrap_or(""))

    .bind(scope.repo_id)

    .bind(scope.repo_id)

    .bind(scope.runner.as_deref().unwrap_or(""))

    .bind(scope.runner.as_deref().unwrap_or(""))

    .fetch_all(pool)

    .await

}

/// Gather a scope's numbers over `range`, ending now.

pub async fn gather(pool: &Pool, scope: &Scope, range: Range) -> sqlx::Result<Insights> {

    let now = db::now_ms();

    let start = now - range.ms();

    let jobs = rows(pool, scope, start).await?;

    let runners = db::runners::list(pool).await?;

    let n = range.buckets();

    let width = range.bucket_ms();

    let bucket = |t: i64| -> Option<usize> { (t >= start && t <= now).then(|| (((t - start) / width) as usize).min(n - 1)) };

    let mut pickups: Vec<Vec<f64>> = vec![Vec::new(); n];

    let mut durations: Vec<Vec<f64>> = vec![Vec::new(); n];

    let (mut passed, mut failed, mut other) = (vec![0f64; n], vec![0f64; n], vec![0f64; n]);

    let (mut all_pickups, mut all_durations) = (Vec::new(), Vec::new());

    for j in &jobs {

        if let (Some(p), Some(b)) = (j.pickup(), j.started_ms.and_then(bucket)) {

            pickups[b].push(p);

            all_pickups.push(p);

        }

        if let (Some(f), true) = (j.finished_ms, j.ended()) {

            if let Some(b) = bucket(f) {

                match j.status.as_str() {

                    "passed" => passed[b] += 1.0,

                    "failed" => failed[b] += 1.0,

                    _ => other[b] += 1.0,

                }

                if let Some(d) = j.duration() {

                    durations[b].push(d);

                    all_durations.push(d);

                }

            }

        }

    }

    // The queue and the running jobs at each bucket's end.

    let (mut queued_at, mut running_at) = (vec![0f64; n], vec![0f64; n]);

    for (i, (q, r)) in queued_at.iter_mut().zip(running_at.iter_mut()).enumerate() {

        let t = (start + (i as i64 + 1) * width).min(now);

        for j in &jobs {

            let entered = j.queued_ms.unwrap_or(i64::MAX);

            if entered <= t && j.left_queue().is_none_or(|l| l > t) && !(j.ended() && j.finished_ms.is_none()) {

                *q += 1.0;

            }

            if j.started_ms.is_some_and(|s| s <= t) && j.finished_ms.is_none_or(|f| f > t) && !(j.ended() && j.finished_ms.is_none()) {

                *r += 1.0;

            }

        }

    }

    // Slots: the runners serving the scope that are online now.

    let serving: Vec<&db::runners::Runner> =

        runners.iter().filter(|r| scope.runner.as_deref().is_none_or(|n| n == r.name)).filter(|r| r.state != "removed").collect();

    let slots_total: i64 = serving.iter().filter(|r| r.online()).map(|r| r.max_jobs.max(1)).sum();

    let utilisation: Vec<Option<f64>> =

        running_at.iter().map(|r| (slots_total > 0).then(|| (r / slots_total as f64 * 100.0).min(100.0))).collect();

    let quantiles = |sets: &mut Vec<Vec<f64>>, p: f64| -> Vec<Option<f64>> { sets.iter_mut().map(|v| percentile(v, p)).collect() };

    let pickup = vec![

        Series { name: "median", class: "s-main", values: quantiles(&mut pickups, 0.5) },

        Series { name: "95th percentile", class: "s-warn", values: quantiles(&mut pickups, 0.95) },

    ];

    let durations_series = vec![

        Series { name: "median", class: "s-main", values: quantiles(&mut durations, 0.5) },

        Series { name: "95th percentile", class: "s-warn", values: quantiles(&mut durations, 0.95) },

    ];

    let some = |v: Vec<f64>| v.into_iter().map(Some).collect::<Vec<_>>();

    let ended_total = |v: &[f64]| v.iter().sum::<f64>() as i64;

    let (p_total, f_total, o_total) = (ended_total(&passed), ended_total(&failed), ended_total(&other));

    // Runners: what each did in the range.

    let mut runner_rows: Vec<RunnerRow> = serving

        .iter()

        .filter(|r| scope.owner.is_none() && scope.repo_id.is_none() || jobs.iter().any(|j| j.runner == r.name))

        .map(|r| {

            let mine: Vec<&JobRow> = jobs.iter().filter(|j| j.runner == r.name).collect();

            let mut p: Vec<f64> = mine.iter().filter(|j| j.started_ms.is_some_and(|s| s >= start)).filter_map(|j| j.pickup()).collect();

            let mut d: Vec<f64> = mine.iter().filter(|j| j.ended()).filter_map(|j| j.duration()).collect();

            RunnerRow {

                name: r.name.clone(),

                status: r.status(),

                running: mine.iter().filter(|j| j.status == "running").count() as i64,

                max_jobs: r.max_jobs.max(1),

                jobs: mine.iter().filter(|j| j.ended()).count() as i64,

                failed: mine.iter().filter(|j| j.status == "failed").count() as i64,

                pickup_median: percentile(&mut p, 0.5),

                duration_median: percentile(&mut d, 0.5),

                last_seen_ms: r.last_seen_ms,

            }

        })

        .collect();

    runner_rows.sort_by(|a, b| b.jobs.cmp(&a.jobs).then_with(|| a.name.cmp(&b.name)));

    // Repositories, busiest first (a repository's own page has just itself).

    let mut by_repo: std::collections::BTreeMap<String, (i64, i64, Vec<f64>, i64)> = Default::default();

    for j in &jobs {

        let e = by_repo.entry(format!("{}/{}", j.owner, j.repo)).or_default();

        if j.ended() && j.finished_ms.is_some_and(|f| f >= start) {

            e.0 += 1;

            if j.status == "failed" {

                e.1 += 1;

            }

        }

        if let Some(p) = j.pickup().filter(|_| j.started_ms.is_some_and(|s| s >= start)) {

            e.2.push(p);

        }

        if j.status == "queued" {

            e.3 += 1;

        }

    }

    let mut repos: Vec<RepoRow> = by_repo

        .into_iter()

        .map(|(slug, (jobs, failed, mut p, queued))| RepoRow {

            slug,

            jobs,

            failed,

            pickup_median: percentile(&mut p, 0.5),

            pickup_p95: percentile(&mut p, 0.95),

            queued,

        })

        .collect();

    repos.sort_by(|a, b| (b.jobs + b.queued).cmp(&(a.jobs + a.queued)).then_with(|| a.slug.cmp(&b.slug)));

    repos.truncate(15);

    Ok(Insights {

        range,

        start_ms: start,

        now_ms: now,

        pickup,

        results: vec![

            Series { name: "passed", class: "s-ok", values: some(passed) },

            Series { name: "failed", class: "s-bad", values: some(failed) },

            Series { name: "other (error, cancelled, skipped)", class: "s-mute", values: some(other) },

        ],

        durations: durations_series,

        load: vec![

            Series { name: "queued", class: "s-warn", values: some(queued_at) },

            Series { name: "running", class: "s-main", values: some(running_at) },

        ],

        utilisation: vec![Series { name: "busy slots", class: "s-main", values: utilisation }],

        ended: p_total + f_total + o_total,

        passed: p_total,

        failed: f_total,

        pickup_median: percentile(&mut all_pickups, 0.5),

        pickup_p95: percentile(&mut all_pickups, 0.95),

        duration_median: percentile(&mut all_durations, 0.5),

        queued_now: jobs.iter().filter(|j| j.status == "queued").count() as i64,

        running_now: jobs.iter().filter(|j| j.status == "running").count() as i64,

        runners_online: serving.iter().filter(|r| r.online()).count() as i64,

        runners_total: serving.len() as i64,

        slots_busy: jobs.iter().filter(|j| j.status == "running").count() as i64,

        slots_total,

        runners: runner_rows,

        repos,

    })

}

/// A span of seconds, short: `42 s`, `3.5 min`, `1.2 h`.

pub fn seconds(s: f64) -> String {

    if s < 60.0 {

        format!("{s:.0} s")

    } else if s < 3600.0 {

        let m = s / 60.0;

        if m < 10.0 {

            format!("{m:.1} min")

        } else {

            format!("{m:.0} min")

        }

    } else {

        format!("{:.1} h", s / 3600.0)

    }

}

#[cfg(test)]

mod tests {

    use super::*;

    #[test]

    fn percentiles_are_nearest_rank() {

        let mut v = vec![5.0, 1.0, 3.0, 2.0, 4.0];

        assert_eq!(percentile(&mut v, 0.5), Some(3.0));

        assert_eq!(percentile(&mut v, 0.95), Some(5.0));

        assert_eq!(percentile(&mut [], 0.5), None);

        assert_eq!(seconds(42.0), "42 s");

        assert_eq!(seconds(210.0), "3.5 min");

        assert_eq!(seconds(4320.0), "1.2 h");

    }

}

//! Charts drawn by the hub (oxygenhub#257): inline SVG, coloured by CSS

//! classes from the theme's tokens, so they need no script, no library and

//! nothing the Content-Security-Policy forbids. Lines for times and levels,

//! stacked bars for counts; the x axis is relative (`−24 h` … `now`).

use crate::insights::{Range, Series};

use std::fmt::Write;

const W: f64 = 640.0;

const H: f64 = 190.0;

const LEFT: f64 = 52.0;

const RIGHT: f64 = 8.0;

const TOP: f64 = 10.0;

const BOTTOM: f64 = 22.0;

fn esc(s: &str) -> String {

    s.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;").replace('"', "&quot;")

}

/// A round top for the y axis: 1, 2 or 5 times a power of ten.

fn nice(max: f64) -> f64 {

    if max <= 0.0 {

        return 1.0;

    }

    let p = 10f64.powf(max.log10().floor());

    for m in [1.0, 2.0, 5.0, 10.0] {

        if max <= m * p {

            return m * p;

        }

    }

    10.0 * p

}

/// The frame: grid lines with the y labels, and the relative x labels.

fn frame(out: &mut String, top: f64, range: Range, label: &dyn Fn(f64) -> String) {

    let plot_h = H - TOP - BOTTOM;

    for i in 0..=2 {

        let v = top * i as f64 / 2.0;

        let y = TOP + plot_h - plot_h * i as f64 / 2.0;

        let _ = write!(out, r#"<line class="grid" x1="{LEFT}" x2="{}" y1="{y:.1}" y2="{y:.1}"/>"#, W - RIGHT);

        let _ = write!(out, r#"<text class="axis" x="{}" y="{:.1}" text-anchor="end">{}</text>"#, LEFT - 6.0, y + 4.0, esc(&label(v)));

    }

    let (first, mid) = match range {

        Range::Hour => ("−60 min", "−30 min"),

        Range::Day => ("−24 h", "−12 h"),

        Range::Week => ("−7 d", "−3.5 d"),

    };

    let base = H - 6.0;

    let mid_x = LEFT + (W - LEFT - RIGHT) / 2.0;

    let _ = write!(out, r#"<text class="axis" x="{LEFT}" y="{base}">{first}</text>"#);

    let _ = write!(out, r#"<text class="axis" x="{mid_x}" y="{base}" text-anchor="middle">{mid}</text>"#);

    let _ = write!(out, r#"<text class="axis" x="{}" y="{base}" text-anchor="end">now</text>"#, W - RIGHT);

}

fn open(out: &mut String, title: &str) {

    let _ = write!(out, r#"<svg class="chart" viewBox="0 0 {W} {H}" role="img" aria-label="{}">"#, esc(title));

}

/// Lines, one per series; a missing value breaks its line.

pub fn lines(title: &str, range: Range, series: &[Series], label: &dyn Fn(f64) -> String) -> String {

    let max = series.iter().flat_map(|s| s.values.iter().flatten()).fold(0f64, |a, b| a.max(*b));

    let top = nice(max);

    let mut out = String::new();

    open(&mut out, title);

    frame(&mut out, top, range, label);

    let n = series.first().map_or(0, |s| s.values.len()).max(1);

    let step = (W - LEFT - RIGHT) / n as f64;

    let plot_h = H - TOP - BOTTOM;

    for s in series {

        let mut d = String::new();

        let mut pen = false;

        for (i, v) in s.values.iter().enumerate() {

            match v {

                Some(v) => {

                    let x = LEFT + step * (i as f64 + 0.5);

                    let y = TOP + plot_h - plot_h * (v / top).min(1.0);

                    let _ = write!(d, "{}{x:.1},{y:.1} ", if pen { "L" } else { "M" });

                    pen = true;

                }

                None => pen = false,

            }

        }

        if !d.is_empty() {

            let _ = write!(out, r#"<path class="line {}" d="{}"><title>{}</title></path>"#, s.class, d.trim_end(), esc(s.name));

        }

        // A lone point between gaps would not show as a line: a dot.

        for (i, v) in s.values.iter().enumerate() {

            let alone = v.is_some() && (i == 0 || s.values[i - 1].is_none()) && s.values.get(i + 1).is_none_or(|n| n.is_none());

            if let (true, Some(v)) = (alone, v) {

                let x = LEFT + step * (i as f64 + 0.5);

                let y = TOP + plot_h - plot_h * (v / top).min(1.0);

                let _ = write!(out, r#"<circle class="dot {}" cx="{x:.1}" cy="{y:.1}" r="2.5"/>"#, s.class);

            }

        }

    }

    out.push_str("</svg>");

    out

}

/// Stacked bars: each bucket's series on top of one another.

pub fn stacked(title: &str, range: Range, series: &[Series], label: &dyn Fn(f64) -> String) -> String {

    let n = series.first().map_or(0, |s| s.values.len());

    let totals: Vec<f64> = (0..n).map(|i| series.iter().map(|s| s.values[i].unwrap_or(0.0)).sum()).collect();

    let top = nice(totals.iter().fold(0f64, |a, b| a.max(*b)));

    let mut out = String::new();

    open(&mut out, title);

    frame(&mut out, top, range, label);

    let step = (W - LEFT - RIGHT) / n.max(1) as f64;

    let plot_h = H - TOP - BOTTOM;

    for i in 0..n {

        let mut y = TOP + plot_h;

        let x = LEFT + step * i as f64 + step * 0.1;

        for s in series {

            let v = s.values[i].unwrap_or(0.0);

            if v <= 0.0 {

                continue;

            }

            let h = plot_h * v / top;

            y -= h;

            let _ = write!(

                out,

                r#"<rect class="bar {}" x="{x:.1}" y="{y:.1}" width="{:.1}" height="{h:.1}"><title>{} {}</title></rect>"#,

                s.class,

                step * 0.8,

                label(v),

                esc(s.name)

            );

        }

    }

    out.push_str("</svg>");

    out

}

#[cfg(test)]

mod tests {

    use super::*;

    #[test]

    fn charts_are_svg_with_classes_and_no_styles() {

        let s = vec![Series { name: "median", class: "s-main", values: vec![Some(1.0), None, Some(3.0), Some(4.0)] }];

        let svg = lines("Pickup", Range::Hour, &s, &|v| format!("{v:.0}"));

        assert!(svg.starts_with("<svg class=\"chart\"") && svg.ends_with("</svg>"));

        assert!(svg.contains("class=\"line s-main\"") && svg.contains("class=\"dot s-main\""), "{svg}");

        assert!(!svg.contains("style="), "no inline styles: the CSP");

        let bars = stacked("Jobs", Range::Day, &s, &|v| format!("{v:.0}"));

        assert_eq!(bars.matches("<rect").count(), 3);

        assert_eq!(nice(7.0), 10.0);

        assert_eq!(nice(130.0), 200.0);

    }

}

//! Insights pages (oxygenhub#257): one page at three scopes, so the numbers

//! always agree: the site (`/admin/insights`, administrators), an owner

//! (`/insights/{owner}`, the owner, its members and administrators) and a

//! repository (`/{owner}/{name}/insights`, anyone who reads it). `?range=`

//! is `1h`, `24h` (default) or `7d`; the site page also takes `repo` and

//! `runner`; `?wall=1` is the second-monitor view. The page refreshes its

//! numbers every 30 seconds (hub.js), and works as a snapshot without it.

use super::admin::require_admin;

use super::charts;

use super::{filters, lookup, Error, Page, RepoShell, Shell};

use crate::auth::Viewer;

use crate::insights::{self, Insights, Range, Scope};

use crate::{db, Shared};

use askama::Template;

use axum::extract::{Path, Query, State};

use serde::Deserialize;

#[derive(Deserialize, Default)]

pub struct InsightsQuery {

    #[serde(default)]

    range: String,

    #[serde(default)]

    repo: String,

    #[serde(default)]

    runner: String,

    #[serde(default)]

    wall: String,

}

/// The charts, drawn.

pub struct Charts {

    pub pickup: String,

    pub results: String,

    pub durations: String,

    pub load: String,

    pub utilisation: String,

}

#[derive(Template)]

#[template(path = "insights.html")]

pub struct InsightsPage {

    shell: Shell,

    /// The repository's bar, on a repository's page.

    repo: Option<RepoShell>,

    /// What the page covers: `the site`, an owner, `owner/name`.

    scope: String,

    /// This page's path, for the range links.

    base: String,

    /// The filters to keep in the range links (`&repo=…&runner=…`).

    keep: String,

    repo_filter: String,

    runner_filter: String,

    /// The site page's filter choices.

    repo_choices: Vec<String>,

    runner_choices: Vec<String>,

    i: Insights,

    charts: Charts,

    wall: bool,

    grafana: String,

    /// The wider scope's page, when there is one the viewer may see.

    wider: Option<(String, String)>,

}

impl InsightsPage {

    pub fn secs(&self, s: &Option<f64>) -> String {

        s.map(insights::seconds).unwrap_or_else(|| "–".into())

    }

    pub fn pass_rate(&self) -> String {

        if self.i.passed + self.i.failed == 0 {

            "–".into()

        } else {

            format!("{:.0}%", self.i.passed as f64 * 100.0 / (self.i.passed + self.i.failed) as f64)

        }

    }

    pub fn range_on(&self, key: &str) -> bool {

        self.i.range.key() == key

    }

}

fn draw(i: &Insights) -> Charts {

    let secs = |v: f64| insights::seconds(v);

    let count = |v: f64| format!("{v:.0}");

    let pct = |v: f64| format!("{v:.0}%");

    Charts {

        pickup: charts::lines("Time to pickup", i.range, &i.pickup, &secs),

        results: charts::stacked("Jobs finished", i.range, &i.results, &count),

        durations: charts::lines("Job duration", i.range, &i.durations, &secs),

        load: charts::lines("Queued and running", i.range, &i.load, &count),

        utilisation: charts::lines("Runner utilisation", i.range, &i.utilisation, &pct),

    }

}

fn keep(q: &InsightsQuery) -> String {

    let mut k = String::new();

    for (name, v) in [("repo", &q.repo), ("runner", &q.runner)] {

        if !v.is_empty() {

            k.push_str(&format!("&{name}={}", super::settings::urlencode(v)));

        }

    }

    if q.wall == "1" {

        k.push_str("&wall=1");

    }

    k

}

#[allow(clippy::too_many_arguments)]

async fn page(

    app: &Shared,

    shell: Shell,

    repo: Option<RepoShell>,

    scope_label: String,

    base: String,

    scope: Scope,

    q: &InsightsQuery,

    choices: (Vec<String>, Vec<String>),

    wider: Option<(String, String)>,

) -> Result<Page<InsightsPage>, Error> {

    let range = Range::parse(&q.range);

    let i = insights::gather(&app.db, &scope, range).await.map_err(|e| Error::from_db(shell.clone(), e))?;

    let charts = draw(&i);

    Ok(Page(InsightsPage {

        shell,

        repo,

        scope: scope_label,

        base,

        keep: keep(q),

        repo_filter: q.repo.clone(),

        runner_filter: q.runner.clone(),

        repo_choices: choices.0,

        runner_choices: choices.1,

        i,

        charts,

        wall: q.wall == "1",

        grafana: app.config.server.grafana_url.clone(),

        wider,

    }))

}

/// `GET /admin/insights`: the whole site, for administrators.

pub async fn site(State(app): State<Shared>, viewer: Viewer, Query(q): Query<InsightsQuery>) -> Result<Page<InsightsPage>, Error> {

    let shell = Shell::new(&app, "/admin/insights", &viewer);

    require_admin(&viewer, &shell)?;

    let mut scope = Scope::default();

    if let Some((owner, name)) = q.repo.split_once('/') {

        let row = db::repos::get(&app.db, owner, name).await.map_err(|e| Error::from_db(shell.clone(), e))?;

        scope.repo_id = Some(row.map_or(-1, |r| r.id));

    } else if !q.repo.is_empty() {

        scope.owner = Some(q.repo.clone());

    }

    if !q.runner.is_empty() {

        scope.runner = Some(q.runner.clone());

    }

    let runners: Vec<String> =

        db::runners::list(&app.db).await.map_err(|e| Error::from_db(shell.clone(), e))?.into_iter().map(|r| r.name).collect();

    let repos: Vec<String> = sqlx::query_scalar(

        "SELECT DISTINCT p.owner || '/' || p.name FROM flow_runs r JOIN repositories p ON p.id = r.repo_id WHERE r.created_ms >= $1 ORDER BY 1",

    )

    .bind(db::now_ms() - Range::Week.ms())

    .fetch_all(&app.db)

    .await

    .map_err(|e| Error::from_db(shell.clone(), e))?;

    let label = match (q.repo.is_empty(), q.runner.is_empty()) {

        (true, true) => "the site".to_string(),

        (false, true) => q.repo.clone(),

        (true, false) => format!("runner {}", q.runner),

        (false, false) => format!("{} on {}", q.repo, q.runner),

    };

    page(&app, shell, None, label, "/admin/insights".into(), scope, &q, (repos, runners), None).await

}

/// Whether the viewer may see an owner's numbers: the owner, a member of

/// the organisation, or an administrator.

async fn sees_owner(app: &Shared, viewer: &Viewer, owner: &str) -> sqlx::Result<bool> {

    let Some(me) = viewer.account() else { return Ok(false) };

    if me.is_admin || me.username.eq_ignore_ascii_case(owner) {

        return Ok(true);

    }

    match db::accounts::org_by_name(&app.db, owner).await? {

        Some(org) => Ok(db::accounts::org_role(&app.db, org.id, me.id).await?.is_some()),

        None => Ok(false),

    }

}

/// `GET /insights/{owner}`: every repository of an owner.

pub async fn owner(

    State(app): State<Shared>,

    viewer: Viewer,

    Path(owner): Path<String>,

    Query(q): Query<InsightsQuery>,

) -> Result<Page<InsightsPage>, Error> {

    let shell = Shell::new(&app, &format!("/insights/{owner}"), &viewer);

    if !sees_owner(&app, &viewer, &owner).await.map_err(|e| Error::from_db(shell.clone(), e))? {

        return Err(Error::not_found(shell));

    }

    let q = InsightsQuery { repo: String::new(), runner: String::new(), ..q };

    let scope = Scope { owner: Some(owner.clone()), ..Default::default() };

    let wider = viewer.is_admin().then(|| ("the site".to_string(), "/admin/insights".to_string()));

    page(&app, shell, None, owner.clone(), format!("/insights/{owner}"), scope, &q, Default::default(), wider).await

}

/// `GET /{owner}/{name}/insights`: one repository, for anyone who reads it.

pub async fn repo(

    State(app): State<Shared>,

    viewer: Viewer,

    Path((owner, name)): Path<(String, String)>,

    Query(q): Query<InsightsQuery>,

) -> Result<Page<InsightsPage>, Error> {

    let shell = Shell::new(&app, &format!("/{owner}/{name}/insights"), &viewer);

    let (_repo, row, role) = lookup(&app, &shell, &viewer, &owner, &name).await?;

    let q = InsightsQuery { repo: String::new(), runner: String::new(), ..q };

    let scope = Scope { repo_id: Some(row.id), ..Default::default() };

    let wider = if sees_owner(&app, &viewer, &owner).await.map_err(|e| Error::from_db(shell.clone(), e))? {

        Some((format!("all of {owner}'s repositories"), format!("/insights/{owner}")))

    } else {

        None

    };

    let repo_shell = RepoShell::new(&app, &row, "insights").with_role(role);

    page(

        &app,

        shell,

        Some(repo_shell),

        format!("{owner}/{name}"),

        format!("/{owner}/{name}/insights"),

        scope,

        &q,

        Default::default(),

        wider,

    )

    .await

}

{% extends "base.html" %}

{% import "repo_tabs.html" as m %}

{% block title %}Insights · {{ scope }}{% endblock %}

{% block bodyattrs %}{% if wall %} class="wall"{% endif %}{% endblock %}

{% block tabs %}{% if !wall %}{% if let Some(r) = repo %}{% call m::tabs(r, "insights") %}{% endif %}{% endif %}{% endblock %}

{% block content %}

<div class="insights" data-refresh="30" data-fragment="insights">

<div class="head"><h1>Insights <small>{{ scope }} · OxygenFlow, the last {% if range_on("1h") %}hour{% else if range_on("7d") %}7 days{% else %}24 hours{% endif %}</small></h1></div>

<div class="bar insights-bar">

  <span class="modes">

    <a href="{{ base }}?range=1h{{ keep }}"{% if range_on("1h") %} class="on" aria-current="page"{% endif %}>1 hour</a>

    <a href="{{ base }}?range=24h{{ keep }}"{% if range_on("24h") %} class="on" aria-current="page"{% endif %}>24 hours</a>

    <a href="{{ base }}?range=7d{{ keep }}"{% if range_on("7d") %} class="on" aria-current="page"{% endif %}>7 days</a>

  </span>

  {% if !repo_choices.is_empty() || !runner_choices.is_empty() %}

  <form class="filters" method="get" action="{{ base }}">

    <input type="hidden" name="range" value="{{ i.range.key() }}">{% if wall %}<input type="hidden" name="wall" value="1">{% endif %}

    <select name="repo" aria-label="Repository or owner"><option value="">every repository</option>{% for r in repo_choices %}<option{% if *r == repo_filter %} selected{% endif %}>{{ r }}</option>{% endfor %}</select>

    <select name="runner" aria-label="Runner"><option value="">every runner</option>{% for r in runner_choices %}<option{% if *r == runner_filter %} selected{% endif %}>{{ r }}</option>{% endfor %}</select>

    <button>filter</button>

  </form>

  {% endif %}

  <span class="insights-links small">

    {% if let Some((label, url)) = wider %}<a href="{{ url }}?range={{ i.range.key() }}">{{ label }}</a>{% endif %}

    {% if wall %}<a href="{{ base }}?range={{ i.range.key() }}{% if !repo_filter.is_empty() %}&repo={{ repo_filter|urlencode }}{% endif %}{% if !runner_filter.is_empty() %}&runner={{ runner_filter|urlencode }}{% endif %}">leave the wall view</a>{% else %}<a href="{{ base }}?range={{ i.range.key() }}{{ keep }}&wall=1" target="_blank" rel="noopener" title="For a second monitor: large, no menus, refreshing itself">wall view ↗</a>{% endif %}

    {% if !grafana.is_empty() %}<a href="{{ grafana }}" target="_blank" rel="noopener">open in Grafana ↗</a>{% endif %}

  </span>

</div>

<div class="kpis">

  <div class="kpi"><span class="kpi-v">{{ secs(i.pickup_median) }}</span><span class="kpi-l">time to pickup, median</span><span class="kpi-s">95th percentile {{ secs(i.pickup_p95) }}</span></div>

  <div class="kpi"><span class="kpi-v">{{ i.ended }}</span><span class="kpi-l">jobs finished</span><span class="kpi-s">{{ pass_rate() }} passed of those that passed or failed</span></div>

  <div class="kpi"><span class="kpi-v">{{ secs(i.duration_median) }}</span><span class="kpi-l">job duration, median</span><span class="kpi-s">started to finished</span></div>

  <div class="kpi{% if i.queued_now > 0 && i.slots_total > 0 && i.slots_busy >= i.slots_total %} kpi-warn{% endif %}"><span class="kpi-v">{{ i.queued_now }}</span><span class="kpi-l">queued now</span><span class="kpi-s">{{ i.running_now }} running</span></div>

  <div class="kpi{% if i.runners_total > 0 && i.runners_online == 0 %} kpi-bad{% endif %}"><span class="kpi-v">{{ i.runners_online }}/{{ i.runners_total }}</span><span class="kpi-l">runners online</span><span class="kpi-s">{{ i.slots_busy }} of {{ i.slots_total }} slots busy</span></div>

</div>

<div class="chart-grid">

  <figure class="chart-card"><figcaption>Time to pickup <span class="mute small">ready to started</span></figcaption>{{ charts.pickup|safe }}<div class="legend">{% for s in i.pickup %}<span class="key {{ s.class }}">{{ s.name }}</span>{% endfor %}</div></figure>

  <figure class="chart-card"><figcaption>Jobs finished <span class="mute small">per {% if range_on("1h") %}minute{% else if range_on("7d") %}3 hours{% else %}30 minutes{% endif %}</span></figcaption>{{ charts.results|safe }}<div class="legend">{% for s in i.results %}<span class="key {{ s.class }}">{{ s.name }}</span>{% endfor %}</div></figure>

  <figure class="chart-card"><figcaption>Queued and running</figcaption>{{ charts.load|safe }}<div class="legend">{% for s in i.load %}<span class="key {{ s.class }}">{{ s.name }}</span>{% endfor %}</div></figure>

  <figure class="chart-card"><figcaption>Job duration <span class="mute small">started to finished</span></figcaption>{{ charts.durations|safe }}<div class="legend">{% for s in i.durations %}<span class="key {{ s.class }}">{{ s.name }}</span>{% endfor %}</div></figure>

  <figure class="chart-card"><figcaption>Runner utilisation <span class="mute small">running jobs over the slots of the runners online now</span></figcaption>{{ charts.utilisation|safe }}<div class="legend">{% for s in i.utilisation %}<span class="key {{ s.class }}">{{ s.name }}</span>{% endfor %}</div></figure>

</div>

{% if !i.runners.is_empty() %}

<section class="panel"><h2>Runners</h2>

<div class="table"><table>

<tr><th>Runner</th><th>Status</th><th>Busy</th><th>Jobs</th><th>Failed</th><th>Pickup, median</th><th>Duration, median</th><th>Last heard from</th></tr>

{% for r in i.runners %}<tr><td><strong>{{ r.name }}</strong></td><td><span class="runner-status st-{{ r.status }}">{{ r.status }}</span></td><td>{{ r.running }}/{{ r.max_jobs }}</td><td>{{ r.jobs }}</td><td>{% if r.failed > 0 %}{{ r.failed }}{% else %}<span class="mute">0</span>{% endif %}</td><td>{{ secs(r.pickup_median) }}</td><td>{{ secs(r.duration_median) }}</td><td class="small">{% if let Some(t) = r.last_seen_ms %}<time title="{{ t|when_i }}">{{ t|ago_i }}</time>{% else %}<span class="mute">never</span>{% endif %}</td></tr>{% endfor %}

</table></div>

</section>

{% endif %}

{% if i.repos.len() > 1 %}

<section class="panel"><h2>Repositories <small class="mute">busiest first</small></h2>

<div class="table"><table>

<tr><th>Repository</th><th>Jobs</th><th>Failed</th><th>Queued now</th><th>Pickup, median</th><th>Pickup, 95th percentile</th></tr>

{% for r in i.repos %}<tr><td><a href="/{{ r.slug }}/insights?range={{ i.range.key() }}">{{ r.slug }}</a></td><td>{{ r.jobs }}</td><td>{% if r.failed > 0 %}{{ r.failed }}{% else %}<span class="mute">0</span>{% endif %}</td><td>{{ r.queued }}</td><td>{{ secs(r.pickup_median) }}</td><td>{{ secs(r.pickup_p95) }}</td></tr>{% endfor %}

</table></div>

</section>

{% endif %}

<p class="small mute">Updated <time title="{{ i.now_ms|when_i }}">{{ i.now_ms|ago_i }}</time>; this page refreshes itself every 30 seconds. Pickup is from when a job could run (queued, its dependencies passed, approved) to when a runner took it.</p>

</div>

{% endblock %}

/* Insights (oxygenhub#257): a part marked data-refresh="SECONDS"

   data-fragment="NAME" is fetched again on that interval while the page is

   visible, and swapped in place; never while someone is using a control in

   it. Without this file the page is a snapshot; a reload updates it. */

(function () {

  "use strict";

  var parts = document.querySelectorAll("[data-refresh][data-fragment]");

  if (!parts.length || !window.fetch || !window.DOMParser) return;

  var every = Math.max(10, parseInt(parts[0].getAttribute("data-refresh"), 10) || 30) * 1000;

  var busy = false;

  var inUse = function (el) {

    var a = document.activeElement;

    return a && el.contains(a) && /^(SELECT|INPUT|TEXTAREA)$/.test(a.tagName);

  };

  var tick = function () {

    if (busy || document.hidden) return;

    busy = true;

    fetch(location.href, { credentials: "same-origin", headers: { "X-Fragment": "page" } })

      .then(function (r) { return r.ok ? r.text() : null; })

      .then(function (html) {

        busy = false;

        if (html === null) return;

        var doc = new DOMParser().parseFromString(html, "text/html");

        Array.prototype.forEach.call(parts, function (el) {

          if (inUse(el)) return;

          var fresh = doc.querySelector('[data-fragment="' + el.getAttribute("data-fragment") + '"]');

          if (fresh) el.innerHTML = fresh.innerHTML;

        });

      })

      .catch(function () { busy = false; });

  };

  setInterval(tick, every);

  document.addEventListener("visibilitychange", function () { if (!document.hidden) tick(); });

})();

    /// A Grafana for this hub's metrics, linked from Insights (oxygenhub#257)

    /// for charts of one's own; empty, no link.

    pub grafana_url: String,

            grafana_url: String::new(),

    /// When it entered the queue (made, or run again), and when it became

    /// ready to run: set at its claim (migration 0080, oxygenhub#257).

    #[sqlx(default)]

    pub queued_ms: Option<i64>,

    #[sqlx(default)]

    pub ready_ms: Option<i64>,

            "INSERT INTO flow_jobs (run_id, name, depth, depends, spec, sandbox, runtime, status, need_disk_mib, approval, environment, priority, queued_ms) \

             VALUES ($1, $2, $3, $4, $5, $6, $7, 'queued', $8, $9, $10, $11, $12)",

        .bind(now)

        // When it became ready: queued, its dependencies passed, approved.

        let mut ready = j.queued_ms.unwrap_or(0).max(j.approved_ms.unwrap_or(0));

                let st: Option<(String, Option<i64>)> =

                    sqlx::query_as("SELECT status, finished_ms FROM flow_jobs WHERE run_id = $1 AND name = $2")

                        .bind(j.run_id)

                        .bind(d)

                        .fetch_optional(&mut *tx)

                        .await?;

                if let Some((status, finished)) = st {

                    if status == "passed" {

                        passed += 1;

                        ready = ready.max(finished.unwrap_or(0));

                    }

            "UPDATE flow_jobs SET status = 'running', runner = $1, token_hash = $2, started_ms = $3, waiting = '', ready_ms = $5 \

             WHERE id = $4 AND status = 'queued'",

        .bind(if ready > 0 { ready.min(now) } else { now })

             approved_ms = NULL, approval_seal = '', approved_via = '', queued_ms = $2, ready_ms = NULL WHERE id = $1",

        .bind(now_ms())

pub mod insights;

    // The transparency log, the metrics and Insights (oxygenhub#238, #212, #257).

    "log",

    "metrics",

    "insights",

pub mod charts;

mod insights;

        .route("/admin/insights", get(insights::site))

        .route("/insights/{owner}", get(insights::owner))

        .route("/{owner}/{name}/insights", get(insights::repo))

/// Insights (oxygenhub#257): a job is ready when it could run (queued, its

/// dependencies passed), so its pickup time is from then to its claim; the

/// repository's page charts its jobs, an owner's numbers are its own and

/// the site's are the administrators'.

#[tokio::test(flavor = "multi_thread")]

async fn insights_chart_pickup_from_when_a_job_could_run() {

    let h = hub().await;

    create(&h.app, "andy", "demo").await;

    let work = tempfile::tempdir().unwrap();

    let c = client(&format!("{}/andy/demo", h.base), work.path());

    write(work.path(), ".flow/pipeline.kdl", &pipeline(""));

    write(work.path(), "test.sh", "echo ok\n");

    c.snapshot("test").unwrap();

    c.commit("Add a pipeline", &[]).unwrap();

    assert!(c.push("origin", false).unwrap().iter().all(|r| r.ok));

    assert_eq!(runs_of(&h.app, "andy", "demo").await.len(), 1);

    let mut finished = std::collections::HashMap::new();

    for want in ["build", "lint", "test"] {

        let j = claim(&h.app).await.expect("a job");

        assert_eq!(j["job"]["name"].as_str(), Some(want));

        let id = j["job_id"].as_i64().unwrap();

        let row = db::flow::job(&h.app.db, id).await.unwrap().unwrap();

        let (queued, ready, started) = (row.queued_ms.unwrap(), row.ready_ms.unwrap(), row.started_ms.unwrap());

        assert!(queued <= ready && ready <= started, "{want}: {queued} {ready} {started}");

        if want == "test" {

            assert_eq!(Some(ready), finished["build"], "ready when build passed, not when it was queued");

        }

        let ph = j["pipeline_hash"].as_str().unwrap();

        let st = finish(&h.app, id, serde_json::json!({ "exit_code": 0, "duration_ms": 700, "receipt": receipt(want, ph, 0) })).await;

        assert_eq!(st, StatusCode::NO_CONTENT);

        finished.insert(want, db::flow::job(&h.app.db, id).await.unwrap().unwrap().finished_ms);

    }

    let (st, page) = call(&h.app, "GET", "/andy/demo/insights", None, None).await;

    assert_eq!(st, StatusCode::OK, "{page}");

    assert!(page.contains("<span class=\"kpi-v\">3</span><span class=\"kpi-l\">jobs finished"), "{page}");

    assert!(page.contains("100% passed"), "{page}");

    assert_eq!(page.matches("<svg class=\"chart\"").count(), 5, "five charts");

    assert!(page.contains("class=\"bar s-ok\""), "the passed jobs drawn");

    assert!(page.contains("data-refresh=\"30\" data-fragment=\"insights\""), "it refreshes itself");

    assert!(page.contains("<summary class=\"on\">More</summary>"), "the bar's More menu holds it");

    let (_, wall) = call(&h.app, "GET", "/andy/demo/insights?range=7d&wall=1", None, None).await;

    assert!(wall.contains("<body class=\"wall\"") && wall.contains("the last 7 days") && wall.contains("leave the wall view"), "{wall}");

    // An owner's numbers are not anyone's; the site's are administrators'.

    assert_eq!(call(&h.app, "GET", "/insights/andy", None, None).await.0, StatusCode::NOT_FOUND);

    assert_eq!(call(&h.app, "GET", "/admin/insights", None, None).await.0, StatusCode::NOT_FOUND);

    let res = oxygenhub::router(h.app.clone())

        .oneshot(

            Request::post("/register")

                .header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")

                .body(Body::from("username=andy&email=a%40example.com&password=correct-horse-battery&setup=test-setup"))

                .unwrap(),

        )

        .await

        .unwrap();

    let cookie = res.headers().get(header::SET_COOKIE).unwrap().to_str().unwrap().split(';').next().unwrap().to_string();

    let andy = db::accounts::account_by_name(&h.app.db, "andy").await.unwrap().unwrap();

    db::accounts::set_admin(&h.app.db, andy.id, true).await.unwrap();

    for path in ["/insights/andy", "/admin/insights?runner=test-1&range=1h"] {

        let res = oxygenhub::router(h.app.clone())

            .oneshot(Request::get(path).header(header::COOKIE, cookie.clone()).body(Body::empty()).unwrap())

            .await

            .unwrap();

        let st = res.status();

        let body = String::from_utf8_lossy(&res.into_body().collect().await.unwrap().to_bytes()).into_owned();

        assert_eq!(st, StatusCode::OK, "{path}: {body}");

        assert!(body.contains("<svg class=\"chart\""), "{path}");

    }

}

| `/{owner}/{name}/insights`, `/insights/{owner}`, `/admin/insights` | Insights (§13.5.6): OxygenFlow's charts for a repository (anyone who reads it), an owner (the owner, its members, administrators) or the site (administrators) |

### 13.5.6 Insights (oxygenhub#257)

The standard OxygenFlow charts, for people who will not build their own in Grafana: one page at three scopes, so the numbers always agree, the site (`/admin/insights`, administrators, filtered by repository, owner or runner), an owner (`/insights/{owner}`: the owner, the organisation's members, administrators) and a repository (`/{owner}/{name}/insights`, in its bar's More menu and linked from its Flow tab). `?range=` is `1h`, `24h` (the default) or `7d`. It shows:

- headline numbers: time to pickup (median, 95th percentile), jobs finished and the share that passed, the median job duration, the jobs queued and running now, runners online and their slots in use;

- charts: time to pickup (median and 95th percentile by the job's start), jobs finished per bucket by result, queued and running jobs over time, job duration, and runner utilisation (running jobs over the slots of the runners online now);

- tables: each runner (status, busy slots, jobs, failures, median pickup and duration, last heard from) and, for the site and an owner, the busiest repositories.

**Pickup** is from when a job could run to when a runner took it. A job is *queued* when it is made or run again (`queued_ms`) and *ready* at the latest of that, its dependencies passing and its approval (`ready_ms`, set at its claim; migration 0080, which estimates earlier jobs from their run's start and the earlier stages' ends). A job a runner hands back keeps its times: the person still waits. The charts are inline SVG drawn by the hub (no script or library; coloured by theme classes). The page refreshes its numbers every 30 seconds while visible (`data-refresh`), and is a snapshot without JavaScript. **Wall view** (`?wall=1`, opened in a new window from the page): no menus, larger numbers, for a second monitor. The runners page's queue links its filters to the site's Insights (**Charts ↗**). `[server] grafana_url`, when set, adds an *open in Grafana* link for one's own charts from `/metrics`.

    <a class="go end" href="/admin/insights?range=24h{% if !filter.repo.is_empty() %}&repo={{ filter.repo|urlencode }}{% endif %}{% if !filter.runner.is_empty() %}&runner={{ filter.runner|urlencode }}{% endif %}" target="_blank" rel="noopener" title="Pickup time, jobs, durations and runners, charted (opens a new window)">Charts ↗</a>

<div class="head"><h1>Flow <small>pipeline runs · <a href="{{ repo.url }}/deployments">deployments</a> · <a href="{{ repo.url }}/insights" target="_blank" rel="noopener" title="Pickup time, jobs, durations and runners, charted">charts ↗</a></small></h1>

      <summary{% if tab == "bookmarks" || tab == "diffusion" || tab == "trust" || tab == "pushes" || tab == "conflicts" || tab == "insights" %} class="on"{% endif %}>More</summary>

        <a href="{{ repo.url }}/insights"{% if tab == "insights" %} aria-current="page"{% endif %}>Insights</a>

/* Insights (oxygenhub#257): headline numbers, then charts drawn by the hub

   as inline SVG, coloured through currentColor from these classes. */

.insights-bar { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 14px; }

.insights-bar form.filters { margin: 0; }

.insights-links { margin-left: auto; display: flex; flex-wrap: wrap; gap: 14px; }

.kpis { display: grid; grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); gap: 12px; margin: 14px 0; }

.kpi { display: flex; flex-direction: column; gap: 2px; padding: 12px 14px; background: var(--panel); border: 1px solid var(--line); border-radius: 8px; min-width: 0; }

.kpi-v { font-size: 28px; font-weight: 600; line-height: 1.15; font-variant-numeric: tabular-nums; }

.kpi-l { font-size: 13px; }

.kpi-s { font-size: 12px; color: var(--mute); }

.kpi-warn { border-color: var(--warn); }

.kpi-bad { border-color: var(--bad); }

.chart-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 440px), 1fr)); gap: 12px; margin-bottom: 14px; }

.chart-card { margin: 0; padding: 10px 12px; background: var(--panel); border: 1px solid var(--line); border-radius: 8px; min-width: 0; }

.chart-card figcaption { font-weight: 600; margin-bottom: 4px; }

svg.chart { display: block; width: 100%; height: auto; }

svg.chart .grid { stroke: var(--line); stroke-width: 1; }

svg.chart .axis { fill: var(--mute); font-size: 11px; }

svg.chart .line { fill: none; stroke: currentColor; stroke-width: 2; stroke-linejoin: round; }

svg.chart .dot, svg.chart .bar { fill: currentColor; }

.s-main { color: var(--acc); }

.s-warn { color: var(--warn); }

.s-ok { color: var(--ok); }

.s-bad { color: var(--bad); }

.s-mute { color: var(--mute); }

.legend { display: flex; flex-wrap: wrap; gap: 4px 14px; font-size: 12px; margin-top: 4px; }

.legend .key { color: var(--ink); }

.legend .key::before { content: ""; display: inline-block; width: 10px; height: 10px; margin-right: 6px; border-radius: 2px; background: var(--acc); vertical-align: -1px; }

.legend .key.s-warn::before { background: var(--warn); }

.legend .key.s-ok::before { background: var(--ok); }

.legend .key.s-bad::before { background: var(--bad); }

.legend .key.s-mute::before { background: var(--mute); }

/* The wall view (?wall=1): for a second monitor, no menus, larger. */

body.wall > header, body.wall > .band, body.wall > footer, body.wall > .skip { display: none; }

body.wall main { max-width: none; padding: 18px 28px; }

body.wall .kpi-v { font-size: 40px; }

body.wall .kpi-l { font-size: 15px; }

body.wall .chart-grid { grid-template-columns: repeat(auto-fit, minmax(min(100%, 560px), 1fr)); }