runners.rs 77.8%

383 of 492 lines run, 59 of 176 functions · run #517 on main · run missed
//! Admin → Runners (hub spec §13.2): the OxygenFlow runners of this hub,
//! what they are and are running, their history, registration tokens, and
//! draining, disabling and removing them.

use super::settings::csrf;
use super::{filters, Error, Page, Shell};
use crate::auth::Viewer;
use crate::db::{self, accounts, runners};
use crate::Shared;
use askama::Template;
use axum::extract::{Form, Path, Query, State};
use axum::response::{IntoResponse, Redirect, Response};
use serde::Deserialize;
use std::collections::HashMap;

fn require_admin(viewer: &Viewer, shell: &Shell) -> Result<accounts::Account, Error> {
    match viewer.account() {
        Some(a) if a.is_admin => Ok(a.clone()),
        _ => Err(Error::not_found(shell.clone())),
    }
}

pub struct RunnerRow {
    pub r: runners::Runner,
    /// Jobs it is running now.
    pub running: i64,
    /// The repositories it serves, in words.
    pub scope: String,
    /// Where its peers reach it in the image swarm, and its site
    /// (oxygenhub#144); empty when it is not in one.
    pub swarm: String,
    /// The jobs it runs now: (repository, run, job, started).
    pub jobs: Vec<(String, i64, String, Option<i64>)>,
}

impl RunnerRow {
    /// What it is doing (oxygenhub#243): `disabled`, `draining`, `idle`,
    /// or `running N of M`.
    pub fn doing(&self) -> String {
        match self.r.state.as_str() {
            "disabled" => "disabled".into(),
            "draining" if self.running > 0 => format!("draining, {} left", self.running),
            "draining" => "draining".into(),
            _ if self.running > 0 => format!("running {} of {}", self.running, self.r.max_jobs),
            _ => "idle".into(),
        }
    }

    /// The jobs it runs, for the status's tooltip.
    pub fn jobs_line(&self) -> String {
        let now = crate::db::now_ms();
        self.jobs
            .iter()
            .map(|(repo, run, job, started)| match started {
                Some(t) => format!("{repo} #{run} {job}, {} min", (now - t).max(0) / 60_000),
                None => format!("{repo} #{run} {job}"),
            })
            .collect::<Vec<_>>()
            .join("; ")
    }
}

/// The queue's filters (oxygenhub#243), kept in the URL.
#[derive(Deserialize, Default, Clone)]
pub struct QueueFilter {
    #[serde(default)]
    pub repo: String,
    #[serde(default)]
    pub status: String,
    #[serde(default)]
    pub runner: String,
}

impl QueueFilter {
    fn keeps(&self, q: &db::flow::QueueEntry) -> bool {
        (self.repo.is_empty() || format!("{}/{}", q.owner, q.name) == self.repo)
            && (self.status.is_empty() || q.status == self.status)
            && (self.runner.is_empty() || q.runner == self.runner)
    }

    pub fn any(&self) -> bool {
        !(self.repo.is_empty() && self.status.is_empty() && self.runner.is_empty())
    }
}

/// A scope in words: the whole site, an owner's repositories, or one
/// repository.
async fn scope_words(app: &Shared, s: &runners::Scope) -> String {
    match s.repo_id {
        Some(id) => match db::repos::get_by_id(&app.db, id).await.ok().flatten() {
            Some(r) => format!("{}/{}", r.owner, r.name),
            None => "a removed repository".into(),
        },
        None if s.owner.is_empty() => "the whole site".into(),
        None => format!("{}'s repositories", s.owner),
    }
}

/// A scope as typed: empty for the whole site, an owner, or
/// `owner/repository`.
async fn parse_scope(app: &Shared, text: &str) -> Result<runners::Scope, String> {
    let text = text.trim().trim_matches('/');
    if text.is_empty() {
        return Ok(runners::Scope::default());
    }
    if let Some((owner, name)) = text.split_once('/') {
        return match db::repos::get(&app.db, owner, name).await {
            Ok(Some(r)) => Ok(runners::Scope { owner: r.owner.clone(), repo_id: Some(r.id) }),
            _ => Err(format!("There is no repository {text}.")),
        };
    }
    let user = accounts::account_by_name(&app.db, text).await.ok().flatten().is_some();
    let org = accounts::org_by_name(&app.db, text).await.ok().flatten().is_some();
    if user || org {
        Ok(runners::Scope { owner: text.to_string(), repo_id: None })
    } else {
        Err(format!("There is no user or organisation {text}."))
    }
}

#[derive(Template)]
#[template(path = "admin_runners.html")]
pub struct RunnersPage {
    shell: Shell,
    runners: Vec<RunnerRow>,
    /// Registration tokens waiting to be used: (created by, created, expires).
    pending: Vec<(String, i64, i64)>,
    /// A token just made, shown once, with the command that uses it.
    new_token: Option<String>,
    hub: String,
    notice: Option<String>,
    error: Option<String>,
    /// Running jobs, then queued ones in the order runners claim them,
    /// as the filter keeps them.
    queue: Vec<db::flow::QueueEntry>,
    /// How many there are, unfiltered.
    queue_total: usize,
    filter: QueueFilter,
    /// The repositories and runners in the queue, for the filter's choices.
    queue_repos: Vec<String>,
    queue_runners: Vec<String>,
}

async fn page(
    app: &Shared,
    shell: Shell,
    new_token: Option<String>,
    notice: Option<String>,
    error: Option<String>,
) -> Result<Page<RunnersPage>, Error> {
    page_filtered(app, shell, new_token, notice, error, QueueFilter::default()).await
}

async fn page_filtered(
    app: &Shared,
    shell: Shell,
    new_token: Option<String>,
    notice: Option<String>,
    error: Option<String>,
    filter: QueueFilter,
) -> Result<Page<RunnersPage>, Error> {
    let e = |e| Error::from_db(shell.clone(), e);
    let list = runners::list(&app.db).await.map_err(e)?;
    let running: HashMap<i64, i64> = runners::running(&app.db).await.map_err(e)?.into_iter().collect();
    let pending = runners::pending_registrations(&app.db).await.map_err(e)?;
    let all = db::flow::queue(&app.db).await.map_err(e)?;
    let mut rows = Vec::with_capacity(list.len());
    for r in list {
        let scope = scope_words(app, &r.scope()).await;
        let swarm = app
            .swarm
            .addr(r.id)
            .map(|s| if s.site.is_empty() { s.addr } else { format!("{} · site {}", s.addr, s.site) })
            .unwrap_or_default();
        let jobs = all
            .iter()
            .filter(|q| q.status == "running" && q.runner == r.name)
            .map(|q| (format!("{}/{}", q.owner, q.name), q.run_number, q.job.clone(), q.started_ms))
            .collect();
        rows.push(RunnerRow { running: running.get(&r.id).copied().unwrap_or(0), scope, swarm, jobs, r });
    }
    let mut queue_repos: Vec<String> = all.iter().map(|q| format!("{}/{}", q.owner, q.name)).collect();
    queue_repos.sort();
    queue_repos.dedup();
    let mut queue_runners: Vec<String> = all.iter().filter(|q| !q.runner.is_empty()).map(|q| q.runner.clone()).collect();
    queue_runners.sort();
    queue_runners.dedup();
    let queue_total = all.len();
    let queue = all.into_iter().filter(|q| filter.keeps(q)).collect();
    Ok(Page(RunnersPage {
        hub: app.config.base_url().to_string(),
        shell,
        runners: rows,
        pending,
        new_token,
        notice,
        error,
        queue,
        queue_total,
        filter,
        queue_repos,
        queue_runners,
    }))
}

/// `GET /admin/runners`.
pub async fn list(State(app): State<Shared>, viewer: Viewer, Query(filter): Query<QueueFilter>) -> Result<Page<RunnersPage>, Error> {
    let shell = Shell::new(&app, "/admin/runners", &viewer);
    require_admin(&viewer, &shell)?;
    page_filtered(&app, shell, None, None, None, filter).await
}

#[derive(Deserialize)]
pub struct CsrfOnly {
    #[serde(rename = "_csrf")]
    csrf_token: String,
}

#[derive(Deserialize)]
pub struct ScopeForm {
    #[serde(rename = "_csrf")]
    csrf_token: String,
    /// Empty for the whole site, an owner, or `owner/repository`.
    #[serde(default)]
    scope: String,
}

/// `POST /admin/runners/token`: a one-time registration token, shown once,
/// for a runner of the scope given.
pub async fn new_token(State(app): State<Shared>, viewer: Viewer, Form(f): Form<ScopeForm>) -> Result<Page<RunnersPage>, Error> {
    let shell = Shell::new(&app, "/admin/runners", &viewer);
    let me = require_admin(&viewer, &shell)?;
    csrf(&viewer, &f.csrf_token, &shell)?;
    let scope = match parse_scope(&app, &f.scope).await {
        Ok(s) => s,
        Err(e) => return page(&app, shell, None, None, Some(e)).await,
    };
    let token = format!("flowreg_{}", crate::auth::random_hex(24));
    runners::add_registration(&app.db, &blake3::hash(token.as_bytes()).to_hex(), &me.username, &scope)
        .await
        .map_err(|e| Error::from_db(shell.clone(), e))?;
    let words = scope_words(&app, &scope).await;
    accounts::audit(&app.db, Some(&me), "flow.runner.token", "", &format!("registration token created for {words}")).await.ok();
    page(&app, shell, Some(token), Some(format!("The runner it registers will take jobs from {words}.")), None).await
}

/// `POST /admin/runners/{id}/scope`: change the repositories a runner serves.
pub async fn set_scope(
    State(app): State<Shared>,
    viewer: Viewer,
    Path(id): Path<i64>,
    Form(f): Form<ScopeForm>,
) -> Result<Page<RunnersPage>, Error> {
    let shell = Shell::new(&app, "/admin/runners", &viewer);
    let me = require_admin(&viewer, &shell)?;
    csrf(&viewer, &f.csrf_token, &shell)?;
    let r =
        runners::get(&app.db, id).await.map_err(|e| Error::from_db(shell.clone(), e))?.ok_or_else(|| Error::not_found(shell.clone()))?;
    let scope = match parse_scope(&app, &f.scope).await {
        Ok(s) => s,
        Err(e) => return page(&app, shell, None, None, Some(e)).await,
    };
    runners::set_scope(&app.db, id, &scope).await.map_err(|e| Error::from_db(shell.clone(), e))?;
    let words = scope_words(&app, &scope).await;
    accounts::audit(&app.db, Some(&me), "flow.runner.scope", &r.name, &words).await.ok();
    page(&app, shell, None, Some(format!("{} now takes jobs from {words}.", r.name)), None).await
}

#[derive(Deserialize)]
pub struct LimitsForm {
    #[serde(rename = "_csrf")]
    csrf_token: String,
    /// Comma-separated.
    #[serde(default)]
    labels: String,
    /// Minutes; empty for the site's limit.
    #[serde(default)]
    timeout_minutes: String,
}

/// `POST /admin/runners/{id}/limits`: the labels an administrator gives a
/// runner and its own job time limit (#98).
pub async fn set_limits(
    State(app): State<Shared>,
    viewer: Viewer,
    Path(id): Path<i64>,
    Form(f): Form<LimitsForm>,
) -> Result<Response, Error> {
    let shell = Shell::new(&app, &format!("/admin/runners/{id}"), &viewer);
    let me = require_admin(&viewer, &shell)?;
    csrf(&viewer, &f.csrf_token, &shell)?;
    let r =
        runners::get(&app.db, id).await.map_err(|e| Error::from_db(shell.clone(), e))?.ok_or_else(|| Error::not_found(shell.clone()))?;
    let mut labels: Vec<String> = Vec::new();
    for l in f.labels.split([',', ' ']).map(str::trim).filter(|l| !l.is_empty()) {
        if !l.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) || l.len() > 40 {
            return Err(Error::bad(shell, format!("label {l:?}: letters, digits, - _ and . only, up to 40 characters")));
        }
        if !labels.iter().any(|x| x == l) {
            labels.push(l.to_string());
        }
    }
    let timeout = match f.timeout_minutes.trim() {
        "" => None,
        t => match t.parse::<i64>() {
            Ok(m) if (1..=24 * 60).contains(&m) => Some(m),
            _ => return Err(Error::bad(shell, "a time limit is a whole number of minutes from 1 to 1440")),
        },
    };
    runners::set_limits(&app.db, id, &labels.join(","), timeout).await.map_err(|e| Error::from_db(shell.clone(), e))?;
    let words = format!(
        "labels {}; time limit {}",
        if labels.is_empty() { "none".to_string() } else { labels.join(", ") },
        timeout.map(|m| format!("{m} min")).unwrap_or_else(|| "the site's".into())
    );
    accounts::audit(&app.db, Some(&me), "flow.runner.limits", &r.name, &words).await.ok();
    Ok(Redirect::to(&format!("/admin/runners/{id}")).into_response())
}

#[derive(Deserialize)]
pub struct StateForm {
    #[serde(rename = "_csrf")]
    csrf_token: String,
    /// `active`, `draining` or `disabled`.
    state: String,
}

/// `POST /admin/runners/{id}/state`: drain, disable or resume a runner.
pub async fn set_state(
    State(app): State<Shared>,
    viewer: Viewer,
    Path(id): Path<i64>,
    Form(f): Form<StateForm>,
) -> Result<Response, Error> {
    let shell = Shell::new(&app, "/admin/runners", &viewer);
    let me = require_admin(&viewer, &shell)?;
    csrf(&viewer, &f.csrf_token, &shell)?;
    if !matches!(f.state.as_str(), "active" | "draining" | "disabled") {
        return Err(Error::bad(shell, "a runner is active, draining or disabled"));
    }
    let r =
        runners::get(&app.db, id).await.map_err(|e| Error::from_db(shell.clone(), e))?.ok_or_else(|| Error::not_found(shell.clone()))?;
    runners::set_state(&app.db, id, &f.state).await.map_err(|e| Error::from_db(shell.clone(), e))?;
    accounts::audit(&app.db, Some(&me), "flow.runner.state", &r.name, &f.state).await.ok();
    Ok(Redirect::to("/admin/runners").into_response())
}

/// `POST /admin/runners/{id}/remove`: forget a runner that runs nothing.
pub async fn remove(
    State(app): State<Shared>,
    viewer: Viewer,
    Path(id): Path<i64>,
    Form(f): Form<CsrfOnly>,
) -> Result<Page<RunnersPage>, Error> {
    let shell = Shell::new(&app, "/admin/runners", &viewer);
    let me = require_admin(&viewer, &shell)?;
    csrf(&viewer, &f.csrf_token, &shell)?;
    let e = |e| Error::from_db(shell.clone(), e);
    let r = runners::get(&app.db, id).await.map_err(e)?.ok_or_else(|| Error::not_found(shell.clone()))?;
    let busy = runners::running(&app.db).await.map_err(e)?.into_iter().any(|(rid, n)| rid == id && n > 0);
    if busy {
        return page(
            &app,
            shell,
            None,
            None,
            Some(format!("{} is running jobs: drain it first, then remove it when they finish.", r.name)),
        )
        .await;
    }
    runners::remove(&app.db, id).await.map_err(e)?;
    accounts::audit(&app.db, Some(&me), "flow.runner.remove", &r.name, "").await.ok();
    let also = if r.source == "config" {
        " It is configured in hub.toml; remove its token there too, or it comes back on its next claim."
    } else {
        ""
    };
    page(&app, shell, None, Some(format!("Removed {}.{also}", r.name)), None).await
}

#[derive(Template)]
#[template(path = "admin_runner.html")]
pub struct RunnerPage {
    shell: Shell,
    r: runners::Runner,
    running: i64,
    history: Vec<runners::RunnerJob>,
    declines: Vec<crate::db::flow::RunnerDecline>,
    older: Option<i64>,
    jobs: i64,
    passed: i64,
    busy_ms: i64,
}

impl RunnerPage {
    /// Time spent running jobs, as pages show durations.
    pub fn busy(&self) -> String {
        crate::flow::human_ms(self.busy_ms)
    }
}

#[derive(Deserialize, Default)]
pub struct HistoryQuery {
    before: Option<i64>,
}

const PER_PAGE: i64 = 50;

/// `GET /admin/runners/{id}`: a runner and the jobs it ran.
pub async fn show(
    State(app): State<Shared>,
    viewer: Viewer,
    Path(id): Path<i64>,
    Query(q): Query<HistoryQuery>,
) -> Result<Page<RunnerPage>, Error> {
    let shell = Shell::new(&app, &format!("/admin/runners/{id}"), &viewer);
    require_admin(&viewer, &shell)?;
    let e = |e| Error::from_db(shell.clone(), e);
    let r = runners::get(&app.db, id).await.map_err(e)?.ok_or_else(|| Error::not_found(shell.clone()))?;
    let mut history = runners::history(&app.db, id, PER_PAGE + 1, q.before).await.map_err(e)?;
    let older = (history.len() as i64 > PER_PAGE).then(|| history[PER_PAGE as usize - 1].started_ms).flatten();
    history.truncate(PER_PAGE as usize);
    let (jobs, passed, busy_ms) = runners::totals(&app.db, id).await.map_err(e)?;
    let running = runners::running(&app.db).await.map_err(e)?.into_iter().find(|(rid, _)| *rid == id).map(|(_, n)| n).unwrap_or(0);
    let declines = crate::db::flow::declines_of_runner(&app.db, id, 20).await.map_err(e)?;
    Ok(Page(RunnerPage { shell, r, running, history, declines, older, jobs, passed, busy_ms }))
}

pub fn duration(j: &runners::RunnerJob) -> String {
    match (j.started_ms, j.finished_ms) {
        (Some(a), Some(b)) => crate::flow::human_ms(b - a),
        (Some(a), None) => crate::flow::human_ms(db::now_ms() - a),
        _ => "–".into(),
    }
}

// ---------------------------------------------------------------- settings

#[derive(Template)]
#[template(path = "admin_runner_settings.html")]
pub struct SettingsPage {
    shell: Shell,
    default_m: u64,
    max_m: u64,
    grace_m: i64,
    /// What hub.toml says, shown beside a setting the page overrides.
    file_default_m: u64,
    flow_repos: Vec<String>,
    /// The site's priority for jobs (oxygenhub#244): low, normal or high.
    priority: String,
    /// The most expedited runs one owner may have going at once.
    expedite_cap: i64,
    notice: Option<String>,
    error: Option<String>,
}

async fn settings_page(app: &Shared, shell: Shell, notice: Option<String>, error: Option<String>) -> Page<SettingsPage> {
    let l = crate::flow::limits(app).await;
    Page(SettingsPage {
        shell,
        default_m: l.default_s / 60,
        max_m: l.max_s / 60,
        grace_m: l.grace_ms / 60_000,
        file_default_m: app.config.flow.job_timeout_minutes,
        flow_repos: app.config.flow.repos.clone(),
        expedite_cap: accounts::setting(&app.db, "flow_expedite_cap").await.ok().flatten().and_then(|v| v.parse().ok()).unwrap_or(2),
        priority: accounts::setting(&app.db, "flow_priority")
            .await
            .ok()
            .flatten()
            .filter(|p| !p.is_empty())
            .unwrap_or_else(|| "normal".into()),
        notice,
        error,
    })
}

#[derive(Deserialize)]
pub struct PriorityForm {
    #[serde(rename = "_csrf")]
    csrf_token: String,
    priority: String,
    #[serde(default)]
    expedite_cap: Option<String>,
}

/// `POST /admin/runners/priority` (oxygenhub#244): the level every job has
/// unless its organisation, repository or run says otherwise.
pub async fn save_priority(State(app): State<Shared>, viewer: Viewer, Form(f): Form<PriorityForm>) -> Result<Page<SettingsPage>, Error> {
    let shell = Shell::new(&app, "/admin/runners/settings", &viewer);
    let me = require_admin(&viewer, &shell)?;
    csrf(&viewer, &f.csrf_token, &shell)?;
    if !["low", "normal", "high"].contains(&f.priority.as_str()) {
        return Ok(settings_page(&app, shell, None, Some("Pick low, normal or high (urgent is for expediting one run).".into())).await);
    }
    let cap = match f.expedite_cap.as_deref().map(str::trim) {
        None | Some("") => None,
        Some(v) => match v.parse::<i64>() {
            Ok(n) if (0..=100).contains(&n) => Some(n),
            _ => return Ok(settings_page(&app, shell, None, Some("The cap is a whole number from 0 to 100.".into())).await),
        },
    };
    accounts::set_setting(&app.db, "flow_priority", &f.priority).await.map_err(|e| Error::from_db(shell.clone(), e))?;
    if let Some(n) = cap {
        accounts::set_setting(&app.db, "flow_expedite_cap", &n.to_string()).await.map_err(|e| Error::from_db(shell.clone(), e))?;
    }
    accounts::audit(&app.db, Some(&me), "flow.priority", "site", &f.priority).await.ok();
    Ok(settings_page(
        &app,
        shell,
        Some(format!("Saved: jobs are {} unless their organisation, repository or run says otherwise.", f.priority)),
        None,
    )
    .await)
}

/// `GET /admin/runners/settings` (oxygenhub#89).
pub async fn settings(State(app): State<Shared>, viewer: Viewer) -> Result<Page<SettingsPage>, Error> {
    let shell = Shell::new(&app, "/admin/runners/settings", &viewer);
    require_admin(&viewer, &shell)?;
    Ok(settings_page(&app, shell, None, None).await)
}

#[derive(Deserialize)]
pub struct SettingsForm {
    #[serde(rename = "_csrf")]
    csrf_token: String,
    default_minutes: String,
    max_minutes: String,
    grace_minutes: String,
}

/// `POST /admin/runners/settings`: the site's job time limits.
pub async fn save_settings(State(app): State<Shared>, viewer: Viewer, Form(f): Form<SettingsForm>) -> Result<Page<SettingsPage>, Error> {
    let shell = Shell::new(&app, "/admin/runners/settings", &viewer);
    let me = require_admin(&viewer, &shell)?;
    csrf(&viewer, &f.csrf_token, &shell)?;
    let num = |v: &str| v.trim().parse::<u64>().ok().filter(|n| *n > 0);
    let (Some(d), Some(m), Some(g)) = (num(&f.default_minutes), num(&f.max_minutes), num(&f.grace_minutes)) else {
        return Ok(settings_page(&app, shell, None, Some("Each limit is a whole number of minutes, at least 1.".into())).await);
    };
    if d > m {
        return Ok(settings_page(&app, shell, None, Some("The default cannot be longer than the maximum.".into())).await);
    }
    if m > 24 * 60 {
        return Ok(settings_page(&app, shell, None, Some("A job may run for a day at most (1440 minutes).".into())).await);
    }
    let e = |e| Error::from_db(shell.clone(), e);
    accounts::set_setting(&app.db, crate::flow::SETTING_DEFAULT, &d.to_string()).await.map_err(e)?;
    accounts::set_setting(&app.db, crate::flow::SETTING_MAX, &m.to_string()).await.map_err(e)?;
    accounts::set_setting(&app.db, crate::flow::SETTING_GRACE, &g.to_string()).await.map_err(e)?;
    accounts::audit(
        &app.db,
        Some(&me),
        "flow.settings",
        "",
        &format!("job time limit {d} min by default, {m} min at most; lost after {g} min more"),
    )
    .await
    .ok();
    Ok(settings_page(&app, shell, Some("Saved. Jobs claimed from now on run under these limits.".into()), None).await)
}

/// Where one image is: the hub, a runner holding it, or a runner fetching
/// it from the swarm now.
pub struct Place {
    pub name: String,
    /// `hub`, `holds` or `fetching`.
    pub kind: &'static str,
    pub title: String,
}

pub struct ImageRow {
    pub hash: String,
    pub arch: String,
    /// Its size, when the hub keeps it (runners do not say).
    pub size: Option<i64>,
    pub places: Vec<Place>,
    /// Who sent it to the hub, and when.
    pub sent: Option<(String, i64)>,
    /// Repositories whose jobs ran on it in the last 90 days.
    pub uses: Vec<(String, i64, i64)>,
    pub last_used: i64,
}

impl ImageRow {
    /// Nowhere a job could get it: not on the hub, on no runner.
    /// Nowhere a job could get it: nobody holds it, and the hub has at most
    /// its manifest (oxygenflow#94).
    pub fn stranded(&self) -> bool {
        self.places.iter().all(|p| p.kind == "peer")
    }
}

#[derive(Template)]
#[template(path = "admin_images.html")]
pub struct ImagesPage {
    shell: Shell,
    images: Vec<ImageRow>,
    on_hub: usize,
    /// Runners in the image swarm, as "name (site)".
    swarm: Vec<String>,
    /// The swarm backdrop's data (docs/backgrounds.md §2): the hub and the
    /// runners, and for each image who holds it and who fetches it now.
    backdrop: String,
}

/// The swarm's data, `{scene, nodes: [name], images: [{held, fetching}]}`,
/// the hub first, from the page's own rows.
fn swarm_json(images: &[ImageRow]) -> String {
    let mut nodes: Vec<String> = vec!["hub".into()];
    let mut list = Vec::new();
    for img in images {
        let (mut held, mut fetching) = (Vec::new(), Vec::new());
        for p in &img.places {
            let i = match p.kind {
                "hub" => 0,
                _ => match nodes.iter().position(|n| *n == p.name) {
                    Some(i) => i,
                    None => {
                        nodes.push(p.name.clone());
                        nodes.len() - 1
                    }
                },
            };
            if p.kind == "fetching" {
                fetching.push(i);
            } else {
                held.push(i);
            }
        }
        list.push(serde_json::json!({ "held": held, "fetching": fetching }));
    }
    serde_json::json!({ "scene": "swarm", "nodes": nodes, "images": list }).to_string().replace('<', "\\u003c")
}

/// `GET /admin/runners/images` (C0, docs/cylinder.md): every runner image,
/// where it is and what uses it.
pub async fn images(State(app): State<Shared>, viewer: Viewer) -> Result<Page<ImagesPage>, Error> {
    let shell = Shell::new(&app, "/admin/runners/images", &viewer);
    require_admin(&viewer, &shell)?;
    let e = |e| Error::from_db(shell.clone(), e);
    let kept = crate::images::kept(&app.db).await.map_err(e)?;
    let uses = crate::images::uses(&app.db, db::now_ms() - 90 * 24 * 3600 * 1000).await.map_err(e)?;
    let archs = crate::images::reported_archs(&app.db).await.map_err(e)?;
    let list = runners::list(&app.db).await.map_err(e)?;
    let fetching = app.swarm.fetching_now();
    let mut rows: HashMap<String, ImageRow> = HashMap::new();
    let row = |rows: &mut HashMap<String, ImageRow>, hash: &str| -> () {
        rows.entry(hash.to_ascii_lowercase()).or_insert_with(|| ImageRow {
            hash: hash.to_ascii_lowercase(),
            arch: archs.get(&hash.to_ascii_lowercase()).cloned().unwrap_or_default(),
            size: None,
            places: Vec::new(),
            sent: None,
            uses: Vec::new(),
            last_used: 0,
        });
    };
    for k in &kept {
        row(&mut rows, &k.hash);
        let r = rows.get_mut(&k.hash.to_ascii_lowercase()).expect("added");
        r.size = Some(k.size);
        if r.arch.is_empty() {
            r.arch = k.arch.clone();
        }
        r.sent = Some((k.uploader.clone(), k.created_ms));
        // An image only runners hold (oxygenflow#94): the hub keeps its
        // manifest, and the runners holding it serve its chunks.
        r.places.push(if k.source == "peer" {
            Place {
                name: "manifest".into(),
                kind: "peer",
                title: format!("The hub keeps only its manifest, sent by {}; runners holding it serve its {} chunks", k.holder, k.chunks),
            }
        } else {
            Place { name: "hub".into(), kind: "hub", title: format!("{} chunks kept by the hub", k.chunks) }
        });
    }
    let mut swarm = Vec::new();
    for runner in &list {
        if let Some(a) = app.swarm.addr(runner.id) {
            swarm.push(if a.site.is_empty() { runner.name.clone() } else { format!("{} ({})", runner.name, a.site) });
        }
        for h in runner.image_list().unwrap_or_default() {
            row(&mut rows, &h);
            rows.get_mut(&h.to_ascii_lowercase()).expect("added").places.push(Place {
                name: runner.name.clone(),
                kind: "holds",
                title: format!("{} holds it ({})", runner.name, runner.status()),
            });
        }
    }
    for (image, who) in &fetching {
        row(&mut rows, image);
        for id in who {
            if let Some(runner) = list.iter().find(|r| r.id == *id) {
                rows.get_mut(image).expect("added").places.push(Place {
                    name: runner.name.clone(),
                    kind: "fetching",
                    title: format!("{} is fetching it from the swarm now", runner.name),
                });
            }
        }
    }
    for u in uses {
        if !jet::images::is_hash(&u.image) {
            continue;
        }
        row(&mut rows, &u.image);
        let r = rows.get_mut(&u.image).expect("added");
        r.last_used = r.last_used.max(u.last_ms);
        r.uses.push((format!("{}/{}", u.owner, u.name), u.jobs, u.last_ms));
    }
    let mut images: Vec<ImageRow> = rows.into_values().collect();
    images.sort_by(|a, b| b.last_used.cmp(&a.last_used).then_with(|| a.hash.cmp(&b.hash)));
    let on_hub = kept.len();
    let backdrop = swarm_json(&images);
    Ok(Page(ImagesPage { shell, images, on_hub, swarm, backdrop }))
}
`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet
Coverage / crates/hub/src/web/runners.rs 77.8%

Keyboard
