/metrics: HTTP requests by route class with latency histograms (refs #212) b9b74d997cec

    metrics::observe(method.as_str(), &path, res.status().as_u16(), took.as_secs_f64());

//! The process, the state cache, repositories kept in buckets, OxygenFlow's

//! jobs and runners, the transparency log, and HTTP requests by route class

//! and status with latency histograms. The rest the issue lists comes next.

use std::collections::HashMap;

use std::sync::Mutex;

/// Latency buckets, in seconds.

const BUCKETS: [f64; 12] = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0];

/// At most this many route classes are kept; more go to `other`.

const MAX_ROUTES: usize = 300;

#[derive(Default)]

struct Hist {

    /// Requests by status class (2xx … 5xx).

    by_status: [u64; 4],

    buckets: [u64; BUCKETS.len()],

    sum: f64,

    count: u64,

}

static HTTP: Mutex<Option<HashMap<(String, String), Hist>>> = Mutex::new(None);

/// A request's route class: a few dozen at most, so a label never holds a

/// user's or repository's name.

pub fn route_class(path: &str) -> String {

    let segs: Vec<&str> = path.trim_start_matches('/').split('/').filter(|s| !s.is_empty()).collect();

    let tab = |s: &str| s.chars().all(|c| c.is_ascii_lowercase() || c == '-' || c == '.') && s.len() <= 24;

    match segs.as_slice() {

        [] => "page:home".into(),

        ["static", ..] => "static".into(),

        ["api", "v1", area, ..] if tab(area) => format!("api:{area}"),

        ["api", ..] => "api".into(),

        ["log", ..] => "tlog".into(),

        [".well-known", ..] => "well-known".into(),

        ["live", ..] => "live".into(),

        ["avatar", ..] => "avatar".into(),

        [_, _, "oxy", "v1", endpoint, ..] if tab(endpoint) => format!("protocol:{endpoint}"),

        [_, _, "oxy", ..] => "protocol".into(),

        [_, _, t, ..] if tab(t) => format!("page:{t}"),

        [_, _] => "page:repository".into(),

        [one] if tab(one) => format!("page:{one}"),

        _ => "page:other".into(),

    }

}

/// Count a request (from the router's outermost timing layer).

pub fn observe(method: &str, path: &str, status: u16, seconds: f64) {

    let mut g = HTTP.lock().unwrap_or_else(|p| p.into_inner());

    let map = g.get_or_insert_with(HashMap::new);

    let mut route = route_class(path);

    if !map.contains_key(&(route.clone(), method.to_string())) && map.len() >= MAX_ROUTES {

        route = "other".into();

    }

    let h = map.entry((route, method.to_string())).or_default();

    let class = (status / 100).clamp(2, 5) as usize - 2;

    h.by_status[class] += 1;

    for (i, b) in BUCKETS.iter().enumerate() {

        if seconds <= *b {

            h.buckets[i] += 1;

        }

    }

    h.sum += seconds;

    h.count += 1;

}

fn http_families(out: &mut String) {

    let g = HTTP.lock().unwrap_or_else(|p| p.into_inner());

    let Some(map) = g.as_ref() else { return };

    let mut keys: Vec<&(String, String)> = map.keys().collect();

    keys.sort();

    let _ = writeln!(out, "# HELP oxyhub_http_requests_total HTTP requests by route class, method and status class.");

    let _ = writeln!(out, "# TYPE oxyhub_http_requests_total counter");

    for k in &keys {

        let h = &map[*k];

        for (i, n) in h.by_status.iter().enumerate().filter(|(_, n)| **n > 0) {

            let _ = writeln!(

                out,

                "oxyhub_http_requests_total{{route=\"{}\",method=\"{}\",status=\"{}xx\"}} {n}",

                label(&k.0),

                label(&k.1),

                i + 2

            );

        }

    }

    let _ = writeln!(out, "# HELP oxyhub_http_request_duration_seconds How long requests took, by route class and method.");

    let _ = writeln!(out, "# TYPE oxyhub_http_request_duration_seconds histogram");

    for k in &keys {

        let h = &map[*k];

        let l = format!("route=\"{}\",method=\"{}\"", label(&k.0), label(&k.1));

        for (i, b) in BUCKETS.iter().enumerate() {

            let _ = writeln!(out, "oxyhub_http_request_duration_seconds_bucket{{{l},le=\"{b}\"}} {}", h.buckets[i]);

        }

        let _ = writeln!(out, "oxyhub_http_request_duration_seconds_bucket{{{l},le=\"+Inf\"}} {}", h.count);

        let _ = writeln!(out, "oxyhub_http_request_duration_seconds_sum{{{l}}} {}", h.sum);

        let _ = writeln!(out, "oxyhub_http_request_duration_seconds_count{{{l}}} {}", h.count);

    }

}

    http_families(&mut out);

}

#[cfg(test)]

mod tests {

    #[test]

    fn route_classes_hold_no_names() {

        for (path, class) in [

            ("/", "page:home"),

            ("/static/hub.js", "static"),

            ("/api/v1/repos/andy/demo/issues", "api:repos"),

            ("/andy/demo/oxy/v1/negotiate", "protocol:negotiate"),

            ("/andy/demo/issue/3", "page:issue"),

            ("/andy/demo", "page:repository"),

            ("/explore", "page:explore"),

            ("/Andy_McCall", "page:other"),

            ("/log/tile/0/000", "tlog"),

        ] {

            assert_eq!(super::route_class(path), class, "{path}");

        }

    }

    // Requests through the router are counted by route class.

    call(&h.app, "GET", "/andy/demo", None, None).await;

    let text = oxygenhub::metrics::render(&h.app).await;

    assert!(text.contains("oxyhub_http_requests_total{route=\"page:repository\",method=\"GET\",status=\"2xx\"}"), "{text}");

    assert!(text.contains("oxyhub_http_request_duration_seconds_bucket{route=\"page:repository\",method=\"GET\",le=\"+Inf\"}"));

- the transparency log's entries;

- HTTP requests by route class (`page:<tab>`, `api:<area>`, `protocol:<endpoint>`, `static` and so on, never a user's or repository's name; at most 300 classes), method and status class, with latency histograms (5 ms to 30 s).

Pushes and fetches, backups, Diffusion, mail and the database pool are the next slices; the runner's own metrics too.