OXYGENHUB
# c30444e35f2e oxygenhub / crates / hub / src / metrics.rs
5.8 KiB · Rust · last changed Live L4: the bell, the inbox and the dashboard (closes #206); /metrics, first slice (refs #212) 18 days ago source blame history raw
8a48dd83 Andy McCall ↶ 1
//! `/metrics` (oxygenhub#212): the hub's numbers in the Prometheus text
2
//! format, for a monitoring stack (oxygenhub-gcp's Prometheus and Grafana).
3
//! Private: served only to a bearer of `[server] metrics_token`, and not
4
//! at all while that is unset.
5
//!
6
//! First slice: the process, the state cache, repositories kept in buckets,
7
//! OxygenFlow's jobs and runners, and the transparency log. Per-route
8
//! request histograms and the rest the issue lists come next.
9
10
use crate::{db, Shared};
11
use axum::extract::State;
12
use axum::http::{header, HeaderMap, StatusCode};
13
use axum::response::{IntoResponse, Response};
14
use std::fmt::Write;
15
16
/// One metric family: its help, type and samples (label set, value).
17
fn family(out: &mut String, name: &str, help: &str, kind: &str, samples: &[(String, f64)]) {
18
    let _ = writeln!(out, "# HELP {name} {help}");
19
    let _ = writeln!(out, "# TYPE {name} {kind}");
20
    for (labels, v) in samples {
21
        if labels.is_empty() {
22
            let _ = writeln!(out, "{name} {v}");
23
        } else {
24
            let _ = writeln!(out, "{name}{{{labels}}} {v}");
25
        }
26
    }
27
}
28
29
/// A label value, escaped as the text format asks.
30
fn label(v: &str) -> String {
31
    v.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', "\\n")
32
}
33
34
/// The process's resident memory, from /proc (Linux), in bytes.
35
fn resident_bytes() -> Option<f64> {
36
    let statm = std::fs::read_to_string("/proc/self/statm").ok()?;
37
    let pages: f64 = statm.split_whitespace().nth(1)?.parse().ok()?;
38
    Some(pages * 4096.0)
39
}
40
41
/// The metrics as text, for whoever is let in.
42
pub async fn render(app: &Shared) -> String {
43
    let mut out = String::new();
44
    family(
45
        &mut out,
46
        "oxyhub_build_info",
47
        "The hub's version and build.",
48
        "gauge",
49
        &[(format!("version=\"{}\",build=\"{}\"", env!("CARGO_PKG_VERSION"), label(env!("OXY_BUILD_ID"))), 1.0)],
50
    );
51
    family(
52
        &mut out,
53
        "oxyhub_uptime_seconds",
54
        "Seconds since the hub started.",
55
        "gauge",
56
        &[(String::new(), app.started.elapsed().as_secs_f64())],
57
    );
58
    if let Some(rss) = resident_bytes() {
59
        family(&mut out, "oxyhub_resident_memory_bytes", "The process's resident memory.", "gauge", &[(String::new(), rss)]);
60
    }
61
    let (states, weight, summaries) = app.repos.cache().stats();
62
    family(&mut out, "oxyhub_state_cache_states", "States the state cache holds.", "gauge", &[(String::new(), states as f64)]);
63
    family(
64
        &mut out,
65
        "oxyhub_state_cache_bytes",
66
        "What the state cache's states are estimated to weigh.",
67
        "gauge",
68
        &[(String::new(), weight as f64)],
69
    );
70
    family(
71
        &mut out,
72
        "oxyhub_state_cache_budget_bytes",
73
        "What the state cache may hold.",
74
        "gauge",
75
        &[(String::new(), crate::repos::cache::budget() as f64)],
76
    );
77
    family(&mut out, "oxyhub_state_cache_summaries", "Summaries the state cache holds.", "gauge", &[(String::new(), summaries as f64)]);
78
    let strata = oxy_strata::stats();
79
    family(
80
        &mut out,
81
        "oxyhub_bucket_clients_opened_total",
82
        "Bucket store clients opened.",
83
        "counter",
84
        &[(String::new(), strata.opens as f64)],
85
    );
86
    family(
87
        &mut out,
88
        "oxyhub_bucket_requests_total",
89
        "Requests to bucket stores, by kind.",
90
        "counter",
91
        &strata.ops().map(|(k, n)| (format!("kind=\"{k}\""), n as f64)).collect::<Vec<_>>(),
92
    );
93
    family(
94
        &mut out,
95
        "oxyhub_bucket_request_seconds_total",
96
        "Time spent in requests to bucket stores.",
97
        "counter",
98
        &[(String::new(), strata.millis as f64 / 1000.0)],
99
    );
100
    let jobs: Vec<(String, i64)> =
101
        sqlx::query_as("SELECT status, COUNT(*) FROM flow_jobs WHERE status IN ('queued', 'running') GROUP BY status")
102
            .fetch_all(&app.db)
103
            .await
104
            .unwrap_or_default();
105
    let count = |s: &str| jobs.iter().find(|(k, _)| k == s).map(|(_, n)| *n).unwrap_or(0) as f64;
106
    let waiting: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM flow_jobs WHERE status = 'queued' AND approval = 'waiting'")
107
        .fetch_one(&app.db)
108
        .await
109
        .unwrap_or(0);
110
    family(
111
        &mut out,
112
        "oxyhub_flow_jobs",
113
        "OxygenFlow jobs now, by state (waiting: for a person's approval).",
114
        "gauge",
115
        &[
116
            ("state=\"queued\"".into(), count("queued") - waiting as f64),
117
            ("state=\"waiting\"".into(), waiting as f64),
118
            ("state=\"running\"".into(), count("running")),
119
        ],
120
    );
121
    let runners = db::runners::list(&app.db).await.unwrap_or_default();
122
    family(
123
        &mut out,
124
        "oxyhub_flow_runners",
125
        "Registered OxygenFlow runners, by whether they reported in the last minutes.",
126
        "gauge",
127
        &[
128
            ("online=\"true\"".into(), runners.iter().filter(|r| r.online()).count() as f64),
129
            ("online=\"false\"".into(), runners.iter().filter(|r| !r.online()).count() as f64),
130
        ],
131
    );
132
    family(&mut out, "oxyhub_tlog_entries", "Entries in the transparency log.", "gauge", &[(String::new(), app.tlog.size() as f64)]);
133
    out
134
}
135
136
/// `GET /metrics`.
137
pub async fn metrics(State(app): State<Shared>, headers: HeaderMap) -> Response {
138
    let want = &app.config.server.metrics_token;
139
    if want.is_empty() {
140
        return (StatusCode::NOT_FOUND, "metrics are off: set [server] metrics_token in hub.toml\n").into_response();
141
    }
142
    let given = oxy_server::bearer_token(&headers).unwrap_or("");
143
    if !crate::auth::mac_equal(given, want) {
144
        return (StatusCode::UNAUTHORIZED, [(header::WWW_AUTHENTICATE, "Bearer")], "a bearer token is needed\n").into_response();
145
    }
146
    ([(header::CONTENT_TYPE, "text/plain; version=0.0.4; charset=utf-8"), (header::CACHE_CONTROL, "no-store")], render(&app).await)
147
        .into_response()
148
}