| 8a48dd83 Andy McCall ↶ |
1 |
|
| 2 |
|
| 3 |
|
| 4 |
|
| 5 |
|
| 6 |
|
| 7 |
|
| 8 |
|
| 9 |
|
| 10 |
use crate::{db, Shared}; |
| 11 |
use axum::extract::State; |
| 12 |
use axum::http::{header, HeaderMap, StatusCode}; |
| 13 |
use axum::response::{IntoResponse, Response}; |
| 14 |
use std::fmt::Write; |
| 15 |
|
| 16 |
|
| 17 |
fn family(out: &mut String, name: &str, help: &str, kind: &str, samples: &[(String, f64)]) { |
| 18 |
let _ = writeln!(out, "# HELP {name} {help}"); |
| 19 |
let _ = writeln!(out, "# TYPE {name} {kind}"); |
| 20 |
for (labels, v) in samples { |
| 21 |
if labels.is_empty() { |
| 22 |
let _ = writeln!(out, "{name} {v}"); |
| 23 |
} else { |
| 24 |
let _ = writeln!(out, "{name}{{{labels}}} {v}"); |
| 25 |
} |
| 26 |
} |
| 27 |
} |
| 28 |
|
| 29 |
|
| 30 |
fn label(v: &str) -> String { |
| 31 |
v.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', "\\n") |
| 32 |
} |
| 33 |
|
| 34 |
|
| 35 |
fn resident_bytes() -> Option<f64> { |
| 36 |
let statm = std::fs::read_to_string("/proc/self/statm").ok()?; |
| 37 |
let pages: f64 = statm.split_whitespace().nth(1)?.parse().ok()?; |
| 38 |
Some(pages * 4096.0) |
| 39 |
} |
| 40 |
|
| 41 |
|
| 42 |
pub async fn render(app: &Shared) -> String { |
| 43 |
let mut out = String::new(); |
| 44 |
family( |
| 45 |
&mut out, |
| 46 |
"oxyhub_build_info", |
| 47 |
"The hub's version and build.", |
| 48 |
"gauge", |
| 49 |
&[(format!("version=\"{}\",build=\"{}\"", env!("CARGO_PKG_VERSION"), label(env!("OXY_BUILD_ID"))), 1.0)], |
| 50 |
); |
| 51 |
family( |
| 52 |
&mut out, |
| 53 |
"oxyhub_uptime_seconds", |
| 54 |
"Seconds since the hub started.", |
| 55 |
"gauge", |
| 56 |
&[(String::new(), app.started.elapsed().as_secs_f64())], |
| 57 |
); |
| 58 |
if let Some(rss) = resident_bytes() { |
| 59 |
family(&mut out, "oxyhub_resident_memory_bytes", "The process's resident memory.", "gauge", &[(String::new(), rss)]); |
| 60 |
} |
| 61 |
let (states, weight, summaries) = app.repos.cache().stats(); |
| 62 |
family(&mut out, "oxyhub_state_cache_states", "States the state cache holds.", "gauge", &[(String::new(), states as f64)]); |
| 63 |
family( |
| 64 |
&mut out, |
| 65 |
"oxyhub_state_cache_bytes", |
| 66 |
"What the state cache's states are estimated to weigh.", |
| 67 |
"gauge", |
| 68 |
&[(String::new(), weight as f64)], |
| 69 |
); |
| 70 |
family( |
| 71 |
&mut out, |
| 72 |
"oxyhub_state_cache_budget_bytes", |
| 73 |
"What the state cache may hold.", |
| 74 |
"gauge", |
| 75 |
&[(String::new(), crate::repos::cache::budget() as f64)], |
| 76 |
); |
| 77 |
family(&mut out, "oxyhub_state_cache_summaries", "Summaries the state cache holds.", "gauge", &[(String::new(), summaries as f64)]); |
| 78 |
let strata = oxy_strata::stats(); |
| 79 |
family( |
| 80 |
&mut out, |
| 81 |
"oxyhub_bucket_clients_opened_total", |
| 82 |
"Bucket store clients opened.", |
| 83 |
"counter", |
| 84 |
&[(String::new(), strata.opens as f64)], |
| 85 |
); |
| 86 |
family( |
| 87 |
&mut out, |
| 88 |
"oxyhub_bucket_requests_total", |
| 89 |
"Requests to bucket stores, by kind.", |
| 90 |
"counter", |
| 91 |
&strata.ops().map(|(k, n)| (format!("kind=\"{k}\""), n as f64)).collect::<Vec<_>>(), |
| 92 |
); |
| 93 |
family( |
| 94 |
&mut out, |
| 95 |
"oxyhub_bucket_request_seconds_total", |
| 96 |
"Time spent in requests to bucket stores.", |
| 97 |
"counter", |
| 98 |
&[(String::new(), strata.millis as f64 / 1000.0)], |
| 99 |
); |
| 100 |
let jobs: Vec<(String, i64)> = |
| 101 |
sqlx::query_as("SELECT status, COUNT(*) FROM flow_jobs WHERE status IN ('queued', 'running') GROUP BY status") |
| 102 |
.fetch_all(&app.db) |
| 103 |
.await |
| 104 |
.unwrap_or_default(); |
| 105 |
let count = |s: &str| jobs.iter().find(|(k, _)| k == s).map(|(_, n)| *n).unwrap_or(0) as f64; |
| 106 |
let waiting: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM flow_jobs WHERE status = 'queued' AND approval = 'waiting'") |
| 107 |
.fetch_one(&app.db) |
| 108 |
.await |
| 109 |
.unwrap_or(0); |
| 110 |
family( |
| 111 |
&mut out, |
| 112 |
"oxyhub_flow_jobs", |
| 113 |
"OxygenFlow jobs now, by state (waiting: for a person's approval).", |
| 114 |
"gauge", |
| 115 |
&[ |
| 116 |
("state=\"queued\"".into(), count("queued") - waiting as f64), |
| 117 |
("state=\"waiting\"".into(), waiting as f64), |
| 118 |
("state=\"running\"".into(), count("running")), |
| 119 |
], |
| 120 |
); |
| 121 |
let runners = db::runners::list(&app.db).await.unwrap_or_default(); |
| 122 |
family( |
| 123 |
&mut out, |
| 124 |
"oxyhub_flow_runners", |
| 125 |
"Registered OxygenFlow runners, by whether they reported in the last minutes.", |
| 126 |
"gauge", |
| 127 |
&[ |
| 128 |
("online=\"true\"".into(), runners.iter().filter(|r| r.online()).count() as f64), |
| 129 |
("online=\"false\"".into(), runners.iter().filter(|r| !r.online()).count() as f64), |
| 130 |
], |
| 131 |
); |
| 132 |
family(&mut out, "oxyhub_tlog_entries", "Entries in the transparency log.", "gauge", &[(String::new(), app.tlog.size() as f64)]); |
| 133 |
out |
| 134 |
} |
| 135 |
|
| 136 |
|
| 137 |
pub async fn metrics(State(app): State<Shared>, headers: HeaderMap) -> Response { |
| 138 |
let want = &app.config.server.metrics_token; |
| 139 |
if want.is_empty() { |
| 140 |
return (StatusCode::NOT_FOUND, "metrics are off: set [server] metrics_token in hub.toml\n").into_response(); |
| 141 |
} |
| 142 |
let given = oxy_server::bearer_token(&headers).unwrap_or(""); |
| 143 |
if !crate::auth::mac_equal(given, want) { |
| 144 |
return (StatusCode::UNAUTHORIZED, [(header::WWW_AUTHENTICATE, "Bearer")], "a bearer token is needed\n").into_response(); |
| 145 |
} |
| 146 |
([(header::CONTENT_TYPE, "text/plain; version=0.0.4; charset=utf-8"), (header::CACHE_CONTROL, "no-store")], render(&app).await) |
| 147 |
.into_response() |
| 148 |
} |