Metrics: OxygenFlow runs and runners' heartbeats, and mail 2c47eecfeab6
The fourth /metrics slice: runs finished in the last 24 hours by result, seconds since each runner last called the hub, and messages sent and failed since the hub started. Refs #212
- Author
- Andy McCall @andymccall <[email protected]>
- via
- agent
agent:claude@andymccall <[email protected]> · keyless, under your grant for changes,refs,seal · key f8a0faac8317
- When
- · 2026-10-18 05:17 UTC
- Signature
- ◈ sealed by agent:claude@andymccall
- Hash
2c47eecfeab69e364c03b79523eaeb3e72145307b46e8b0b36feaf30ae2713d5- Depends on
- 34beb1a7e376 3556d08e17f9 42fa95540468 8a48dd830178 93be9ce4653e a6d824c5ebf5 b9b74d997cec
- Depended on by
- 85ec74fd82d8
- In bookmarks
- api-unknown-fields cancel-landed-runs diffusion-pin-bookmark draft-ready main oh-comment-body-file proposal-list-fast
- In tags
- v0.6.0 v0.6.1 v0.7.0 v0.8.0 v0.9.0 v0.9.1 v0.9.11 v0.9.15 v0.9.16 v0.9.19 v0.9.2 v0.9.20 v0.9.21 v0.9.22 v0.9.23 v0.9.24 v0.9.3 v0.9.4 v0.9.5 v0.9.6 v0.9.7 v0.9.8 v0.9.9
- Edits
- 12 atoms, 0 tree operations, 4 files
Browse the tree at this change
modified crates/hub/src/mail.rs +15 −0
| … | 52 unchanged lines |
||
| 53 | 53 | if !enabled(app) { |
|
| 54 | 54 | return Err("this hub does not send mail".into()); |
|
| 55 | 55 | } |
|
| 56 | + | let sent = deliver(app, to_name, to_addr, subject, body, action, unsubscribe).await; |
|
| 57 | + | let counter = if sent.is_ok() { &crate::metrics::MAIL_SENT } else { &crate::metrics::MAIL_FAILED }; |
|
| 58 | + | counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed); |
|
| 59 | + | sent |
|
| 60 | + | } |
|
| 61 | + | ||
| 62 | + | async fn deliver( |
|
| 63 | + | app: &Shared, |
|
| 64 | + | to_name: &str, |
|
| 65 | + | to_addr: &str, |
|
| 66 | + | subject: &str, |
|
| 67 | + | body: &str, |
|
| 68 | + | action: Option<Action<'_>>, |
|
| 69 | + | unsubscribe: Option<&str>, |
|
| 70 | + | ) -> Result<(), String> { |
|
| 56 | 71 | let cfg = settings(app); |
|
| 57 | 72 | let subject = format!("[{}] {subject}", app.config.site.name); |
|
| 58 | 73 | if let Some(dir) = cfg.smtp.strip_prefix("file://") { |
|
| … | 18 unchanged lines |
modified crates/hub/src/metrics.rs +45 −4
| … | 3 unchanged lines |
||
| 4 | 4 | //! at all while that is unset. |
|
| 5 | 5 | //! |
|
| 6 | 6 | //! The process, the state cache, repositories kept in buckets, OxygenFlow's |
|
| 7 | - | //! jobs and runners, the transparency log, and HTTP requests by route class |
|
| 8 | - | //! and status with latency histograms. The rest the issue lists comes next. |
|
| 7 | + | //! jobs, runs and runners, the transparency log, the database pool, pushes, |
|
| 8 | + | //! backups, mail, and HTTP requests by route class and status with latency |
|
| 9 | + | //! histograms. The rest the issue lists comes next. |
|
| 9 | 10 | ||
| 10 | 11 | use crate::{db, Shared}; |
|
| 11 | 12 | use axum::extract::State; |
|
| … | 1 unchanged lines |
||
| 13 | 14 | use axum::response::{IntoResponse, Response}; |
|
| 14 | 15 | use std::collections::HashMap; |
|
| 15 | 16 | use std::fmt::Write; |
|
| 17 | + | use std::sync::atomic::{AtomicU64, Ordering}; |
|
| 16 | 18 | use std::sync::Mutex; |
|
| 17 | 19 | ||
| 20 | + | /// Messages the hub sent, and failed to send, since it started. |
|
| 21 | + | pub static MAIL_SENT: AtomicU64 = AtomicU64::new(0); |
|
| 22 | + | pub static MAIL_FAILED: AtomicU64 = AtomicU64::new(0); |
|
| 23 | + | ||
| 18 | 24 | /// Latency buckets, in seconds. |
|
| 19 | 25 | const BUCKETS: [f64; 12] = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0]; |
|
| 20 | 26 | /// At most this many route classes are kept; more go to `other`. |
|
| … | 201 unchanged lines |
||
| 222 | 228 | ("online=\"true\"".into(), runners.iter().filter(|r| r.online()).count() as f64), |
|
| 223 | 229 | ("online=\"false\"".into(), runners.iter().filter(|r| !r.online()).count() as f64), |
|
| 224 | 230 | ], |
|
| 231 | + | ); |
|
| 232 | + | let now = db::now_ms(); |
|
| 233 | + | family( |
|
| 234 | + | &mut out, |
|
| 235 | + | "oxyhub_flow_runner_last_seen_seconds", |
|
| 236 | + | "Seconds since each registered runner last called the hub.", |
|
| 237 | + | "gauge", |
|
| 238 | + | &runners |
|
| 239 | + | .iter() |
|
| 240 | + | .filter_map(|r| r.last_seen_ms.map(|t| (format!("runner=\"{}\"", label(&r.name)), (now - t).max(0) as f64 / 1000.0))) |
|
| 241 | + | .collect::<Vec<_>>(), |
|
| 242 | + | ); |
|
| 243 | + | let since = now - 86_400_000; |
|
| 244 | + | let runs: Vec<(String, i64)> = sqlx::query_as("SELECT status, COUNT(*) FROM flow_runs WHERE finished_ms > $1 GROUP BY status") |
|
| 245 | + | .bind(since) |
|
| 246 | + | .fetch_all(&app.db) |
|
| 247 | + | .await |
|
| 248 | + | .unwrap_or_default(); |
|
| 249 | + | family( |
|
| 250 | + | &mut out, |
|
| 251 | + | "oxyhub_flow_runs_last_day", |
|
| 252 | + | "OxygenFlow runs finished in the last 24 hours, by result.", |
|
| 253 | + | "gauge", |
|
| 254 | + | &["passed", "failed", "error"] |
|
| 255 | + | .iter() |
|
| 256 | + | .map(|r| (format!("result=\"{r}\""), runs.iter().find(|(s, _)| s == r).map(|(_, n)| *n).unwrap_or(0) as f64)) |
|
| 257 | + | .collect::<Vec<_>>(), |
|
| 225 | 258 | ); |
|
| 226 | 259 | family(&mut out, "oxyhub_tlog_entries", "Entries in the transparency log.", "gauge", &[(String::new(), app.tlog.size() as f64)]); |
|
| 227 | 260 | family( |
|
| 228 | 261 | &mut out, |
|
| 262 | + | "oxyhub_mail_total", |
|
| 263 | + | "Messages the hub sent and failed to send since it started.", |
|
| 264 | + | "counter", |
|
| 265 | + | &[ |
|
| 266 | + | ("result=\"sent\"".into(), MAIL_SENT.load(Ordering::Relaxed) as f64), |
|
| 267 | + | ("result=\"failed\"".into(), MAIL_FAILED.load(Ordering::Relaxed) as f64), |
|
| 268 | + | ], |
|
| 269 | + | ); |
|
| 270 | + | family( |
|
| 271 | + | &mut out, |
|
| 229 | 272 | "oxyhub_db_connections", |
|
| 230 | 273 | "The database pool's connections, by whether they are idle.", |
|
| 231 | 274 | "gauge", |
|
| … | 2 unchanged lines |
||
| 234 | 277 | ("idle=\"false\"".into(), (app.db.size() as f64 - app.db.num_idle() as f64).max(0.0)), |
|
| 235 | 278 | ], |
|
| 236 | 279 | ); |
|
| 237 | - | let since = db::now_ms() - 86_400_000; |
|
| 238 | 280 | let (ok, refused): (Option<i64>, Option<i64>) = |
|
| 239 | 281 | sqlx::query_as("SELECT SUM(CASE WHEN ok THEN 1 ELSE 0 END), SUM(CASE WHEN ok THEN 0 ELSE 1 END) FROM pushes WHERE at_ms > $1") |
|
| 240 | 282 | .bind(since) |
|
| … | 13 unchanged lines |
||
| 254 | 296 | .fetch_all(&app.db) |
|
| 255 | 297 | .await |
|
| 256 | 298 | .unwrap_or_default(); |
|
| 257 | - | let now = db::now_ms(); |
|
| 258 | 299 | family( |
|
| 259 | 300 | &mut out, |
|
| 260 | 301 | "oxyhub_backup_last_run_age_seconds", |
|
| … | 52 unchanged lines |
modified crates/hub/tests/flow.rs +3 −0
| … | 5306 unchanged lines |
||
| 5307 | 5307 | "oxyhub_db_connections{idle=\"true\"}", |
|
| 5308 | 5308 | "oxyhub_pushes_last_day{result=\"refused\"} 0", |
|
| 5309 | 5309 | "# TYPE oxyhub_backup_last_run_ok gauge", |
|
| 5310 | + | "# TYPE oxyhub_flow_runner_last_seen_seconds gauge", |
|
| 5311 | + | "oxyhub_flow_runs_last_day{result=\"failed\"} 0", |
|
| 5312 | + | "oxyhub_mail_total{result=\"sent\"}", |
|
| 5310 | 5313 | ] { |
|
| 5311 | 5314 | assert!(text.contains(want), "missing {want}:\n{text}"); |
|
| 5312 | 5315 | } |
|
| … | 203 unchanged lines |
modified docs/spec.md +3 −2
| … | 641 unchanged lines |
||
| 642 | 642 | - the build (`oxyhub_build_info`), uptime and resident memory; |
|
| 643 | 643 | - the state cache's states, estimated weight, budget and summaries; |
|
| 644 | 644 | - requests to bucket stores by kind (with pack listings sent and kept apart), and the time spent in them; |
|
| 645 | - | - OxygenFlow jobs queued, waiting for approval and running, and runners online and not; |
|
| 645 | + | - OxygenFlow jobs queued, waiting for approval and running; runs finished in the last 24 hours by result; runners online and not, and seconds since each last called; |
|
| 646 | 646 | - the transparency log's entries; |
|
| 647 | 647 | - the database pool's connections (idle and in use); |
|
| 648 | 648 | - reference updates pushed in the last 24 hours, accepted and refused; |
|
| 649 | 649 | - each owner's latest backup: seconds since it started, and whether it succeeded; |
|
| 650 | + | - mail sent and failed since the hub started; |
|
| 650 | 651 | - HTTP requests by route class (`page:<tab>`, `api:<area>`, `protocol:<endpoint>`, `static` and so on, never a user's or repository's name; at most 300 classes), method and status class, with latency histograms (5 ms to 30 s). |
|
| 651 | 652 | ||
| 652 | - | Push and fetch sizes and durations, Diffusion, mail and query latency are the next slices; the runner's own metrics too. |
|
| 653 | + | Push and fetch sizes, Diffusion, notifications and query latency are the next slices; the runner's own metrics too. |
|
| 653 | 654 | ||
| 654 | 655 | ### 13.6 Coverage reports (oxygenhub#113, done 2026-10-06) |
|
| 655 | 656 | ||
| … | 168 unchanged lines |