OXYGENHUB
◆ v0.5.1 oxygenflow / crates / flow / src / metrics.rs
6.1 KiB · Rust · last changed A runner's own metrics (refs #35) 14 days ago source blame history raw
1
//! The runner's own metrics (oxygenflow#35, oxygenhub#212): a Prometheus
2
//! text page on an address of the runner's choosing, off unless
3
//! `metrics = "127.0.0.1:9464"` is set under `[runner]`. With
4
//! `metrics_token`, a scrape must send it as a bearer token.
5
6
use std::io::{BufRead, BufReader, Write};
7
use std::sync::atomic::{AtomicU64, Ordering::Relaxed};
8
use std::time::Instant;
9
10
/// Jobs this runner claimed.
11
pub static CLAIMS: AtomicU64 = AtomicU64::new(0);
12
/// Jobs declined before running (the preflight said no).
13
pub static DECLINED: AtomicU64 = AtomicU64::new(0);
14
/// Finished jobs by result.
15
pub static PASSED: AtomicU64 = AtomicU64::new(0);
16
pub static FAILED: AtomicU64 = AtomicU64::new(0);
17
pub static ERRORS: AtomicU64 = AtomicU64::new(0);
18
pub static CANCELLED: AtomicU64 = AtomicU64::new(0);
19
/// Milliseconds finished jobs ran, together.
20
pub static JOB_MS: AtomicU64 = AtomicU64::new(0);
21
/// Jobs running now.
22
pub static RUNNING: AtomicU64 = AtomicU64::new(0);
23
24
/// Count a job's end: by exit code, its error, and how long it ran.
25
pub fn finished(exit_code: Option<i32>, error: Option<&str>, duration_ms: u64, cancelled: bool) {
26
    let counter = if cancelled {
27
        &CANCELLED
28
    } else if error.is_some() {
29
        &ERRORS
30
    } else if exit_code == Some(0) {
31
        &PASSED
32
    } else {
33
        &FAILED
34
    };
35
    counter.fetch_add(1, Relaxed);
36
    JOB_MS.fetch_add(duration_ms, Relaxed);
37
}
38
39
/// The page.
40
pub fn render(name: &str, started: Instant) -> String {
41
    let n = name.replace('\\', "\\\\").replace('"', "\\\"");
42
    let mut out = String::new();
43
    let mut family = |metric: &str, help: &str, kind: &str, samples: &[(String, u64)]| {
44
        out.push_str(&format!("# HELP {metric} {help}\n# TYPE {metric} {kind}\n"));
45
        for (labels, v) in samples {
46
            let sep = if labels.is_empty() { "" } else { "," };
47
            out.push_str(&format!("{metric}{{runner=\"{n}\"{sep}{labels}}} {v}\n"));
48
        }
49
    };
50
    family("flow_runner_build_info", "This runner's version.", "gauge", &[(format!("version=\"{}\"", env!("CARGO_PKG_VERSION")), 1)]);
51
    family("flow_runner_uptime_seconds", "Seconds since the runner started.", "gauge", &[(String::new(), started.elapsed().as_secs())]);
52
    family("flow_runner_claims_total", "Jobs claimed from the hub.", "counter", &[(String::new(), CLAIMS.load(Relaxed))]);
53
    family("flow_runner_declined_total", "Claimed jobs declined before running.", "counter", &[(String::new(), DECLINED.load(Relaxed))]);
54
    family(
55
        "flow_runner_jobs_finished_total",
56
        "Finished jobs by result.",
57
        "counter",
58
        &[
59
            ("result=\"passed\"".into(), PASSED.load(Relaxed)),
60
            ("result=\"failed\"".into(), FAILED.load(Relaxed)),
61
            ("result=\"error\"".into(), ERRORS.load(Relaxed)),
62
            ("result=\"cancelled\"".into(), CANCELLED.load(Relaxed)),
63
        ],
64
    );
65
    family(
66
        "flow_runner_job_seconds_total",
67
        "Seconds finished jobs ran, together.",
68
        "counter",
69
        &[(String::new(), JOB_MS.load(Relaxed) / 1000)],
70
    );
71
    family("flow_runner_jobs_running", "Jobs running now.", "gauge", &[(String::new(), RUNNING.load(Relaxed))]);
72
    out
73
}
74
75
/// Serve the page at `addr` on a thread of its own; a failure to listen is
76
/// said and the runner goes on without it.
77
pub fn serve(addr: &str, token: Option<String>, name: String) {
78
    let listener = match std::net::TcpListener::bind(addr) {
79
        Ok(l) => l,
80
        Err(e) => {
81
            eprintln!("flow runner {name}: not serving metrics on {addr}: {e}");
82
            return;
83
        }
84
    };
85
    eprintln!("flow runner {name}: metrics on http://{addr}/metrics");
86
    let started = Instant::now();
87
    std::thread::spawn(move || {
88
        for stream in listener.incoming().flatten() {
89
            let _ = answer(stream, token.as_deref(), &name, started);
90
        }
91
    });
92
}
93
94
fn answer(mut stream: std::net::TcpStream, token: Option<&str>, name: &str, started: Instant) -> std::io::Result<()> {
95
    stream.set_read_timeout(Some(std::time::Duration::from_secs(5)))?;
96
    let mut reader = BufReader::new(stream.try_clone()?);
97
    let mut line = String::new();
98
    reader.read_line(&mut line)?;
99
    let path = line.split_whitespace().nth(1).unwrap_or("").to_string();
100
    let mut authorized = token.is_none();
101
    loop {
102
        let mut h = String::new();
103
        if reader.read_line(&mut h)? == 0 || h.trim().is_empty() {
104
            break;
105
        }
106
        if let (Some(t), Some((k, v))) = (token, h.split_once(':')) {
107
            if k.trim().eq_ignore_ascii_case("authorization") && v.trim() == format!("Bearer {t}") {
108
                authorized = true;
109
            }
110
        }
111
    }
112
    let (status, body) = match (path.as_str(), authorized) {
113
        ("/metrics", true) => ("200 OK", render(name, started)),
114
        ("/metrics", false) => ("401 Unauthorized", "a bearer token is needed\n".to_string()),
115
        _ => ("404 Not Found", "not found\n".to_string()),
116
    };
117
    write!(
118
        stream,
119
        "HTTP/1.1 {status}\r\nContent-Type: text/plain; version=0.0.4\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
120
        body.len()
121
    )
122
}
123
124
#[cfg(test)]
125
mod tests {
126
    use super::*;
127
    use std::io::Read;
128
129
    #[test]
130
    fn a_scrape_needs_the_token_and_reads_the_counters() {
131
        let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
132
        let addr = l.local_addr().unwrap().to_string();
133
        drop(l);
134
        serve(&addr, Some("t0k".into()), "earth-1".into());
135
        CLAIMS.fetch_add(1, Relaxed);
136
        finished(Some(0), None, 2500, false);
137
        let get = |auth: &str| {
138
            let mut s = std::net::TcpStream::connect(&addr).unwrap();
139
            write!(s, "GET /metrics HTTP/1.1\r\nHost: x\r\n{auth}\r\n").unwrap();
140
            let mut body = String::new();
141
            s.read_to_string(&mut body).unwrap();
142
            body
143
        };
144
        assert!(get("").starts_with("HTTP/1.1 401"));
145
        let page = get("Authorization: Bearer t0k\r\n");
146
        assert!(page.starts_with("HTTP/1.1 200"), "{page}");
147
        assert!(page.contains("flow_runner_claims_total{runner=\"earth-1\"} ") && page.contains("result=\"passed\"}"), "{page}");
148
    }
149
}