OXYGENHUB
# e9b9619cd857 oxygenhub / crates / hub / src / web / runners.rs
19.2 KiB · Rust · last changed Runners: Connectivity and Status apart, a heartbeat, a filterable queue 20 days ago source blame history raw
1
//! Admin → Runners (hub spec §13.2): the OxygenFlow runners of this hub,
2
//! what they are and are running, their history, registration tokens, and
3
//! draining, disabling and removing them.
4
5
use super::settings::csrf;
6
use super::{filters, Error, Page, Shell};
7
use crate::auth::Viewer;
8
use crate::db::{self, accounts, runners};
9
use crate::Shared;
10
use askama::Template;
11
use axum::extract::{Form, Path, Query, State};
12
use axum::response::{IntoResponse, Redirect, Response};
13
use serde::Deserialize;
14
use std::collections::HashMap;
15
16
fn require_admin(viewer: &Viewer, shell: &Shell) -> Result<accounts::Account, Error> {
17
    match viewer.account() {
18
        Some(a) if a.is_admin => Ok(a.clone()),
19
        _ => Err(Error::not_found(shell.clone())),
20
    }
21
}
22
23
pub struct RunnerRow {
24
    pub r: runners::Runner,
25
    /// Jobs it is running now.
26
    pub running: i64,
27
    /// The repositories it serves, in words.
28
    pub scope: String,
29
    /// Where its peers reach it in the image swarm, and its site
30
    /// (oxygenhub#144); empty when it is not in one.
31
    pub swarm: String,
32
    /// The jobs it runs now: (repository, run, job, started).
33
    pub jobs: Vec<(String, i64, String, Option<i64>)>,
34
}
35
36
impl RunnerRow {
37
    /// What it is doing (oxygenhub#243): `disabled`, `draining`, `idle`,
38
    /// or `running N of M`.
39
    pub fn doing(&self) -> String {
40
        match self.r.state.as_str() {
41
            "disabled" => "disabled".into(),
42
            "draining" if self.running > 0 => format!("draining, {} left", self.running),
43
            "draining" => "draining".into(),
44
            _ if self.running > 0 => format!("running {} of {}", self.running, self.r.max_jobs),
45
            _ => "idle".into(),
46
        }
47
    }
48
49
    /// The jobs it runs, for the status's tooltip.
50
    pub fn jobs_line(&self) -> String {
51
        let now = crate::db::now_ms();
52
        self.jobs
53
            .iter()
54
            .map(|(repo, run, job, started)| match started {
55
                Some(t) => format!("{repo} #{run} {job}, {} min", (now - t).max(0) / 60_000),
56
                None => format!("{repo} #{run} {job}"),
57
            })
58
            .collect::<Vec<_>>()
59
            .join("; ")
60
    }
61
}
62
63
/// The queue's filters (oxygenhub#243), kept in the URL.
64
#[derive(Deserialize, Default, Clone)]
65
pub struct QueueFilter {
66
    #[serde(default)]
67
    pub repo: String,
68
    #[serde(default)]
69
    pub status: String,
70
    #[serde(default)]
71
    pub runner: String,
72
}
73
74
impl QueueFilter {
75
    fn keeps(&self, q: &db::flow::QueueEntry) -> bool {
76
        (self.repo.is_empty() || format!("{}/{}", q.owner, q.name) == self.repo)
77
            && (self.status.is_empty() || q.status == self.status)
78
            && (self.runner.is_empty() || q.runner == self.runner)
79
    }
80
81
    pub fn any(&self) -> bool {
82
        !(self.repo.is_empty() && self.status.is_empty() && self.runner.is_empty())
83
    }
84
}
85
86
/// A scope in words: the whole site, an owner's repositories, or one
87
/// repository.
88
async fn scope_words(app: &Shared, s: &runners::Scope) -> String {
89
    match s.repo_id {
90
        Some(id) => match db::repos::get_by_id(&app.db, id).await.ok().flatten() {
91
            Some(r) => format!("{}/{}", r.owner, r.name),
92
            None => "a removed repository".into(),
93
        },
94
        None if s.owner.is_empty() => "the whole site".into(),
95
        None => format!("{}'s repositories", s.owner),
96
    }
97
}
98
99
/// A scope as typed: empty for the whole site, an owner, or
100
/// `owner/repository`.
101
async fn parse_scope(app: &Shared, text: &str) -> Result<runners::Scope, String> {
102
    let text = text.trim().trim_matches('/');
103
    if text.is_empty() {
104
        return Ok(runners::Scope::default());
105
    }
106
    if let Some((owner, name)) = text.split_once('/') {
107
        return match db::repos::get(&app.db, owner, name).await {
108
            Ok(Some(r)) => Ok(runners::Scope { owner: r.owner.clone(), repo_id: Some(r.id) }),
109
            _ => Err(format!("There is no repository {text}.")),
110
        };
111
    }
112
    let user = accounts::account_by_name(&app.db, text).await.ok().flatten().is_some();
113
    let org = accounts::org_by_name(&app.db, text).await.ok().flatten().is_some();
114
    if user || org {
115
        Ok(runners::Scope { owner: text.to_string(), repo_id: None })
116
    } else {
117
        Err(format!("There is no user or organisation {text}."))
118
    }
119
}
120
121
#[derive(Template)]
122
#[template(path = "admin_runners.html")]
123
pub struct RunnersPage {
124
    shell: Shell,
125
    runners: Vec<RunnerRow>,
126
    /// Registration tokens waiting to be used: (created by, created, expires).
127
    pending: Vec<(String, i64, i64)>,
128
    /// A token just made, shown once, with the command that uses it.
129
    new_token: Option<String>,
130
    hub: String,
131
    notice: Option<String>,
132
    error: Option<String>,
133
    /// Running jobs, then queued ones in the order runners claim them,
134
    /// as the filter keeps them.
135
    queue: Vec<db::flow::QueueEntry>,
136
    /// How many there are, unfiltered.
137
    queue_total: usize,
138
    filter: QueueFilter,
139
    /// The repositories and runners in the queue, for the filter's choices.
140
    queue_repos: Vec<String>,
141
    queue_runners: Vec<String>,
142
}
143
144
async fn page(
145
    app: &Shared,
146
    shell: Shell,
147
    new_token: Option<String>,
148
    notice: Option<String>,
149
    error: Option<String>,
150
) -> Result<Page<RunnersPage>, Error> {
151
    page_filtered(app, shell, new_token, notice, error, QueueFilter::default()).await
152
}
153
154
async fn page_filtered(
155
    app: &Shared,
156
    shell: Shell,
157
    new_token: Option<String>,
158
    notice: Option<String>,
159
    error: Option<String>,
160
    filter: QueueFilter,
161
) -> Result<Page<RunnersPage>, Error> {
162
    let e = |e| Error::from_db(shell.clone(), e);
163
    let list = runners::list(&app.db).await.map_err(e)?;
164
    let running: HashMap<i64, i64> = runners::running(&app.db).await.map_err(e)?.into_iter().collect();
165
    let pending = runners::pending_registrations(&app.db).await.map_err(e)?;
166
    let all = db::flow::queue(&app.db).await.map_err(e)?;
167
    let mut rows = Vec::with_capacity(list.len());
168
    for r in list {
169
        let scope = scope_words(app, &r.scope()).await;
170
        let swarm = app
171
            .swarm
172
            .addr(r.id)
173
            .map(|s| if s.site.is_empty() { s.addr } else { format!("{} · site {}", s.addr, s.site) })
174
            .unwrap_or_default();
175
        let jobs = all
176
            .iter()
177
            .filter(|q| q.status == "running" && q.runner == r.name)
178
            .map(|q| (format!("{}/{}", q.owner, q.name), q.run_number, q.job.clone(), q.started_ms))
179
            .collect();
180
        rows.push(RunnerRow { running: running.get(&r.id).copied().unwrap_or(0), scope, swarm, jobs, r });
181
    }
182
    let mut queue_repos: Vec<String> = all.iter().map(|q| format!("{}/{}", q.owner, q.name)).collect();
183
    queue_repos.sort();
184
    queue_repos.dedup();
185
    let mut queue_runners: Vec<String> = all.iter().filter(|q| !q.runner.is_empty()).map(|q| q.runner.clone()).collect();
186
    queue_runners.sort();
187
    queue_runners.dedup();
188
    let queue_total = all.len();
189
    let queue = all.into_iter().filter(|q| filter.keeps(q)).collect();
190
    Ok(Page(RunnersPage {
191
        hub: app.config.base_url().to_string(),
192
        shell,
193
        runners: rows,
194
        pending,
195
        new_token,
196
        notice,
197
        error,
198
        queue,
199
        queue_total,
200
        filter,
201
        queue_repos,
202
        queue_runners,
203
    }))
204
}
205
206
/// `GET /admin/runners`.
207
pub async fn list(State(app): State<Shared>, viewer: Viewer, Query(filter): Query<QueueFilter>) -> Result<Page<RunnersPage>, Error> {
208
    let shell = Shell::new(&app, "/admin/runners", &viewer);
209
    require_admin(&viewer, &shell)?;
210
    page_filtered(&app, shell, None, None, None, filter).await
211
}
212
213
#[derive(Deserialize)]
214
pub struct CsrfOnly {
215
    #[serde(rename = "_csrf")]
216
    csrf_token: String,
217
}
218
219
#[derive(Deserialize)]
220
pub struct ScopeForm {
221
    #[serde(rename = "_csrf")]
222
    csrf_token: String,
223
    /// Empty for the whole site, an owner, or `owner/repository`.
224
    #[serde(default)]
225
    scope: String,
226
}
227
228
/// `POST /admin/runners/token`: a one-time registration token, shown once,
229
/// for a runner of the scope given.
230
pub async fn new_token(State(app): State<Shared>, viewer: Viewer, Form(f): Form<ScopeForm>) -> Result<Page<RunnersPage>, Error> {
231
    let shell = Shell::new(&app, "/admin/runners", &viewer);
232
    let me = require_admin(&viewer, &shell)?;
233
    csrf(&viewer, &f.csrf_token, &shell)?;
234
    let scope = match parse_scope(&app, &f.scope).await {
235
        Ok(s) => s,
236
        Err(e) => return page(&app, shell, None, None, Some(e)).await,
237
    };
238
    let token = format!("flowreg_{}", crate::auth::random_hex(24));
239
    runners::add_registration(&app.db, &blake3::hash(token.as_bytes()).to_hex(), &me.username, &scope)
240
        .await
241
        .map_err(|e| Error::from_db(shell.clone(), e))?;
242
    let words = scope_words(&app, &scope).await;
243
    accounts::audit(&app.db, Some(&me), "flow.runner.token", "", &format!("registration token created for {words}")).await.ok();
244
    page(&app, shell, Some(token), Some(format!("The runner it registers will take jobs from {words}.")), None).await
245
}
246
247
/// `POST /admin/runners/{id}/scope`: change the repositories a runner serves.
248
pub async fn set_scope(
249
    State(app): State<Shared>,
250
    viewer: Viewer,
251
    Path(id): Path<i64>,
252
    Form(f): Form<ScopeForm>,
253
) -> Result<Page<RunnersPage>, Error> {
254
    let shell = Shell::new(&app, "/admin/runners", &viewer);
255
    let me = require_admin(&viewer, &shell)?;
256
    csrf(&viewer, &f.csrf_token, &shell)?;
257
    let r =
258
        runners::get(&app.db, id).await.map_err(|e| Error::from_db(shell.clone(), e))?.ok_or_else(|| Error::not_found(shell.clone()))?;
259
    let scope = match parse_scope(&app, &f.scope).await {
260
        Ok(s) => s,
261
        Err(e) => return page(&app, shell, None, None, Some(e)).await,
262
    };
263
    runners::set_scope(&app.db, id, &scope).await.map_err(|e| Error::from_db(shell.clone(), e))?;
264
    let words = scope_words(&app, &scope).await;
265
    accounts::audit(&app.db, Some(&me), "flow.runner.scope", &r.name, &words).await.ok();
266
    page(&app, shell, None, Some(format!("{} now takes jobs from {words}.", r.name)), None).await
267
}
268
269
#[derive(Deserialize)]
270
pub struct LimitsForm {
271
    #[serde(rename = "_csrf")]
272
    csrf_token: String,
273
    /// Comma-separated.
274
    #[serde(default)]
275
    labels: String,
276
    /// Minutes; empty for the site's limit.
277
    #[serde(default)]
278
    timeout_minutes: String,
279
}
280
281
/// `POST /admin/runners/{id}/limits`: the labels an administrator gives a
282
/// runner and its own job time limit (#98).
283
pub async fn set_limits(
284
    State(app): State<Shared>,
285
    viewer: Viewer,
286
    Path(id): Path<i64>,
287
    Form(f): Form<LimitsForm>,
288
) -> Result<Response, Error> {
289
    let shell = Shell::new(&app, &format!("/admin/runners/{id}"), &viewer);
290
    let me = require_admin(&viewer, &shell)?;
291
    csrf(&viewer, &f.csrf_token, &shell)?;
292
    let r =
293
        runners::get(&app.db, id).await.map_err(|e| Error::from_db(shell.clone(), e))?.ok_or_else(|| Error::not_found(shell.clone()))?;
294
    let mut labels: Vec<String> = Vec::new();
295
    for l in f.labels.split([',', ' ']).map(str::trim).filter(|l| !l.is_empty()) {
296
        if !l.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) || l.len() > 40 {
297
            return Err(Error::bad(shell, format!("label {l:?}: letters, digits, - _ and . only, up to 40 characters")));
298
        }
299
        if !labels.iter().any(|x| x == l) {
300
            labels.push(l.to_string());
301
        }
302
    }
303
    let timeout = match f.timeout_minutes.trim() {
304
        "" => None,
305
        t => match t.parse::<i64>() {
306
            Ok(m) if (1..=24 * 60).contains(&m) => Some(m),
307
            _ => return Err(Error::bad(shell, "a time limit is a whole number of minutes from 1 to 1440")),
308
        },
309
    };
310
    runners::set_limits(&app.db, id, &labels.join(","), timeout).await.map_err(|e| Error::from_db(shell.clone(), e))?;
311
    let words = format!(
312
        "labels {}; time limit {}",
313
        if labels.is_empty() { "none".to_string() } else { labels.join(", ") },
314
        timeout.map(|m| format!("{m} min")).unwrap_or_else(|| "the site's".into())
315
    );
316
    accounts::audit(&app.db, Some(&me), "flow.runner.limits", &r.name, &words).await.ok();
317
    Ok(Redirect::to(&format!("/admin/runners/{id}")).into_response())
318
}
319
320
#[derive(Deserialize)]
321
pub struct StateForm {
322
    #[serde(rename = "_csrf")]
323
    csrf_token: String,
324
    /// `active`, `draining` or `disabled`.
325
    state: String,
326
}
327
328
/// `POST /admin/runners/{id}/state`: drain, disable or resume a runner.
329
pub async fn set_state(
330
    State(app): State<Shared>,
331
    viewer: Viewer,
332
    Path(id): Path<i64>,
333
    Form(f): Form<StateForm>,
334
) -> Result<Response, Error> {
335
    let shell = Shell::new(&app, "/admin/runners", &viewer);
336
    let me = require_admin(&viewer, &shell)?;
337
    csrf(&viewer, &f.csrf_token, &shell)?;
338
    if !matches!(f.state.as_str(), "active" | "draining" | "disabled") {
339
        return Err(Error::bad(shell, "a runner is active, draining or disabled"));
340
    }
341
    let r =
342
        runners::get(&app.db, id).await.map_err(|e| Error::from_db(shell.clone(), e))?.ok_or_else(|| Error::not_found(shell.clone()))?;
343
    runners::set_state(&app.db, id, &f.state).await.map_err(|e| Error::from_db(shell.clone(), e))?;
344
    accounts::audit(&app.db, Some(&me), "flow.runner.state", &r.name, &f.state).await.ok();
345
    Ok(Redirect::to("/admin/runners").into_response())
346
}
347
348
/// `POST /admin/runners/{id}/remove`: forget a runner that runs nothing.
349
pub async fn remove(
350
    State(app): State<Shared>,
351
    viewer: Viewer,
352
    Path(id): Path<i64>,
353
    Form(f): Form<CsrfOnly>,
354
) -> Result<Page<RunnersPage>, Error> {
355
    let shell = Shell::new(&app, "/admin/runners", &viewer);
356
    let me = require_admin(&viewer, &shell)?;
357
    csrf(&viewer, &f.csrf_token, &shell)?;
358
    let e = |e| Error::from_db(shell.clone(), e);
359
    let r = runners::get(&app.db, id).await.map_err(e)?.ok_or_else(|| Error::not_found(shell.clone()))?;
360
    let busy = runners::running(&app.db).await.map_err(e)?.into_iter().any(|(rid, n)| rid == id && n > 0);
361
    if busy {
362
        return page(
363
            &app,
364
            shell,
365
            None,
366
            None,
367
            Some(format!("{} is running jobs: drain it first, then remove it when they finish.", r.name)),
368
        )
369
        .await;
370
    }
371
    runners::remove(&app.db, id).await.map_err(e)?;
372
    accounts::audit(&app.db, Some(&me), "flow.runner.remove", &r.name, "").await.ok();
373
    let also = if r.source == "config" {
374
        " It is configured in hub.toml; remove its token there too, or it comes back on its next claim."
375
    } else {
376
        ""
377
    };
378
    page(&app, shell, None, Some(format!("Removed {}.{also}", r.name)), None).await
379
}
380
381
#[derive(Template)]
382
#[template(path = "admin_runner.html")]
383
pub struct RunnerPage {
384
    shell: Shell,
385
    r: runners::Runner,
386
    running: i64,
387
    history: Vec<runners::RunnerJob>,
388
    declines: Vec<crate::db::flow::RunnerDecline>,
389
    older: Option<i64>,
390
    jobs: i64,
391
    passed: i64,
392
    busy_ms: i64,
393
}
394
395
impl RunnerPage {
396
    /// Time spent running jobs, as pages show durations.
397
    pub fn busy(&self) -> String {
398
        crate::flow::human_ms(self.busy_ms)
399
    }
400
}
401
402
#[derive(Deserialize, Default)]
403
pub struct HistoryQuery {
404
    before: Option<i64>,
405
}
406
407
const PER_PAGE: i64 = 50;
408
409
/// `GET /admin/runners/{id}`: a runner and the jobs it ran.
410
pub async fn show(
411
    State(app): State<Shared>,
412
    viewer: Viewer,
413
    Path(id): Path<i64>,
414
    Query(q): Query<HistoryQuery>,
415
) -> Result<Page<RunnerPage>, Error> {
416
    let shell = Shell::new(&app, &format!("/admin/runners/{id}"), &viewer);
417
    require_admin(&viewer, &shell)?;
418
    let e = |e| Error::from_db(shell.clone(), e);
419
    let r = runners::get(&app.db, id).await.map_err(e)?.ok_or_else(|| Error::not_found(shell.clone()))?;
420
    let mut history = runners::history(&app.db, id, PER_PAGE + 1, q.before).await.map_err(e)?;
421
    let older = (history.len() as i64 > PER_PAGE).then(|| history[PER_PAGE as usize - 1].started_ms).flatten();
422
    history.truncate(PER_PAGE as usize);
423
    let (jobs, passed, busy_ms) = runners::totals(&app.db, id).await.map_err(e)?;
424
    let running = runners::running(&app.db).await.map_err(e)?.into_iter().find(|(rid, _)| *rid == id).map(|(_, n)| n).unwrap_or(0);
425
    let declines = crate::db::flow::declines_of_runner(&app.db, id, 20).await.map_err(e)?;
426
    Ok(Page(RunnerPage { shell, r, running, history, declines, older, jobs, passed, busy_ms }))
427
}
428
429
pub fn duration(j: &runners::RunnerJob) -> String {
430
    match (j.started_ms, j.finished_ms) {
431
        (Some(a), Some(b)) => crate::flow::human_ms(b - a),
432
        (Some(a), None) => crate::flow::human_ms(db::now_ms() - a),
433
        _ => "–".into(),
434
    }
435
}
436
437
// ---------------------------------------------------------------- settings
438
439
#[derive(Template)]
440
#[template(path = "admin_runner_settings.html")]
441
pub struct SettingsPage {
442
    shell: Shell,
443
    default_m: u64,
444
    max_m: u64,
445
    grace_m: i64,
446
    /// What hub.toml says, shown beside a setting the page overrides.
447
    file_default_m: u64,
448
    flow_repos: Vec<String>,
449
    notice: Option<String>,
450
    error: Option<String>,
451
}
452
453
async fn settings_page(app: &Shared, shell: Shell, notice: Option<String>, error: Option<String>) -> Page<SettingsPage> {
454
    let l = crate::flow::limits(app).await;
455
    Page(SettingsPage {
456
        shell,
457
        default_m: l.default_s / 60,
458
        max_m: l.max_s / 60,
459
        grace_m: l.grace_ms / 60_000,
460
        file_default_m: app.config.flow.job_timeout_minutes,
461
        flow_repos: app.config.flow.repos.clone(),
462
        notice,
463
        error,
464
    })
465
}
466
467
/// `GET /admin/runners/settings` (oxygenhub#89).
468
pub async fn settings(State(app): State<Shared>, viewer: Viewer) -> Result<Page<SettingsPage>, Error> {
469
    let shell = Shell::new(&app, "/admin/runners/settings", &viewer);
470
    require_admin(&viewer, &shell)?;
471
    Ok(settings_page(&app, shell, None, None).await)
472
}
473
474
#[derive(Deserialize)]
475
pub struct SettingsForm {
476
    #[serde(rename = "_csrf")]
477
    csrf_token: String,
478
    default_minutes: String,
479
    max_minutes: String,
480
    grace_minutes: String,
481
}
482
483
/// `POST /admin/runners/settings`: the site's job time limits.
484
pub async fn save_settings(State(app): State<Shared>, viewer: Viewer, Form(f): Form<SettingsForm>) -> Result<Page<SettingsPage>, Error> {
485
    let shell = Shell::new(&app, "/admin/runners/settings", &viewer);
486
    let me = require_admin(&viewer, &shell)?;
487
    csrf(&viewer, &f.csrf_token, &shell)?;
488
    let num = |v: &str| v.trim().parse::<u64>().ok().filter(|n| *n > 0);
489
    let (Some(d), Some(m), Some(g)) = (num(&f.default_minutes), num(&f.max_minutes), num(&f.grace_minutes)) else {
490
        return Ok(settings_page(&app, shell, None, Some("Each limit is a whole number of minutes, at least 1.".into())).await);
491
    };
492
    if d > m {
493
        return Ok(settings_page(&app, shell, None, Some("The default cannot be longer than the maximum.".into())).await);
494
    }
495
    if m > 24 * 60 {
496
        return Ok(settings_page(&app, shell, None, Some("A job may run for a day at most (1440 minutes).".into())).await);
497
    }
498
    let e = |e| Error::from_db(shell.clone(), e);
499
    accounts::set_setting(&app.db, crate::flow::SETTING_DEFAULT, &d.to_string()).await.map_err(e)?;
500
    accounts::set_setting(&app.db, crate::flow::SETTING_MAX, &m.to_string()).await.map_err(e)?;
501
    accounts::set_setting(&app.db, crate::flow::SETTING_GRACE, &g.to_string()).await.map_err(e)?;
502
    accounts::audit(
503
        &app.db,
504
        Some(&me),
505
        "flow.settings",
506
        "",
507
        &format!("job time limit {d} min by default, {m} min at most; lost after {g} min more"),
508
    )
509
    .await
510
    .ok();
511
    Ok(settings_page(&app, shell, Some("Saved. Jobs claimed from now on run under these limits.".into()), None).await)
512
}