OXYGENHUB
# c43e4f631d99 oxygenhub / crates / hub / src / queue.rs
4.4 KiB · Rust · last changed A durable work queue, first slice (refs #276) 15 days ago source blame history raw
dd327d76 Andy McCall ↶ 1
//! The worker side of the durable work queue (oxygenhub#276,
2
//! docs/roles.md): every worker process takes due items, a batch at a
3
//! time, and does them. Work that fails where it starts (a notification's
4
//! email, the Git mirror after a push) is queued here instead of being
5
//! lost, and retried with backoff.
6
7
use crate::db::{self, queue::Item};
8
use crate::Shared;
9
use std::time::Duration;
10
11
/// How often an idle worker looks for due items.
12
pub const POLL: Duration = Duration::from_secs(2);
13
/// How long a claim lasts before another worker may take the item.
14
const LEASE_MS: i64 = 5 * 60 * 1000;
15
/// Items taken at once.
16
const BATCH: i64 = 10;
17
18
/// An email to send again (`mail`).
19
pub async fn queue_mail(
20
    app: &Shared,
21
    to_name: &str,
22
    to_addr: &str,
23
    subject: &str,
24
    body: &str,
25
    action: Option<(&str, &str)>,
26
    unsubscribe: Option<&str>,
27
) {
28
    let payload = serde_json::json!({
29
        "to_name": to_name, "to_addr": to_addr, "subject": subject, "body": body,
30
        "action_label": action.map(|a| a.0), "action_url": action.map(|a| a.1), "unsubscribe": unsubscribe,
31
    });
32
    if let Err(e) = db::queue::enqueue(&app.db, "mail", &payload, "", 30_000).await {
33
        tracing::warn!("queueing an email to {to_addr}: {e}");
34
    }
35
}
36
37
/// A repository's Git mirror to refresh again (`mirror.refresh`), once.
38
pub async fn queue_mirror(app: &Shared, repo_id: i64) {
39
    let payload = serde_json::json!({ "repo_id": repo_id });
40
    if let Err(e) = db::queue::enqueue(&app.db, "mirror.refresh", &payload, &repo_id.to_string(), 30_000).await {
41
        tracing::warn!("queueing a Git mirror refresh: {e}");
42
    }
43
}
44
45
/// This process's name as a claimant: its host and pid.
46
fn worker_name() -> String {
47
    let host = std::fs::read_to_string("/etc/hostname").map(|h| h.trim().to_string()).unwrap_or_default();
48
    format!("{host}:{}", std::process::id())
49
}
50
51
/// Take and do due items until the process stops.
52
pub async fn worker(app: Shared) {
53
    let me = worker_name();
54
    loop {
55
        let items = match db::queue::claim(&app.db, &me, BATCH, LEASE_MS).await {
56
            Ok(items) => items,
57
            Err(e) => {
58
                tracing::warn!("work queue: claiming: {e}");
59
                Vec::new()
60
            }
61
        };
62
        let idle = items.is_empty();
63
        for item in items {
64
            run(&app, item).await;
65
        }
66
        if idle {
67
            tokio::time::sleep(POLL).await;
68
        }
69
    }
70
}
71
72
/// Do one item and record how it went.
73
pub async fn run(app: &Shared, item: Item) {
74
    let outcome = match item.kind.as_str() {
75
        "mail" => mail(app, &item).await,
76
        "mirror.refresh" => mirror(app, &item).await,
77
        other => Err(format!("no worker knows {other:?}")),
78
    };
79
    let recorded = match outcome {
80
        Ok(()) => db::queue::done(&app.db, item.id).await,
81
        Err(e) => {
82
            tracing::warn!(kind = %item.kind, attempt = item.attempts, "work queue: {e}");
83
            // An item nobody can do is dead at once.
84
            let item = if item.kind != "mail" && item.kind != "mirror.refresh" {
85
                Item { attempts: db::queue::MAX_ATTEMPTS, ..item }
86
            } else {
87
                item
88
            };
89
            db::queue::failed(&app.db, &item, &e).await
90
        }
91
    };
92
    if let Err(e) = recorded {
93
        tracing::warn!("work queue: recording an outcome: {e}");
94
    }
95
}
96
97
async fn mail(app: &Shared, item: &Item) -> Result<(), String> {
98
    let p: serde_json::Value = serde_json::from_str(&item.payload).map_err(|e| e.to_string())?;
99
    let s = |k: &str| p[k].as_str().unwrap_or_default().to_string();
100
    let (label, url) = (s("action_label"), s("action_url"));
101
    let action = (!url.is_empty()).then_some(crate::mail::Action { label: &label, url: &url });
102
    let unsubscribe = s("unsubscribe");
103
    crate::mail::send_letter(
104
        app,
105
        &s("to_name"),
106
        &s("to_addr"),
107
        &s("subject"),
108
        &s("body"),
109
        action,
110
        (!unsubscribe.is_empty()).then_some(unsubscribe.as_str()),
111
    )
112
    .await
113
}
114
115
async fn mirror(app: &Shared, item: &Item) -> Result<(), String> {
116
    let p: serde_json::Value = serde_json::from_str(&item.payload).map_err(|e| e.to_string())?;
117
    let id = p["repo_id"].as_i64().ok_or("no repo_id")?;
118
    // Gone, or its mirror turned off meanwhile: nothing to do.
119
    let Some(row) = db::repos::get_by_id(&app.db, id).await.map_err(|e| e.to_string())? else { return Ok(()) };
120
    crate::mirror::refresh(app, &row).await
121
}