OXYGENHUB

Coverage / crates/hub/src/db/push_protection.rs 100.0%

211 of 211 lines run, 32 of 64 functions · run #485 on main · run missed

1
//! Push protection's records (migration 0085, oxygenhub#268): fingerprints
2
//! a maintainer allowed, and the findings of recent refusals; and the
3
//! history scan's (migration 0086, oxygenhub#277): its findings, marked
4
//! resolved once the secret was revoked, and each repository's progress.
5
6
use super::{now_ms, Pool};
7
use sqlx::FromRow;
8
9
/// Refused pushes' findings kept per repository: the newest, for
10
/// maintainers to judge. The history scan's are all kept (one per value
11
/// and file).
12
const KEPT_FINDINGS: i64 = 200;
13
14
/// A finding's source: a refused push, or the scan of history.
15
pub const PUSH: &str = "push";
16
pub const HISTORY: &str = "history";
17
18
#[derive(Clone, Debug, FromRow)]
19
pub struct Bypass {
20
    pub id: i64,
21
    pub fingerprint: String,
22
    pub rule: String,
23
    pub reason: String,
24
    pub by_name: String,
25
    pub created_ms: i64,
26
}
27
28
#[derive(Clone, Debug, FromRow)]
29
pub struct FindingRow {
30
    pub id: i64,
31
    pub at_ms: i64,
32
    pub pusher: String,
33
    pub via: String,
34
    pub rule: String,
35
    pub description: String,
36
    pub path: String,
37
    pub line: i64,
38
    pub fingerprint: String,
39
    pub excerpt: String,
40
}
41
42
/// A finding of the history scan (#277): where it is, and whether a
43
/// maintainer marked it resolved.
44
#[derive(Clone, Debug, FromRow)]
45
pub struct HistoryRow {
46
    pub id: i64,
47
    pub at_ms: i64,
48
    pub rule: String,
49
    pub description: String,
50
    pub path: String,
51
    pub line: i64,
52
    pub fingerprint: String,
53
    pub excerpt: String,
54
    pub change_hash: String,
55
    pub bookmark: String,
56
    pub resolved_ms: Option<i64>,
57
    pub resolved_by: String,
58
    pub resolved_reason: String,
59
}
60
61
/// A finding to record.
62
pub struct NewFinding<'a> {
63
    pub rule: &'a str,
64
    pub description: &'a str,
65
    pub path: &'a str,
66
    pub line: i64,
67
    pub fingerprint: &'a str,
68
    pub excerpt: &'a str,
69
}
70
71
/// The fingerprints allowed in a repository.
7217×
pub async fn allowed(pool: &Pool, repo_id: i64) -> sqlx::Result<Vec<String>> {
7317×
    sqlx::query_scalar("SELECT fingerprint FROM secret_bypasses WHERE repo_id = $1").bind(repo_id).fetch_all(pool).await
7417×
}
75
76133×
pub async fn bypasses(pool: &Pool, repo_id: i64) -> sqlx::Result<Vec<Bypass>> {
77133×
    sqlx::query_as("SELECT id, fingerprint, rule, reason, by_name, created_ms FROM secret_bypasses WHERE repo_id = $1 ORDER BY id DESC")
78133×
        .bind(repo_id)
79133×
        .fetch_all(pool)
80133×
        .await
81133×
}
82
83
/// Allow `fingerprint` with `reason`; allowing it again replaces the reason.
843×
pub async fn allow(pool: &Pool, repo_id: i64, fingerprint: &str, rule: &str, reason: &str, by: Option<(i64, &str)>) -> sqlx::Result<()> {
853×
    sqlx::query(
863×
        "INSERT INTO secret_bypasses (repo_id, fingerprint, rule, reason, by_account, by_name, created_ms) VALUES ($1, $2, $3, $4, $5, $6, $7) \
873×
         ON CONFLICT (repo_id, fingerprint) DO UPDATE SET reason = excluded.reason, rule = excluded.rule, by_account = excluded.by_account, by_name = excluded.by_name, created_ms = excluded.created_ms",
88
    )
893×
    .bind(repo_id)
903×
    .bind(fingerprint)
913×
    .bind(rule)
923×
    .bind(reason)
933×
    .bind(by.map(|b| b.0))
943×
    .bind(by.map(|b| b.1).unwrap_or(""))
953×
    .bind(now_ms())
963×
    .execute(pool)
973×
    .await?;
983×
    Ok(())
993×
}
100
101
/// Record a refused push's findings, keeping the newest few hundred.
10214×
pub async fn record(pool: &Pool, repo_id: i64, pusher: &str, via: &str, findings: &[NewFinding<'_>]) -> sqlx::Result<()> {
10314×
    let now = now_ms();
10414×
    let mut tx = super::begin_write(pool).await?;
10518×
    for f in findings {
10618×
        sqlx::query(
10718×
            "INSERT INTO secret_findings (repo_id, at_ms, pusher, via, rule, description, path, line, fingerprint, excerpt) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
10818×
        )
10918×
        .bind(repo_id)
11018×
        .bind(now)
11118×
        .bind(pusher)
11218×
        .bind(via)
11318×
        .bind(f.rule)
11418×
        .bind(f.description)
11518×
        .bind(f.path)
11618×
        .bind(f.line)
11718×
        .bind(f.fingerprint)
11818×
        .bind(f.excerpt)
11918×
        .execute(&mut *tx)
12018×
        .await?;
121
    }
12214×
    sqlx::query(
12314×
        "DELETE FROM secret_findings WHERE repo_id = $1 AND source = 'push' \
12414×
         AND id NOT IN (SELECT id FROM secret_findings WHERE repo_id = $2 AND source = 'push' ORDER BY id DESC LIMIT $3)",
12514×
    )
12614×
    .bind(repo_id)
12714×
    .bind(repo_id)
12814×
    .bind(KEPT_FINDINGS)
12914×
    .execute(&mut *tx)
13014×
    .await?;
13114×
    tx.commit().await
13214×
}
133
134
/// The newest findings of refused pushes, those not since allowed.
135134×
pub async fn recent(pool: &Pool, repo_id: i64, limit: i64) -> sqlx::Result<Vec<FindingRow>> {
136134×
    sqlx::query_as(
137134×
        "SELECT id, at_ms, pusher, via, rule, description, path, line, fingerprint, excerpt FROM secret_findings f \
138134×
         WHERE repo_id = $1 AND source = 'push' \
139134×
         AND NOT EXISTS (SELECT 1 FROM secret_bypasses b WHERE b.repo_id = f.repo_id AND b.fingerprint = f.fingerprint) \
140134×
         ORDER BY id DESC LIMIT $2",
141134×
    )
142134×
    .bind(repo_id)
143134×
    .bind(limit)
144134×
    .fetch_all(pool)
145134×
    .await
146134×
}
147
148
/// The rule a fingerprint was found under in this repository, if it was.
1493×
pub async fn rule_of(pool: &Pool, repo_id: i64, fingerprint: &str) -> sqlx::Result<Option<String>> {
1503×
    sqlx::query_scalar("SELECT rule FROM secret_findings WHERE repo_id = $1 AND fingerprint = $2 ORDER BY id DESC LIMIT 1")
1513×
        .bind(repo_id)
1523×
        .bind(fingerprint)
1533×
        .fetch_optional(pool)
1543×
        .await
1553×
}
156
157
/// Whether pushes adding possible secrets are refused here.
15822×
pub async fn set_enabled(pool: &Pool, repo_id: i64, on: bool) -> sqlx::Result<()> {
15922×
    sqlx::query("UPDATE repositories SET push_protection = $1 WHERE id = $2").bind(on).bind(repo_id).execute(pool).await?;
16022×
    Ok(())
16122×
}
162
163
/// Remove an allowed fingerprint (#277): pushes adding that value are
164
/// refused again. The row removed, if there was one.
1652×
pub async fn remove_bypass(pool: &Pool, repo_id: i64, fingerprint: &str) -> sqlx::Result<Option<Bypass>> {
1662×
    let row: Option<Bypass> = sqlx::query_as(
1672×
        "SELECT id, fingerprint, rule, reason, by_name, created_ms FROM secret_bypasses WHERE repo_id = $1 AND fingerprint = $2",
1682×
    )
1692×
    .bind(repo_id)
1702×
    .bind(fingerprint)
1712×
    .fetch_optional(pool)
1722×
    .await?;
1732×
    if let Some(b) = &row {
1741×
        sqlx::query("DELETE FROM secret_bypasses WHERE id = $1").bind(b.id).execute(pool).await?;
1751×
    }
1762×
    Ok(row)
1772×
}
178
179
/// A finding of the history scan to record.
180
pub struct NewHistoryFinding<'a> {
181
    pub finding: NewFinding<'a>,
182
    pub change_hash: String,
183
    pub bookmark: String,
184
}
185
186
/// Record the history scan's findings: one per value and file, so a value
187
/// found again (in a later change, or by a rescan) keeps its first row and
188
/// whether it was resolved. How many were new.
18919×
pub async fn record_history(pool: &Pool, repo_id: i64, findings: &[NewHistoryFinding<'_>]) -> sqlx::Result<usize> {
19019×
    let now = now_ms();
19119×
    let mut added = 0;
19219×
    for h in findings {
1937×
        let f = &h.finding;
1947×
        let r = sqlx::query(
1957×
            "INSERT INTO secret_findings (repo_id, at_ms, pusher, via, rule, description, path, line, fingerprint, excerpt, source, change_hash, bookmark) \
1967×
             VALUES ($1, $2, '', '', $3, $4, $5, $6, $7, $8, 'history', $9, $10) \
1977×
             ON CONFLICT (repo_id, fingerprint, path) WHERE source = 'history' DO NOTHING",
1987×
        )
1997×
        .bind(repo_id)
2007×
        .bind(now)
2017×
        .bind(f.rule)
2027×
        .bind(f.description)
2037×
        .bind(f.path)
2047×
        .bind(f.line)
2057×
        .bind(f.fingerprint)
2067×
        .bind(f.excerpt)
2077×
        .bind(&h.change_hash)
2087×
        .bind(&h.bookmark)
2097×
        .execute(pool)
2107×
        .await?;
2117×
        added += r.rows_affected() as usize;
212
    }
21319×
    Ok(added)
21419×
}
215
216
const HISTORY_COLUMNS: &str =
217
    "id, at_ms, rule, description, path, line, fingerprint, excerpt, change_hash, bookmark, resolved_ms, resolved_by, resolved_reason";
218
219
/// The history scan's findings still open: not resolved, their value not
220
/// allowed; oldest first, as history has them.
221239×
pub async fn history(pool: &Pool, repo_id: i64, limit: i64) -> sqlx::Result<Vec<HistoryRow>> {
222239×
    sqlx::query_as(&format!(
223239×
        "SELECT {HISTORY_COLUMNS} FROM secret_findings f WHERE repo_id = $1 AND source = 'history' AND resolved_ms IS NULL \
224239×
         AND NOT EXISTS (SELECT 1 FROM secret_bypasses b WHERE b.repo_id = f.repo_id AND b.fingerprint = f.fingerprint) \
225239×
         ORDER BY id LIMIT $2"
226239×
    ))
227239×
    .bind(repo_id)
228239×
    .bind(limit)
229239×
    .fetch_all(pool)
230239×
    .await
231239×
}
232
233
/// The history scan's findings marked resolved, newest mark first.
234135×
pub async fn resolved(pool: &Pool, repo_id: i64, limit: i64) -> sqlx::Result<Vec<HistoryRow>> {
235135×
    sqlx::query_as(&format!(
236135×
        "SELECT {HISTORY_COLUMNS} FROM secret_findings WHERE repo_id = $1 AND source = 'history' AND resolved_ms IS NOT NULL \
237135×
         ORDER BY resolved_ms DESC, id DESC LIMIT $2"
238135×
    ))
239135×
    .bind(repo_id)
240135×
    .bind(limit)
241135×
    .fetch_all(pool)
242135×
    .await
243135×
}
244
245
/// Mark history finding `id` resolved (the secret was revoked) with
246
/// `reason`; the finding, if it was an open one of this repository's.
2472×
pub async fn resolve(pool: &Pool, repo_id: i64, id: i64, by: &str, reason: &str) -> sqlx::Result<Option<HistoryRow>> {
2482×
    let done = sqlx::query(
2492×
        "UPDATE secret_findings SET resolved_ms = $1, resolved_by = $2, resolved_reason = $3 \
2502×
         WHERE id = $4 AND repo_id = $5 AND source = 'history' AND resolved_ms IS NULL",
2512×
    )
2522×
    .bind(now_ms())
2532×
    .bind(by)
2542×
    .bind(reason)
2552×
    .bind(id)
2562×
    .bind(repo_id)
2572×
    .execute(pool)
2582×
    .await?;
2592×
    if done.rows_affected() == 0 {
2601×
        return Ok(None);
2611×
    }
2621×
    sqlx::query_as(&format!("SELECT {HISTORY_COLUMNS} FROM secret_findings WHERE id = $1")).bind(id).fetch_optional(pool).await
2632×
}
264
265
/// A repository's history scan (#277): the rules it ran with, the heads
266
/// whose history is scanned (`done_heads`), the heads a scan under way
267
/// works toward (`target_heads`, empty when none is) and how far it got
268
/// (`position` of `pending` changes, in the scan's order), totals, and a
269
/// maintainer's request for a scan now.
270
#[derive(Clone, Debug, Default, PartialEq, Eq, FromRow)]
271
pub struct ScanRow {
272
    pub repo_id: i64,
273
    pub rules_version: i64,
274
    pub done_heads: String,
275
    pub target_heads: String,
276
    pub position: i64,
277
    pub pending: i64,
278
    pub changes: i64,
279
    pub bytes: i64,
280
    pub requested_ms: Option<i64>,
281
    pub pass_ms: Option<i64>,
282
    pub finished_ms: Option<i64>,
283
    pub error: String,
284
}
285
286
const SCAN_COLUMNS: &str =
287
    "repo_id, rules_version, done_heads, target_heads, position, pending, changes, bytes, requested_ms, pass_ms, finished_ms, error";
288
289178×
pub async fn scan_of(pool: &Pool, repo_id: i64) -> sqlx::Result<Option<ScanRow>> {
290178×
    sqlx::query_as(&format!("SELECT {SCAN_COLUMNS} FROM secret_scans WHERE repo_id = $1")).bind(repo_id).fetch_optional(pool).await
291178×
}
292
293
/// Save a repository's scan progress.
29419×
pub async fn save_scan(pool: &Pool, s: &ScanRow) -> sqlx::Result<()> {
29519×
    sqlx::query(&format!(
29619×
        "INSERT INTO secret_scans ({SCAN_COLUMNS}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) \
29719×
         ON CONFLICT (repo_id) DO UPDATE SET rules_version = excluded.rules_version, done_heads = excluded.done_heads, \
29819×
         target_heads = excluded.target_heads, position = excluded.position, pending = excluded.pending, changes = excluded.changes, \
29919×
         bytes = excluded.bytes, requested_ms = excluded.requested_ms, pass_ms = excluded.pass_ms, finished_ms = excluded.finished_ms, \
30019×
         error = excluded.error"
30119×
    ))
30219×
    .bind(s.repo_id)
30319×
    .bind(s.rules_version)
30419×
    .bind(&s.done_heads)
30519×
    .bind(&s.target_heads)
30619×
    .bind(s.position)
30719×
    .bind(s.pending)
30819×
    .bind(s.changes)
30919×
    .bind(s.bytes)
31019×
    .bind(s.requested_ms)
31119×
    .bind(s.pass_ms)
31219×
    .bind(s.finished_ms)
31319×
    .bind(&s.error)
31419×
    .execute(pool)
31519×
    .await?;
31619×
    Ok(())
31719×
}
318
319
/// A maintainer asked for the whole history to be scanned again now:
320
/// what was scanned is forgotten (findings and their marks are kept) and
321
/// the repository goes first.
3221×
pub async fn request_scan(pool: &Pool, repo_id: i64) -> sqlx::Result<()> {
3231×
    let now = now_ms();
3241×
    sqlx::query(
3251×
        "INSERT INTO secret_scans (repo_id, requested_ms) VALUES ($1, $2) \
3261×
         ON CONFLICT (repo_id) DO UPDATE SET requested_ms = excluded.requested_ms, done_heads = '', target_heads = '', position = 0, pending = 0",
3271×
    )
3281×
    .bind(repo_id)
3291×
    .bind(now)
3301×
    .execute(pool)
3311×
    .await?;
3321×
    Ok(())
3331×
}
334
335
/// Repositories whose maintainers asked for a scan, oldest request first.
33610×
pub async fn requested(pool: &Pool) -> sqlx::Result<Vec<i64>> {
33710×
    sqlx::query_scalar("SELECT repo_id FROM secret_scans WHERE requested_ms IS NOT NULL ORDER BY requested_ms").fetch_all(pool).await
33810×
}