Push protection: refuse a push that adds what looks like a secret 0ee5c3fb32eb

-- Push protection (oxygenhub#268): whether pushes that add possible

-- secrets are refused (on by default), the fingerprints a maintainer let

-- through, and the findings of recent refusals for maintainers to judge.

ALTER TABLE repositories ADD COLUMN push_protection BOOLEAN NOT NULL DEFAULT TRUE;

CREATE TABLE secret_bypasses (

    id         BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,

    repo_id    BIGINT NOT NULL REFERENCES repositories(id) ON DELETE CASCADE,

    fingerprint TEXT NOT NULL,

    rule       TEXT NOT NULL DEFAULT '',

    reason     TEXT NOT NULL,

    by_account BIGINT REFERENCES accounts(id) ON DELETE SET NULL,

    by_name    TEXT NOT NULL DEFAULT '',

    created_ms BIGINT NOT NULL,

    UNIQUE (repo_id, fingerprint)

);

CREATE TABLE secret_findings (

    id          BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,

    repo_id     BIGINT NOT NULL REFERENCES repositories(id) ON DELETE CASCADE,

    at_ms       BIGINT NOT NULL,

    pusher      TEXT NOT NULL DEFAULT '',

    via         TEXT NOT NULL DEFAULT '',

    rule        TEXT NOT NULL,

    description TEXT NOT NULL,

    path        TEXT NOT NULL,

    line        BIGINT NOT NULL,

    fingerprint TEXT NOT NULL,

    excerpt     TEXT NOT NULL

);

CREATE INDEX secret_findings_repo ON secret_findings(repo_id, id);

-- Push protection (oxygenhub#268): whether pushes that add possible

-- secrets are refused (on by default), the fingerprints a maintainer let

-- through, and the findings of recent refusals for maintainers to judge.

ALTER TABLE repositories ADD COLUMN push_protection INTEGER NOT NULL DEFAULT 1;

CREATE TABLE secret_bypasses (

    id         INTEGER PRIMARY KEY,

    repo_id    INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE,

    fingerprint TEXT NOT NULL,

    rule       TEXT NOT NULL DEFAULT '',

    reason     TEXT NOT NULL,

    by_account INTEGER REFERENCES accounts(id) ON DELETE SET NULL,

    by_name    TEXT NOT NULL DEFAULT '',

    created_ms INTEGER NOT NULL,

    UNIQUE (repo_id, fingerprint)

);

CREATE TABLE secret_findings (

    id          INTEGER PRIMARY KEY,

    repo_id     INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE,

    at_ms       INTEGER NOT NULL,

    pusher      TEXT NOT NULL DEFAULT '',

    via         TEXT NOT NULL DEFAULT '',

    rule        TEXT NOT NULL,

    description TEXT NOT NULL,

    path        TEXT NOT NULL,

    line        INTEGER NOT NULL,

    fingerprint TEXT NOT NULL,

    excerpt     TEXT NOT NULL

);

CREATE INDEX secret_findings_repo ON secret_findings(repo_id, id);

//! Push protection's records (migration 0085, oxygenhub#268): fingerprints

//! a maintainer allowed, and the findings of recent refusals.

use super::{now_ms, Pool};

use sqlx::FromRow;

/// Findings kept per repository: the newest, for maintainers to judge.

const KEPT_FINDINGS: i64 = 200;

#[derive(Clone, Debug, FromRow)]

pub struct Bypass {

    pub id: i64,

    pub fingerprint: String,

    pub rule: String,

    pub reason: String,

    pub by_name: String,

    pub created_ms: i64,

}

#[derive(Clone, Debug, FromRow)]

pub struct FindingRow {

    pub id: i64,

    pub at_ms: i64,

    pub pusher: String,

    pub via: String,

    pub rule: String,

    pub description: String,

    pub path: String,

    pub line: i64,

    pub fingerprint: String,

    pub excerpt: String,

}

/// A finding to record.

pub struct NewFinding<'a> {

    pub rule: &'a str,

    pub description: &'a str,

    pub path: &'a str,

    pub line: i64,

    pub fingerprint: &'a str,

    pub excerpt: &'a str,

}

/// The fingerprints allowed in a repository.

pub async fn allowed(pool: &Pool, repo_id: i64) -> sqlx::Result<Vec<String>> {

    sqlx::query_scalar("SELECT fingerprint FROM secret_bypasses WHERE repo_id = $1").bind(repo_id).fetch_all(pool).await

}

pub async fn bypasses(pool: &Pool, repo_id: i64) -> sqlx::Result<Vec<Bypass>> {

    sqlx::query_as("SELECT id, fingerprint, rule, reason, by_name, created_ms FROM secret_bypasses WHERE repo_id = $1 ORDER BY id DESC")

        .bind(repo_id)

        .fetch_all(pool)

        .await

}

/// Allow `fingerprint` with `reason`; allowing it again replaces the reason.

pub async fn allow(pool: &Pool, repo_id: i64, fingerprint: &str, rule: &str, reason: &str, by: Option<(i64, &str)>) -> sqlx::Result<()> {

    sqlx::query(

        "INSERT INTO secret_bypasses (repo_id, fingerprint, rule, reason, by_account, by_name, created_ms) VALUES ($1, $2, $3, $4, $5, $6, $7) \

         ON CONFLICT (repo_id, fingerprint) DO UPDATE SET reason = excluded.reason, rule = excluded.rule, by_account = excluded.by_account, by_name = excluded.by_name, created_ms = excluded.created_ms",

    )

    .bind(repo_id)

    .bind(fingerprint)

    .bind(rule)

    .bind(reason)

    .bind(by.map(|b| b.0))

    .bind(by.map(|b| b.1).unwrap_or(""))

    .bind(now_ms())

    .execute(pool)

    .await?;

    Ok(())

}

/// Record a refused push's findings, keeping the newest few hundred.

pub async fn record(pool: &Pool, repo_id: i64, pusher: &str, via: &str, findings: &[NewFinding<'_>]) -> sqlx::Result<()> {

    let now = now_ms();

    let mut tx = super::begin_write(pool).await?;

    for f in findings {

        sqlx::query(

            "INSERT INTO secret_findings (repo_id, at_ms, pusher, via, rule, description, path, line, fingerprint, excerpt) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",

        )

        .bind(repo_id)

        .bind(now)

        .bind(pusher)

        .bind(via)

        .bind(f.rule)

        .bind(f.description)

        .bind(f.path)

        .bind(f.line)

        .bind(f.fingerprint)

        .bind(f.excerpt)

        .execute(&mut *tx)

        .await?;

    }

    sqlx::query(

        "DELETE FROM secret_findings WHERE repo_id = $1 AND id NOT IN (SELECT id FROM secret_findings WHERE repo_id = $2 ORDER BY id DESC LIMIT $3)",

    )

    .bind(repo_id)

    .bind(repo_id)

    .bind(KEPT_FINDINGS)

    .execute(&mut *tx)

    .await?;

    tx.commit().await

}

/// The newest findings of refused pushes, those not since allowed.

pub async fn recent(pool: &Pool, repo_id: i64, limit: i64) -> sqlx::Result<Vec<FindingRow>> {

    sqlx::query_as(

        "SELECT id, at_ms, pusher, via, rule, description, path, line, fingerprint, excerpt FROM secret_findings f \

         WHERE repo_id = $1 AND NOT EXISTS (SELECT 1 FROM secret_bypasses b WHERE b.repo_id = f.repo_id AND b.fingerprint = f.fingerprint) \

         ORDER BY id DESC LIMIT $2",

    )

    .bind(repo_id)

    .bind(limit)

    .fetch_all(pool)

    .await

}

/// The rule a fingerprint was found under in this repository, if it was.

pub async fn rule_of(pool: &Pool, repo_id: i64, fingerprint: &str) -> sqlx::Result<Option<String>> {

    sqlx::query_scalar("SELECT rule FROM secret_findings WHERE repo_id = $1 AND fingerprint = $2 ORDER BY id DESC LIMIT 1")

        .bind(repo_id)

        .bind(fingerprint)

        .fetch_optional(pool)

        .await

}

/// Whether pushes adding possible secrets are refused here.

pub async fn set_enabled(pool: &Pool, repo_id: i64, on: bool) -> sqlx::Result<()> {

    sqlx::query("UPDATE repositories SET push_protection = $1 WHERE id = $2").bind(on).bind(repo_id).execute(pool).await?;

    Ok(())

}

//! Push protection (hub spec §15.4, oxygenhub#268): a push that adds

//! something that looks like a credential is refused before any of it is

//! stored, saying where, so it never lands in a repository's history.

//!

//! Only what a push adds is read: the lines its changes insert and the new

//! blobs it carries, never history the hub already holds. The scan has a

//! budget ([`Budget`]: bytes per push, the largest file, time), so a push

//! stays fast; past it the rest goes unscanned, the push is accepted and its

//! log entry says how much was left (a refusal would block every large

//! import for what is most likely nothing). Rules are data, in

//! `push_protection.toml`, compiled in and versioned.

//!

//! A finding names its rule, file and line, a fingerprint (BLAKE3 of the

//! rule id and the secret, the first 16 hex digits) and an excerpt with the

//! secret masked. A maintainer lets one finding through by its fingerprint

//! (Settings › Push protection); the setting itself is per repository and

//! on by default.

use oxy_core::hash::ObjectType;

use oxy_core::hash::{ChangeHash, ObjectRef};

use oxy_core::ids::NodeId;

use oxy_core::objects::{Edit, TreeOp};

use oxy_core::protocol::Bundle;

use oxy_core::repo::Repo;

use regex::bytes::Regex;

use serde::Deserialize;

use std::collections::{HashMap, HashSet};

use std::sync::OnceLock;

use std::time::{Duration, Instant};

/// The rules, as compiled in.

const RULES: &str = include_str!("push_protection.toml");

/// How much of a push is scanned: past `bytes` in all, or `time`, the rest

/// is not; a file or inserted text over `file` bytes, or binary, is skipped.

#[derive(Clone, Copy, Debug)]

pub struct Budget {

    pub bytes: u64,

    pub file: u64,

    pub time: Duration,

}

impl Default for Budget {

    fn default() -> Self {

        Budget { bytes: 16 << 20, file: 1 << 20, time: Duration::from_secs(2) }

    }

}

#[derive(Deserialize)]

#[serde(deny_unknown_fields)]

struct RulesFile {

    version: u32,

    rule: Vec<RuleSpec>,

}

#[derive(Deserialize)]

#[serde(deny_unknown_fields)]

struct RuleSpec {

    id: String,

    description: String,

    regex: String,

    #[serde(default)]

    keywords: Vec<String>,

    #[serde(default)]

    requires: Option<String>,

}

/// One compiled rule.

pub struct Rule {

    pub id: String,

    pub description: String,

    regex: Regex,

    /// Lower-cased; empty runs the regex on every text.

    keywords: Vec<String>,

    requires: Option<Regex>,

}

/// The compiled rule set.

pub struct Rules {

    pub version: u32,

    pub rules: Vec<Rule>,

}

/// The compiled-in rules. A rule that does not compile is a build defect,

/// caught by this module's tests.

pub fn rules() -> &'static Rules {

    static RULESET: OnceLock<Rules> = OnceLock::new();

    RULESET.get_or_init(|| compile(RULES).expect("push_protection.toml"))

}

fn compile(text: &str) -> Result<Rules, String> {

    let file: RulesFile = toml::from_str(text).map_err(|e| e.to_string())?;

    let mut rules = Vec::new();

    let mut seen = HashSet::new();

    for r in file.rule {

        if !seen.insert(r.id.clone()) {

            return Err(format!("rule {} is listed twice", r.id));

        }

        let regex = Regex::new(&r.regex).map_err(|e| format!("rule {}: {e}", r.id))?;

        let requires = match r.requires {

            Some(q) => Some(Regex::new(&q).map_err(|e| format!("rule {} requires: {e}", r.id))?),

            None => None,

        };

        rules.push(Rule {

            id: r.id,

            description: r.description,

            regex,

            keywords: r.keywords.iter().map(|k| k.to_ascii_lowercase()).collect(),

            requires,

        });

    }

    Ok(Rules { version: file.version, rules })

}

/// Where a finding's text came from in the push.

#[derive(Clone, Debug, PartialEq, Eq)]

pub enum Origin {

    /// Lines an insert adds; `atom` is the finding's line's atom index in

    /// its change (spec §7.4), counted per file.

    Insert { atom: u32 },

    /// A new blob replacing a file's content.

    Blob,

}

/// A possible secret a push adds.

#[derive(Clone, Debug)]

pub struct Finding {

    pub rule: String,

    pub description: String,

    /// The change that adds it: its index in the bundle and its hash.

    pub change: usize,

    pub change_hash: Option<ChangeHash>,

    pub node: NodeId,

    pub origin: Origin,

    /// The file's path, once [`locate`] found it.

    pub path: String,

    /// 1-based: in the file once [`locate`] worked it out, else in the

    /// inserted text or blob.

    pub line: usize,

    /// The first 16 hex digits of BLAKE3(rule id, NUL, secret).

    pub fingerprint: String,

    /// The finding's line, the secret masked.

    pub excerpt: String,

}

/// What a scan found and how much it read.

#[derive(Clone, Debug, Default)]

pub struct Scan {

    pub findings: Vec<Finding>,

    /// Bytes scanned.

    pub scanned: u64,

    /// Texts and blobs skipped as over the file budget or binary.

    pub skipped: usize,

    /// Bytes left unscanned when the budget ran out, and why.

    pub cut_short: Option<(u64, &'static str)>,

}

impl Scan {

    /// One line for the push log when the scan stopped early.

    pub fn note(&self) -> Option<String> {

        let (left, why) = self.cut_short?;

        Some(format!(

            "push protection scanned {} and stopped at its {why} budget: {} more were not scanned",

            human(self.scanned),

            human(left)

        ))

    }

}

fn human(n: u64) -> String {

    match n {

        n if n >= 1 << 20 => format!("{:.1} MiB", n as f64 / (1u64 << 20) as f64),

        n if n >= 1 << 10 => format!("{:.1} KiB", n as f64 / 1024.0),

        n => format!("{n} bytes"),

    }

}

/// The fingerprint of `secret` under `rule`.

pub fn fingerprint(rule: &str, secret: &[u8]) -> String {

    let mut h = blake3::Hasher::new();

    h.update(rule.as_bytes());

    h.update(&[0]);

    h.update(secret);

    h.finalize().to_hex()[..16].to_string()

}

/// A finding in one text: its rule, 1-based line, fingerprint and excerpt.

#[derive(Clone, Debug)]

pub struct Hit {

    pub rule: usize,

    pub line: usize,

    pub fingerprint: String,

    pub excerpt: String,

}

/// Every rule over one text. Matches overlapping an earlier rule's are

/// dropped, so one secret is reported once, by its most specific rule.

pub fn scan_text(rules: &Rules, text: &[u8]) -> Vec<Hit> {

    let mut hits = Vec::new();

    let mut taken: Vec<(usize, usize)> = Vec::new();

    let mut lower: Option<Vec<u8>> = None;

    for (i, rule) in rules.rules.iter().enumerate() {

        if !rule.keywords.is_empty() {

            let lower = lower.get_or_insert_with(|| text.to_ascii_lowercase());

            if !rule.keywords.iter().any(|k| memchr::memmem::find(lower, k.as_bytes()).is_some()) {

                continue;

            }

        }

        let mut required: Option<bool> = None;

        for caps in rule.regex.captures_iter(text) {

            let m = caps.name("secret").or_else(|| caps.get(0)).expect("a match");

            let (start, end) = (m.start(), m.end());

            if start == end || taken.iter().any(|(s, e)| start < *e && *s < end) {

                continue;

            }

            if let Some(q) = &rule.requires {

                if !*required.get_or_insert_with(|| q.is_match(text)) {

                    break;

                }

            }

            taken.push((start, end));

            hits.push(Hit {

                rule: i,

                line: memchr::memchr_iter(b'\n', &text[..start]).count() + 1,

                fingerprint: fingerprint(&rule.id, m.as_bytes()),

                excerpt: excerpt(text, start, end),

            });

        }

    }

    hits

}

/// The line holding `text[start..end]` with the secret masked: its first

/// four characters kept, the rest starred, at most 160 characters around.

fn excerpt(text: &[u8], start: usize, end: usize) -> String {

    let line_start = memchr::memrchr(b'\n', &text[..start]).map(|i| i + 1).unwrap_or(0);

    let line_end = memchr::memchr(b'\n', &text[start..]).map(|i| start + i).unwrap_or(text.len());

    let secret_end = end.min(line_end);

    let kept = (secret_end - start).min(4);

    let before_from = start.saturating_sub(60).max(line_start);

    let after_to = (secret_end + 40).min(line_end);

    let mut out = String::new();

    if before_from > line_start {

        out.push('…');

    }

    out.push_str(&String::from_utf8_lossy(&text[before_from..start + kept]));

    out.push_str(&"*".repeat((secret_end - start - kept).clamp(4, 16)));

    let mut after = String::from_utf8_lossy(&text[secret_end..after_to]).into_owned();

    // The same secret again on the line is masked too.

    let secret = String::from_utf8_lossy(&text[start..secret_end]).into_owned();

    if secret.len() > 4 {

        after = after.replace(&secret, "****");

    }

    out.push_str(&after);

    if after_to < line_end {

        out.push('…');

    }

    out.trim_end_matches(['\r', '\n']).trim().to_string()

}

/// Scan what `bundle` adds: each insert's lines and each new blob a change

/// puts in place of a file's content, in the push's order, within

/// `budget`. Past the size or time budget the rest is only counted.

pub fn scan(bundle: &Bundle, budget: Budget) -> Scan {

    let rules = rules();

    let started = Instant::now();

    let mut out = Scan::default();

    // The bundle's blobs and chunks by reference, made when a change first

    // replaces a file's content.

    let mut objects: Option<HashMap<ObjectRef, usize>> = None;

    let mut stop: Option<&'static str> = None;

    let mut left: u64 = 0;

    for (ci, (change, _)) in bundle.changes.iter().enumerate() {

        let mut counters: HashMap<NodeId, u32> = HashMap::new();

        for e in &change.edits {

            let (node, origin, size) = match e {

                Edit::Insert { node, lines, .. } => {

                    let idx = counters.entry(*node).or_default();

                    let first = *idx;

                    *idx += lines.len() as u32;

                    let n: usize = lines.iter().map(<[u8]>::len).sum();

                    (*node, Origin::Insert { atom: first }, n as u64)

                }

                Edit::Replace { node, new: Some(r), .. } => {

                    let objects = objects.get_or_insert_with(|| {

                        bundle

                            .objects

                            .iter()

                            .enumerate()

                            .filter(|(_, (ty, _))| matches!(ty, ObjectType::Blob | ObjectType::Chunk))

                            .map(|(i, (ty, body))| (oxy_core::hash::hash_object(*ty, body), i))

                            .collect()

                    });

                    // A blob the push does not carry is already on the hub

                    // (or lazy): nothing new.

                    let Some(blob) = blob_of(bundle, objects, r) else { continue };

                    (*node, Origin::Blob, blob.length)

                }

                _ => continue,

            };

            if size > budget.file {

                out.skipped += 1;

                continue;

            }

            if stop.is_none() {

                if out.scanned + size > budget.bytes {

                    stop = Some("size");

                } else if started.elapsed() > budget.time {

                    stop = Some("time");

                }

            }

            if stop.is_some() {

                left += size;

                continue;

            }

            let blob_text;

            let text: &[u8] = match (e, &origin) {

                (Edit::Insert { lines, .. }, _) => &lines.bytes()[..size as usize],

                (Edit::Replace { new: Some(r), .. }, Origin::Blob) => {

                    let objects = objects.as_ref().expect("made above");

                    match blob_content(bundle, objects, r) {

                        Some(t) if memchr::memchr(0, &t[..t.len().min(8000)]).is_none() => {

                            blob_text = t;

                            &blob_text

                        }

                        // Binary, as Git and oxy judge it (a NUL early on),

                        // or incomplete.

                        _ => {

                            out.skipped += 1;

                            continue;

                        }

                    }

                }

                _ => continue,

            };

            for hit in scan_text(rules, text) {

                let rule = &rules.rules[hit.rule];

                let origin = match origin {

                    Origin::Insert { atom } => Origin::Insert { atom: atom + hit.line as u32 - 1 },

                    Origin::Blob => Origin::Blob,

                };

                out.findings.push(Finding {

                    rule: rule.id.clone(),

                    description: rule.description.clone(),

                    change: ci,

                    change_hash: bundle.hashes.get(ci).copied(),

                    node,

                    origin,

                    path: String::new(),

                    line: hit.line,

                    fingerprint: hit.fingerprint,

                    excerpt: hit.excerpt,

                });

            }

            out.scanned += size;

        }

    }

    if let Some(why) = stop {

        out.cut_short = Some((left, why));

    }

    out

}

fn object<'b>(bundle: &'b Bundle, objects: &HashMap<ObjectRef, usize>, r: &ObjectRef, want: ObjectType) -> Option<&'b [u8]> {

    match objects.get(r).map(|i| &bundle.objects[*i]) {

        Some((ty, body)) if *ty == want => Some(body),

        _ => None,

    }

}

fn blob_of(bundle: &Bundle, objects: &HashMap<ObjectRef, usize>, r: &ObjectRef) -> Option<oxy_core::objects::Blob> {

    oxy_core::objects::Blob::decode(object(bundle, objects, r, ObjectType::Blob)?).ok()

}

/// A blob's content from the bundle's chunks; `None` when one is missing.

fn blob_content(bundle: &Bundle, objects: &HashMap<ObjectRef, usize>, r: &ObjectRef) -> Option<Vec<u8>> {

    let blob = blob_of(bundle, objects, r)?;

    let mut content = Vec::with_capacity(blob.length as usize);

    for c in &blob.chunks {

        content.extend_from_slice(object(bundle, objects, c, ObjectType::Chunk)?);

    }

    Some(content)

}

/// Fill in each finding's path, and its line in the file as the change that

/// adds it leaves the file, from `repo` with the push's changes applied in

/// memory. Done only when there are findings: it loads the state the push

/// builds on. A finding it cannot place keeps its line in the inserted text

/// and is named by the push's own tree operations, or by its node.

pub fn locate(repo: &Repo, bundle: &Bundle, findings: &mut [Finding]) {

    let mut names: HashMap<NodeId, (NodeId, String)> = HashMap::new();

    for (c, _) in &bundle.changes {

        for op in &c.tree_ops {

            match op {

                TreeOp::AddNode { node, parent, name, .. } | TreeOp::Move { node, parent, name, .. } => {

                    names.insert(*node, (*parent, name.clone()));

                }

                _ => {}

            }

        }

    }

    let placed = locate_in_state(repo, bundle, findings).unwrap_or_default();

    for (i, f) in findings.iter_mut().enumerate() {

        if let Some((path, line)) = placed.get(&i) {

            f.path = path.clone();

            if let Some(line) = line {

                f.line = *line;

            }

            continue;

        }

        let mut parts = Vec::new();

        let mut cur = f.node;

        while let Some((parent, name)) = names.get(&cur) {

            parts.push(name.clone());

            cur = *parent;

            if parts.len() > 256 {

                break;

            }

        }

        parts.reverse();

        f.path = if parts.is_empty() { format!("(file {})", &f.node.to_hex()[..12]) } else { parts.join("/") };

    }

}

/// Each finding's (path, line) by index, as the state after its change has

/// it; `None` when that state cannot be built.

fn locate_in_state(repo: &Repo, bundle: &Bundle, findings: &[Finding]) -> Option<HashMap<usize, (String, Option<usize>)>> {

    if bundle.hashes.len() != bundle.changes.len() {

        return None;

    }

    let own: HashSet<ChangeHash> = bundle.hashes.iter().copied().collect();

    let mut outside: Vec<ChangeHash> =

        bundle.changes.iter().flat_map(|(c, _)| c.deps.iter().copied()).filter(|d| !own.contains(d)).collect();

    outside.sort();

    outside.dedup();

    let mut state = repo.load_state(&outside).ok()?;

    let mut out = HashMap::new();

    for (ci, (change, _)) in bundle.changes.iter().enumerate() {

        state.add_hashed(bundle.hashes[ci], change.clone()).ok()?;

        let here: Vec<usize> = (0..findings.len()).filter(|i| findings[*i].change == ci).collect();

        if here.is_empty() {

            continue;

        }

        let view = state.view();

        let by_node = view.by_node();

        let mut lines: HashMap<NodeId, Vec<(ChangeHash, u32)>> = HashMap::new();

        for i in here {

            let f = &findings[i];

            let Some(entry) = by_node.get(&f.node) else { continue };

            let line = match f.origin {

                Origin::Insert { atom } => {

                    let live = lines.entry(f.node).or_insert_with(|| state.live_lines(&f.node).into_iter().map(|(k, _)| k).collect());

                    live.iter().position(|k| *k == (bundle.hashes[ci], atom)).map(|p| p + 1)

                }

                Origin::Blob => None,

            };

            out.insert(i, (entry.path.clone(), line));

        }

    }

    Some(out)

}

/// Findings listed in a refusal; the rest are counted.

const LISTED: usize = 20;

/// The refusal's text: each finding as `path:line: description

/// (fingerprint F)` with its masked excerpt, then what to do. `via` names

/// the agent whose token pushed, if one did; `settings` is where a

/// maintainer allows a finding.

pub fn refusal(findings: &[Finding], settings: &str, via: &str) -> String {

    use std::fmt::Write;

    let n = findings.len();

    let mut out = format!(

        "push protection refused this push: it adds {} that look{} like {}, and nothing of it was stored (rules v{}).\n\n",

        if n == 1 { "a value".to_string() } else { format!("{n} values") },

        if n == 1 { "s" } else { "" },

        if n == 1 { "a secret" } else { "secrets" },

        rules().version

    );

    for f in findings.iter().take(LISTED) {

        let _ = writeln!(out, "  {}:{}: {} (fingerprint {})", f.path, f.line, f.description, f.fingerprint);

        if !f.excerpt.is_empty() {

            let _ = writeln!(out, "      {}", f.excerpt);

        }

    }

    if n > LISTED {

        let _ = writeln!(out, "  … and {} more", n - LISTED);

    }

    out.push_str("\nWhat to do:\n");

    let (it, is) = if n == 1 { ("it", "it is") } else { ("each", "one is") };

    let _ = writeln!(

        out,

        "  - Take {it} out of the change that adds it: edit the file, then `oxy amend` (or `oxy abandon` a change that only adds it), and push again. Read credentials from the environment or a secret store, not from the repository."

    );

    let _ = writeln!(out, "  - If {is} a real credential, revoke or rotate it as well: it has been on your disk and may be elsewhere.");

    let _ = writeln!(

        out,

        "  - If {is} not a secret (an example, a test value), a maintainer can allow its fingerprint, with a reason, at {settings}#push-protection; then push again."

    );

    if !via.is_empty() {

        let _ = write!(

            out,

            "\nThis push came through an agent's token ({via}). As an agent: remove the value from the change (`oxy amend`) and push again; do not try to get around the refusal. If you were asked to commit this value, stop and tell the person you work for; only a maintainer can allow a fingerprint, at {settings}#push-protection."

        );

    }

    out.trim_end().to_string()

}

#[cfg(test)]

mod tests {

    use super::*;

    #[test]

    fn the_rules_compile() {

        let r = rules();

        assert!(r.version >= 1);

        assert!(r.rules.len() >= 15);

    }

    #[test]

    fn the_rules_file_is_not_itself_a_finding() {

        assert!(scan_text(rules(), RULES.as_bytes()).is_empty());

    }

    #[test]

    fn the_hub_s_own_push_protection_sources_pass_it() {

        for (name, text) in [

            ("push_protection.rs", include_str!("push_protection.rs")),

            ("tests/push_protection.rs", include_str!("../tests/push_protection.rs")),

            ("docs/spec.md", include_str!("../../../docs/spec.md")),

        ] {

            assert!(scan_text(rules(), text.as_bytes()).is_empty(), "{name}");

        }

    }

    #[test]

    fn a_secret_is_found_masked_and_fingerprinted() {

        let key = format!("{}{}", "AKIA", "IOSFODNN7EXAMPLE");

        let text = format!("[default]\naws_access_key_id = {key}\n");

        let hits = scan_text(rules(), text.as_bytes());

        assert_eq!(hits.len(), 1, "{hits:?}");

        assert_eq!(rules().rules[hits[0].rule].id, "aws-access-key-id");

        assert_eq!(hits[0].line, 2);

        assert_eq!(hits[0].fingerprint, fingerprint("aws-access-key-id", key.as_bytes()));

        assert_eq!(hits[0].fingerprint.len(), 16);

        assert!(!hits[0].excerpt.contains(&key) && hits[0].excerpt.starts_with("aws_access_key_id = AKIA*"), "{}", hits[0].excerpt);

    }

    #[test]

    fn an_aws_secret_needs_its_key_id_nearby() {

        let secret = format!("{}{}", "wJalrXUtnFEMI/K7MDENG/", "bPxRfiCYEXAMPLEKEY");

        let alone = format!("aws_secret_access_key = {secret}\n");

        assert!(scan_text(rules(), alone.as_bytes()).is_empty());

        let paired = format!("aws_access_key_id = {}{}\n{alone}", "AKIA", "IOSFODNN7EXAMPLE");

        let ids: Vec<&str> = scan_text(rules(), paired.as_bytes()).iter().map(|h| rules().rules[h.rule].id.as_str()).collect();

        assert_eq!(ids, ["aws-access-key-id", "aws-secret-access-key"]);

    }

    #[test]

    fn a_service_account_key_is_reported_once() {

        let pem = format!("-----BEGIN {}-----\\n{}\\n-----END {}-----\\n", "PRIVATE KEY", "A".repeat(64), "PRIVATE KEY");

        let json = format!("{{\n  \"type\": \"service_account\",\n  \"private_key\": \"{pem}\"\n}}\n");

        let hits = scan_text(rules(), json.as_bytes());

        assert_eq!(hits.len(), 1, "{hits:?}");

        assert_eq!(rules().rules[hits[0].rule].id, "gcp-service-account");

        assert_eq!(hits[0].line, 3);

    }

    #[test]

    fn a_pem_header_alone_is_not_a_key() {

        let doc = format!("A key starts with -----BEGIN {}-----.\n", "RSA PRIVATE KEY");

        assert!(scan_text(rules(), doc.as_bytes()).is_empty());

    }

}

# Push protection's rules (hub spec §15.4, oxygenhub#268), compiled into

# the hub. Each rule:

#

#   id          stable name, part of every fingerprint: never reuse one

#   description what a refusal calls it

#   regex       Rust `regex` syntax, matched on bytes; the capture group

#               named `secret`, when there is one, is the secret, else the

#               whole match

#   keywords    optional prefilter: the regex runs on a text only when one

#               of these occurs in it (case-insensitive)

#   requires    optional: a regex the same file or inserted text must also

#               match somewhere, for secrets that are only one when paired

#

# Rules are tried in order and a match overlapping an earlier rule's is

# dropped, so the specific rules come before the general ones. Bump

# `version` whenever a rule is added, changed or removed.

version = 1

[[rule]]

id = "gcp-service-account"

description = "Google Cloud service account key"

keywords = ["service_account"]

regex = '''"private_key"\s*:\s*"(?P<secret>-----BEGIN[^"]{40,})"'''

requires = '''"type"\s*:\s*"service_account"'''

[[rule]]

id = "aws-access-key-id"

description = "AWS access key ID"

keywords = ["akia", "asia", "abia", "acca"]

regex = '''\b(?P<secret>(?:AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16})\b'''

[[rule]]

id = "aws-secret-access-key"

description = "AWS secret access key"

keywords = ["aws", "secret"]

regex = '''(?im)(?:aws|secret)[a-z0-9_ .\-]{0,30}['"]?\s*[:=]\s*['"]?(?P<secret>[A-Za-z0-9/+]{40})(?:[^A-Za-z0-9/+=]|$)'''

requires = '''\b(?:AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16}\b'''

[[rule]]

id = "azure-storage-connection-string"

description = "Azure storage account connection string"

keywords = ["accountkey="]

regex = '''(?i)DefaultEndpointsProtocol=https?;AccountName=[A-Za-z0-9]+;AccountKey=(?P<secret>[A-Za-z0-9+/]{40,}={0,2})'''

[[rule]]

id = "github-token"

description = "GitHub token"

keywords = ["ghp_", "gho_", "ghu_", "ghs_", "ghr_"]

regex = '''\b(?P<secret>gh[pousr]_[A-Za-z0-9]{36,255})\b'''

[[rule]]

id = "github-fine-grained-token"

description = "GitHub fine-grained personal access token"

keywords = ["github_pat_"]

regex = '''\b(?P<secret>github_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59})\b'''

[[rule]]

id = "gitlab-token"

description = "GitLab personal access token"

keywords = ["glpat-"]

regex = '''\b(?P<secret>glpat-[A-Za-z0-9_\-.]{20,})'''

[[rule]]

id = "oxygenhub-token"

description = "OxygenHub personal access token"

keywords = ["oxyhub_"]

regex = '''\b(?P<secret>oxyhub_[0-9a-f]{64})\b'''

[[rule]]

id = "oxygenhub-agent-grant"

description = "OxygenHub agent grant token"

keywords = ["oxyagent_"]

regex = '''\b(?P<secret>oxyagent_[0-9a-f]{64})\b'''

[[rule]]

id = "oxygenflow-job-token"

description = "OxygenFlow job token"

keywords = ["flowjob_"]

regex = '''\b(?P<secret>flowjob_[0-9a-f]{64})\b'''

[[rule]]

id = "oxygenflow-runner-credential"

description = "OxygenFlow runner credential"

keywords = ["flowrunner_"]

regex = '''\b(?P<secret>flowrunner_[0-9a-f]{64})\b'''

[[rule]]

id = "oxygenflow-registration-token"

description = "OxygenFlow runner registration token"

keywords = ["flowreg_"]

regex = '''\b(?P<secret>flowreg_[0-9a-f]{48})\b'''

[[rule]]

id = "slack-token"

description = "Slack token"

keywords = ["xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"]

regex = '''\b(?P<secret>xox[baprs]-[A-Za-z0-9-]{10,})'''

[[rule]]

id = "stripe-live-key"

description = "Stripe live secret key"

keywords = ["sk_live_", "rk_live_"]

regex = '''\b(?P<secret>[rs]k_live_[A-Za-z0-9]{24,})\b'''

[[rule]]

id = "openssh-private-key"

description = "OpenSSH private key"

keywords = ["openssh private key"]

regex = '''(?P<secret>-----BEGIN[ ]OPENSSH[ ]PRIVATE[ ]KEY-----[\s\S]{40,}?-----END[ ]OPENSSH[ ]PRIVATE[ ]KEY-----)'''

[[rule]]

id = "private-key"

description = "PEM private key"

keywords = ["private key"]

regex = '''(?P<secret>-----BEGIN[ ](?:[A-Z0-9]+[ ])*PRIVATE[ ]KEY-----[\s\S]{40,}?-----END[ ](?:[A-Z0-9]+[ ])*PRIVATE[ ]KEY-----)'''

//! Push protection (oxygenhub#268, hub spec §15.4): a push adding what

//! looks like a secret is refused with the file, line and fingerprint, and

//! nothing of it is stored; a maintainer's bypass, or the setting off, lets

//! it through; a large push is scanned only up to the budget.

//!

//! Every credential here is made up, in its documented example format, and

//! put together at run time so this file is not itself a finding.

use axum::body::Body;

use axum::http::{header, Request, StatusCode};

use http_body_util::BodyExt;

use oxy_core::hash::ChangeHash;

use oxy_core::repo::Repo;

use oxygenhub::auth::random_hex;

use oxygenhub::config::Config;

use oxygenhub::db;

use oxygenhub::push_protection::fingerprint;

use oxygenhub::repos::RepoName;

use oxygenhub::Shared;

use std::path::Path;

use tower::ServiceExt;

fn hermetic() {

    static ONCE: std::sync::Once = std::sync::Once::new();

    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();

    ONCE.call_once(|| {

        let d = tempfile::tempdir().unwrap();

        std::env::set_var("OXY_HOME", d.path());

        DIR.set(d).ok();

    });

}

fn write(root: &Path, rel: &str, content: &[u8]) {

    let p = root.join(rel);

    std::fs::create_dir_all(p.parent().unwrap()).unwrap();

    std::fs::write(p, content).unwrap();

}

struct Hub {

    app: Shared,

    base: String,

    _dir: tempfile::TempDir,

}

async fn hub(scan_limit: Option<u64>) -> Hub {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let mut config = Config::load(dir.path()).unwrap();

    config.registration.setup_code = "test-setup".into();

    config.registration.mode = "open".into();

    if let Some(n) = scan_limit {

        config.limits.scan_limit = oxygenhub::config::ByteSize(n);

    }

    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();

    let port = listener.local_addr().unwrap().port();

    config.server.base_url = format!("http://127.0.0.1:{port}");

    let app = oxygenhub::build(config).await.unwrap();

    let router = oxygenhub::router(app.clone());

    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });

    Hub { app, base: format!("http://127.0.0.1:{port}"), _dir: dir }

}

struct Res {

    status: StatusCode,

    body: String,

    cookie: Option<String>,

}

async fn send(app: &Shared, method: &str, path: &str, cookie: Option<&str>, form: Option<&str>) -> Res {

    let mut req = Request::builder().method(method).uri(path);

    if let Some(c) = cookie {

        req = req.header(header::COOKIE, format!("oxyhub={c}"));

    }

    let body = match form {

        Some(f) => {

            req = req.header(header::CONTENT_TYPE, "application/x-www-form-urlencoded");

            Body::from(f.to_string())

        }

        None => Body::empty(),

    };

    let res = oxygenhub::router(app.clone()).oneshot(req.body(body).unwrap()).await.unwrap();

    let status = res.status();

    let cookie = res

        .headers()

        .get(header::SET_COOKIE)

        .and_then(|v| v.to_str().ok())

        .and_then(|v| v.split(';').next())

        .and_then(|kv| kv.split_once('='))

        .map(|(_, v)| v.to_string());

    let bytes = res.into_body().collect().await.unwrap().to_bytes();

    Res { status, body: String::from_utf8_lossy(&bytes).into_owned(), cookie }

}

async fn get(app: &Shared, path: &str, cookie: Option<&str>) -> Res {

    send(app, "GET", path, cookie, None).await

}

async fn post(app: &Shared, path: &str, cookie: Option<&str>, form: &str) -> Res {

    send(app, "POST", path, cookie, Some(form)).await

}

async fn register(app: &Shared, user: &str) -> String {

    let r = post(

        app,

        "/register",

        None,

        &format!("username={user}&email={user}%40example.com&password=correct-horse-battery&setup=test-setup"),

    )

    .await;

    assert_eq!(r.status, StatusCode::SEE_OTHER, "{}", r.body);

    r.cookie.unwrap()

}

fn csrf_of(page: &str) -> String {

    let i = page.find("name=\"_csrf\" value=\"").expect("csrf field") + "name=\"_csrf\" value=\"".len();

    page[i..].split('"').next().unwrap().to_string()

}

/// A personal access token for `user`, marked as an agent's when `agent`

/// names one.

async fn token(app: &Shared, user: &str, agent: Option<&str>) -> String {

    let me = db::accounts::account_by_name(&app.db, user).await.unwrap().unwrap();

    let (secret, hash) = oxygenhub::auth::new_token();

    let t = db::accounts::create_token(&app.db, me.id, agent.unwrap_or("laptop"), &hash, "repo:read,repo:write", "", None).await.unwrap();

    if agent.is_some() {

        db::accounts::set_token_agent(&app.db, me.id, t.id, true).await.unwrap();

    }

    secret

}

/// andy/demo, public, with a client checkout at `work` pushing as `tok`

/// and its first change pushed.

async fn demo(h: &Hub, work: &Path, tok: &str) -> (Repo, ChangeHash) {

    let repo = RepoName::parse("andy/demo").unwrap();

    h.app.repos.create(&repo, None).unwrap();

    db::repos::create(&h.app.db, "andy", "demo", "", hub_api::Visibility::Public).await.unwrap();

    let mut client = Repo::init(work).unwrap();

    client.create_identity("Andy McCall", "[email protected]", oxy_core::repo::IdentityScope::User).unwrap();

    client.config.set("remote.origin.url", &format!("{}/andy/demo", h.base)).unwrap();

    client.config.set("remote.origin.token", tok).unwrap();

    client.save_config().unwrap();

    write(work, "README.md", b"one\ntwo\nthree\n");

    client.snapshot("t").unwrap();

    let base = client.commit("base", &[]).unwrap();

    // A push without secrets is unaffected.

    assert!(client.push("origin", false).unwrap().iter().all(|r| r.ok));

    (client, base)

}

fn hosted_heads(app: &Shared) -> Vec<ChangeHash> {

    app.repos.open(&RepoName::parse("andy/demo").unwrap()).unwrap().bookmark_heads().unwrap()

}

/// A PEM private key made for this test and thrown away.

fn pem_key() -> String {

    rcgen::KeyPair::generate().unwrap().serialize_pem()

}

/// What a file should be refused for: (rule, secret, line).

type Expected = Vec<(&'static str, String, usize)>;

/// Each kind of secret: (file, content, what it is refused for).

fn samples() -> Vec<(String, String, Expected)> {

    let aws_id = format!("{}{}", "AKIA", "IOSFODNN7EXAMPLE");

    let aws_secret = format!("{}{}", "wJalrXUtnFEMI/K7MDENG/", "bPxRfiCYEXAMPLEKEY");

    let pem = pem_key();

    let gcp_key = pem.replace('\n', "\\n");

    let azure_key = format!("{}==", random_hex(43).chars().map(|c| if c == 'a' { 'Z' } else { c }).collect::<String>());

    let ghp = format!("{}_{}", "ghp", random_hex(18));

    let pat = format!("{}_{}_{}{}", "github_pat", random_hex(11), random_hex(29), "x");

    let glpat = format!("{}-{}", "glpat", random_hex(10));

    let oxyhub = format!("{}_{}", "oxyhub", random_hex(32));

    let grant = format!("{}_{}", "oxyagent", random_hex(32));

    let job = format!("{}_{}", "flowjob", random_hex(32));

    let slack = format!("{}-{}-{}-{}", "xoxb", "123456789012", "1234567890123", random_hex(12));

    let stripe = format!("{}_{}_{}", "sk", "live", random_hex(12));

    let ssh_body = (0..4).map(|_| random_hex(30)).collect::<Vec<_>>().join("\n");

    let ssh = format!("-----{} OPENSSH PRIVATE KEY-----\n{ssh_body}\n-----{} OPENSSH PRIVATE KEY-----", "BEGIN", "END");

    let head = "# settings\n\n";

    vec![

        (

            "aws/credentials".into(),

            format!("[default]\naws_access_key_id = {aws_id}\naws_secret_access_key = {aws_secret}\n"),

            vec![("aws-access-key-id", aws_id, 2), ("aws-secret-access-key", aws_secret, 3)],

        ),

        (

            "gcp/key.json".into(),

            format!(

                "{{\n  \"type\": \"service_account\",\n  \"project_id\": \"example\",\n  \"private_key\": \"{gcp_key}\",\n  \"client_email\": \"[email protected]\"\n}}\n"

            ),

            vec![("gcp-service-account", gcp_key, 4)],

        ),

        (

            "azure.env".into(),

            format!("{head}AZURE=DefaultEndpointsProtocol=https;AccountName=example;AccountKey={azure_key};EndpointSuffix=core.windows.net\n"),

            vec![("azure-storage-connection-string", azure_key, 3)],

        ),

        ("gh.env".into(), format!("{head}GH_TOKEN={ghp}\nGH_PAT={pat}\n"), vec![("github-token", ghp, 3), ("github-fine-grained-token", pat, 4)]),

        ("gl.env".into(), format!("{head}GITLAB_TOKEN={glpat}\n"), vec![("gitlab-token", glpat, 3)]),

        (

            "hub.env".into(),

            format!("{head}OXY_TOKEN={oxyhub}\nGRANT={grant}\nJOB={job}\n"),

            vec![("oxygenhub-token", oxyhub, 3), ("oxygenhub-agent-grant", grant, 4), ("oxygenflow-job-token", job, 5)],

        ),

        ("slack.env".into(), format!("{head}SLACK={slack}\n"), vec![("slack-token", slack, 3)]),

        ("stripe.env".into(), format!("{head}STRIPE={stripe}\n"), vec![("stripe-live-key", stripe, 3)]),

        ("keys/tls.key".into(), format!("{head}{pem}"), vec![("private-key", pem.trim_end().to_string(), 3)]),

        ("keys/id_ed25519".into(), format!("{head}{ssh}\n"), vec![("openssh-private-key", ssh, 3)]),

    ]

}

#[tokio::test(flavor = "multi_thread")]

async fn a_push_adding_a_secret_is_refused_until_allowed() {

    let h = hub(None).await;

    let app = &h.app;

    let andy = register(app, "andy").await;

    let tok = token(app, "andy", None).await;

    let work = tempfile::tempdir().unwrap();

    let (mut client, base) = demo(&h, work.path(), &tok).await;

    assert!(db::repos::get(&app.db, "andy", "demo").await.unwrap().unwrap().push_protection, "on by default");

    // Each kind, alone, is refused with where it is and its fingerprint,

    // the secret itself never shown; nothing is stored or moved.

    for (file, content, expect) in samples() {

        write(work.path(), &file, content.as_bytes());

        client.snapshot("t").unwrap();

        let h1 = client.commit(&format!("add {file}"), &[]).unwrap();

        let err = client.push("origin", false).err().unwrap_or_else(|| panic!("{file} was accepted")).to_string();

        assert!(err.contains("HTTP 422") && err.contains("push protection refused this push"), "{err}");

        assert!(err.contains("oxy amend") && err.contains(&format!("{}/andy/demo/settings#push-protection", h.base)), "{err}");

        assert!(!err.contains("hint:"), "no credential hint for a refusal: {err}");

        for (rule, secret, line) in &expect {

            let fp = fingerprint(rule, secret.as_bytes());

            assert!(err.contains(&format!("{file}:{line}: ")) && err.contains(&format!("(fingerprint {fp})")), "{rule} in {file}: {err}");

            assert!(!err.contains(secret.as_str()), "{rule}: the secret is shown: {err}");

        }

        assert_eq!(err.matches("(fingerprint ").count(), expect.len(), "{err}");

        assert_eq!(hosted_heads(app), vec![base], "nothing moved");

        let hosted = app.repos.open(&RepoName::parse("andy/demo").unwrap()).unwrap();

        assert!(hosted.store().get_change(&h1).is_err(), "nothing stored");

        client.abandon(&h1).unwrap();

        let _ = std::fs::remove_file(work.path().join(&file));

        client.snapshot("t").unwrap();

    }

    // A secret inserted into an existing file is placed at its line there.

    let ghp = format!("{}_{}", "ghp", random_hex(18));

    write(work.path(), "README.md", format!("one\ntwo\n{ghp}\nthree\n").as_bytes());

    client.snapshot("t").unwrap();

    let edit = client.commit("token in the readme", &[]).unwrap();

    let err = client.push("origin", false).err().unwrap().to_string();

    let fp = fingerprint("github-token", ghp.as_bytes());

    assert!(err.contains(&format!("README.md:3: GitHub token (fingerprint {fp})")), "{err}");

    // The refusal is in the push log and among the findings in Settings.

    let log = db::repos::pushes(&app.db, 1, 10).await.unwrap();

    assert!(

        log.iter().any(|p| !p.ok && p.reference == "main" && p.reason.contains("push protection") && p.reason.contains("README.md:3")),

        "{log:?}"

    );

    let settings = get(app, "/andy/demo/settings", Some(&andy)).await;

    assert!(settings.body.contains("id=\"push-protection\"") && settings.body.contains(&fp), "{}", settings.body);

    assert!(!settings.body.contains(&ghp), "the settings page shows no secret");

    // A writer cannot allow it.

    let bob = register(app, "bob").await;

    let csrf = csrf_of(&settings.body);

    let r = post(app, "/andy/demo/settings/collaborators", Some(&andy), &format!("_csrf={csrf}&username=bob&role=write")).await;

    assert_eq!(r.status, StatusCode::OK, "{}", r.body);

    let bob_page = get(app, "/andy/demo", Some(&bob)).await;

    let bob_csrf = csrf_of(&bob_page.body);

    let r = post(app, "/andy/demo/settings/secret-bypass", Some(&bob), &format!("_csrf={bob_csrf}&fingerprint={fp}&reason=mine")).await;

    assert_eq!(r.status, StatusCode::NOT_FOUND, "write is not maintain");

    assert!(db::push_protection::allowed(&app.db, 1).await.unwrap().is_empty());

    // A maintainer allows it, with a reason, audited; the push then lands.

    let r = post(app, "/andy/demo/settings/secret-bypass", Some(&andy), &format!("_csrf={csrf}&fingerprint={fp}&reason=")).await;

    assert!(r.body.contains("Say why"), "a reason is needed");

    let r =

        post(app, "/andy/demo/settings/secret-bypass", Some(&andy), &format!("_csrf={csrf}&fingerprint={fp}&reason=a+revoked+test+token"))

            .await;

    assert_eq!(r.status, StatusCode::OK, "{}", r.body);

    assert!(r.body.contains(&format!("Fingerprint {fp} is allowed")) && r.body.contains("a revoked test token"), "{}", r.body);

    let audit = db::accounts::audit_rows(&app.db, 20).await.unwrap();

    assert!(audit.iter().any(|a| a.action == "secret.bypass" && a.detail.contains(&fp) && a.detail.contains("github-token")), "{audit:?}");

    assert!(client.push("origin", false).unwrap().iter().all(|r| r.ok));

    assert_eq!(hosted_heads(app), vec![edit]);

    // An agent's push is told what an agent should do.

    let agent_tok = token(app, "andy", Some("Claude")).await;

    client.config.set("remote.origin.token", &agent_tok).unwrap();

    client.save_config().unwrap();

    let stripe = format!("{}_{}_{}", "sk", "live", random_hex(12));

    write(work.path(), "pay.env", format!("STRIPE={stripe}\n").as_bytes());

    client.snapshot("t").unwrap();

    client.commit("pay", &[]).unwrap();

    let err = client.push("origin", false).err().unwrap().to_string();

    assert!(err.contains("an agent's token (Claude)") && err.contains("do not try to get around the refusal"), "{err}");

    let settings = get(app, "/andy/demo/settings", Some(&andy)).await;

    assert!(settings.body.contains("via Claude"), "the finding names the agent");

    // With push protection off, the same push lands; the change is audited.

    let csrf = csrf_of(&settings.body);

    let r = post(app, "/andy/demo/settings/push-protection", Some(&andy), &format!("_csrf={csrf}")).await;

    assert_eq!(r.status, StatusCode::OK, "{}", r.body);

    assert!(!db::repos::get(&app.db, "andy", "demo").await.unwrap().unwrap().push_protection);

    let audit = db::accounts::audit_rows(&app.db, 20).await.unwrap();

    assert!(audit.iter().any(|a| a.action == "repo.push_protection" && a.detail == "off"), "{audit:?}");

    assert!(client.push("origin", false).unwrap().iter().all(|r| r.ok));

    let r = post(app, "/andy/demo/settings/push-protection", Some(&andy), &format!("_csrf={csrf}&protect=on")).await;

    assert_eq!(r.status, StatusCode::OK, "{}", r.body);

    assert!(db::repos::get(&app.db, "andy", "demo").await.unwrap().unwrap().push_protection);

}

/// Past the scan budget the rest of a push goes unscanned: it is accepted,

/// and its push log entry says how much was not scanned.

#[tokio::test(flavor = "multi_thread")]

async fn a_push_past_the_scan_budget_is_accepted_and_says_so() {

    let h = hub(Some(1024)).await;

    let app = &h.app;

    register(app, "andy").await;

    let tok = token(app, "andy", None).await;

    let work = tempfile::tempdir().unwrap();

    let (client, _) = demo(&h, work.path(), &tok).await;

    let filler: String = (0..64).map(|i| format!("line {i} of a long generated file\n")).collect();

    let stripe = format!("{}_{}_{}", "sk", "live", random_hex(12));

    write(work.path(), "big.txt", format!("{filler}STRIPE={stripe}\n").as_bytes());

    client.snapshot("t").unwrap();

    let big = client.commit("big", &[]).unwrap();

    assert!(client.push("origin", false).unwrap().iter().all(|r| r.ok));

    assert!(hosted_heads(app).contains(&big));

    let log = db::repos::pushes(&app.db, 1, 10).await.unwrap();

    let mut said = false;

    for _ in 0..50 {

        said = db::repos::pushes(&app.db, 1, 10)

            .await

            .unwrap()

            .iter()

            .any(|p| p.ok && p.reason.contains("stopped at its size budget") && p.reason.contains("not scanned"));

        if said {

            break;

        }

        tokio::time::sleep(std::time::Duration::from_millis(50)).await;

    }

    assert!(said, "{log:?}");

}

 "memchr",

# Push protection's keyword prefilter (oxygenhub#268); already in the tree through regex.

memchr = "2"

    /// Push protection's budget (oxygenhub#268): what one push may have

    /// scanned, the largest file or inserted text scanned, and how long

    /// the scan may take. Past them the rest goes unscanned.

    pub scan_limit: ByteSize,

    pub scan_file_limit: ByteSize,

    pub scan_time_ms: u64,

        let scan = crate::push_protection::Budget::default();

        Limits {

            render_limit: ByteSize(2 << 20),

            asset_limit: ByteSize(512 << 20),

            push_body_limit: ByteSize(4 << 30),

            scan_limit: ByteSize(scan.bytes),

            scan_file_limit: ByteSize(scan.file),

            scan_time_ms: scan.time.as_millis() as u64,

        }

pub mod push_protection;

    /// Pushes that add possible secrets are refused (#268, migration

    /// 0085). On by default.

    #[sqlx(default)]

    pub push_protection: bool,