push_protection.rs 90.3%

337 of 373 lines run, 55 of 82 functions · run #504 on main · run missed
//! Push protection (hub spec §15.4, oxygenhub#268): a push that adds
//! something that looks like a credential is refused before any of it is
//! stored, saying where, so it never lands in a repository's history.
//!
//! Only what a push adds is read: the lines its changes insert and the new
//! blobs it carries, never history the hub already holds. The scan has a
//! budget ([`Budget`]: bytes per push, the largest file, time), so a push
//! stays fast; past it the rest goes unscanned, the push is accepted and its
//! log entry says how much was left (a refusal would block every large
//! import for what is most likely nothing). Rules are data, in
//! `push_protection.toml`, compiled in and versioned. What a repository
//! already holds is scanned with the same rules by the history scan
//! ([`crate::history_scan`], #277).
//!
//! A finding names its rule, file and line, a fingerprint (BLAKE3 of the
//! rule id and the secret, the first 16 hex digits) and an excerpt with the
//! secret masked. A maintainer lets one finding through by its fingerprint
//! (Settings › Push protection); the setting itself is per repository and
//! on by default.

use oxy_core::hash::ObjectType;
use oxy_core::hash::{ChangeHash, ObjectRef};
use oxy_core::ids::NodeId;
use oxy_core::objects::{Edit, TreeOp};
use oxy_core::protocol::Bundle;
use oxy_core::repo::Repo;
use regex::bytes::Regex;
use serde::Deserialize;
use std::collections::{HashMap, HashSet};
use std::sync::OnceLock;
use std::time::{Duration, Instant};

/// The rules, as compiled in.
const RULES: &str = include_str!("push_protection.toml");

/// How much of a push is scanned: past `bytes` in all, or `time`, the rest
/// is not; a file or inserted text over `file` bytes, or binary, is skipped.
#[derive(Clone, Copy, Debug)]
pub struct Budget {
    pub bytes: u64,
    pub file: u64,
    pub time: Duration,
}

impl Default for Budget {
    fn default() -> Self {
        Budget { bytes: 16 << 20, file: 1 << 20, time: Duration::from_secs(2) }
    }
}

#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct RulesFile {
    version: u32,
    rule: Vec<RuleSpec>,
}

#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct RuleSpec {
    id: String,
    description: String,
    regex: String,
    #[serde(default)]
    keywords: Vec<String>,
    #[serde(default)]
    requires: Option<String>,
}

/// One compiled rule.
pub struct Rule {
    pub id: String,
    pub description: String,
    regex: Regex,
    /// Lower-cased; empty runs the regex on every text.
    keywords: Vec<String>,
    requires: Option<Regex>,
}

/// The compiled rule set.
pub struct Rules {
    pub version: u32,
    pub rules: Vec<Rule>,
}

/// The compiled-in rules. A rule that does not compile is a build defect,
/// caught by this module's tests.
pub fn rules() -> &'static Rules {
    static RULESET: OnceLock<Rules> = OnceLock::new();
    RULESET.get_or_init(|| compile(RULES).expect("push_protection.toml"))
}

fn compile(text: &str) -> Result<Rules, String> {
    let file: RulesFile = toml::from_str(text).map_err(|e| e.to_string())?;
    let mut rules = Vec::new();
    let mut seen = HashSet::new();
    for r in file.rule {
        if !seen.insert(r.id.clone()) {
            return Err(format!("rule {} is listed twice", r.id));
        }
        let regex = Regex::new(&r.regex).map_err(|e| format!("rule {}: {e}", r.id))?;
        let requires = match r.requires {
            Some(q) => Some(Regex::new(&q).map_err(|e| format!("rule {} requires: {e}", r.id))?),
            None => None,
        };
        rules.push(Rule {
            id: r.id,
            description: r.description,
            regex,
            keywords: r.keywords.iter().map(|k| k.to_ascii_lowercase()).collect(),
            requires,
        });
    }
    Ok(Rules { version: file.version, rules })
}

/// Where a finding's text came from in the push.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Origin {
    /// Lines an insert adds; `atom` is the finding's line's atom index in
    /// its change (spec §7.4), counted per file.
    Insert { atom: u32 },
    /// A new blob replacing a file's content.
    Blob,
}

/// A possible secret a push adds.
#[derive(Clone, Debug)]
pub struct Finding {
    pub rule: String,
    pub description: String,
    /// The change that adds it: its index in the bundle and its hash.
    pub change: usize,
    pub change_hash: Option<ChangeHash>,
    pub node: NodeId,
    pub origin: Origin,
    /// The file's path, once [`locate`] found it.
    pub path: String,
    /// 1-based: in the file once [`locate`] worked it out, else in the
    /// inserted text or blob.
    pub line: usize,
    /// The first 16 hex digits of BLAKE3(rule id, NUL, secret).
    pub fingerprint: String,
    /// The finding's line, the secret masked.
    pub excerpt: String,
}

/// What a scan found and how much it read.
#[derive(Clone, Debug, Default)]
pub struct Scan {
    pub findings: Vec<Finding>,
    /// Bytes scanned.
    pub scanned: u64,
    /// Texts and blobs skipped as over the file budget or binary.
    pub skipped: usize,
    /// Bytes left unscanned when the budget ran out, and why.
    pub cut_short: Option<(u64, &'static str)>,
}

impl Scan {
    /// One line for the push log when the scan stopped early.
    pub fn note(&self) -> Option<String> {
        let (left, why) = self.cut_short?;
        Some(format!(
            "push protection scanned {} and stopped at its {why} budget: {} more were not scanned",
            human(self.scanned),
            human(left)
        ))
    }
}

fn human(n: u64) -> String {
    match n {
        n if n >= 1 << 20 => format!("{:.1} MiB", n as f64 / (1u64 << 20) as f64),
        n if n >= 1 << 10 => format!("{:.1} KiB", n as f64 / 1024.0),
        n => format!("{n} bytes"),
    }
}

/// The fingerprint of `secret` under `rule`.
pub fn fingerprint(rule: &str, secret: &[u8]) -> String {
    let mut h = blake3::Hasher::new();
    h.update(rule.as_bytes());
    h.update(&[0]);
    h.update(secret);
    h.finalize().to_hex()[..16].to_string()
}

/// A finding in one text: its rule, 1-based line, fingerprint and excerpt.
#[derive(Clone, Debug)]
pub struct Hit {
    pub rule: usize,
    pub line: usize,
    pub fingerprint: String,
    pub excerpt: String,
}

/// Every rule over one text. Matches overlapping an earlier rule's are
/// dropped, so one secret is reported once, by its most specific rule.
pub fn scan_text(rules: &Rules, text: &[u8]) -> Vec<Hit> {
    let mut hits = Vec::new();
    let mut taken: Vec<(usize, usize)> = Vec::new();
    let mut lower: Option<Vec<u8>> = None;
    for (i, rule) in rules.rules.iter().enumerate() {
        if !rule.keywords.is_empty() {
            let lower = lower.get_or_insert_with(|| text.to_ascii_lowercase());
            if !rule.keywords.iter().any(|k| memchr::memmem::find(lower, k.as_bytes()).is_some()) {
                continue;
            }
        }
        let mut required: Option<bool> = None;
        for caps in rule.regex.captures_iter(text) {
            let m = caps.name("secret").or_else(|| caps.get(0)).expect("a match");
            let (start, end) = (m.start(), m.end());
            if start == end || taken.iter().any(|(s, e)| start < *e && *s < end) {
                continue;
            }
            if let Some(q) = &rule.requires {
                if !*required.get_or_insert_with(|| q.is_match(text)) {
                    break;
                }
            }
            taken.push((start, end));
            hits.push(Hit {
                rule: i,
                line: memchr::memchr_iter(b'\n', &text[..start]).count() + 1,
                fingerprint: fingerprint(&rule.id, m.as_bytes()),
                excerpt: excerpt(text, start, end),
            });
        }
    }
    hits
}

/// The line holding `text[start..end]` with the secret masked: its first
/// four characters kept, the rest starred, at most 160 characters around.
fn excerpt(text: &[u8], start: usize, end: usize) -> String {
    let line_start = memchr::memrchr(b'\n', &text[..start]).map(|i| i + 1).unwrap_or(0);
    let line_end = memchr::memchr(b'\n', &text[start..]).map(|i| start + i).unwrap_or(text.len());
    let secret_end = end.min(line_end);
    let kept = (secret_end - start).min(4);
    let before_from = start.saturating_sub(60).max(line_start);
    let after_to = (secret_end + 40).min(line_end);
    let mut out = String::new();
    if before_from > line_start {
        out.push('…');
    }
    out.push_str(&String::from_utf8_lossy(&text[before_from..start + kept]));
    out.push_str(&"*".repeat((secret_end - start - kept).clamp(4, 16)));
    let mut after = String::from_utf8_lossy(&text[secret_end..after_to]).into_owned();
    // The same secret again on the line is masked too.
    let secret = String::from_utf8_lossy(&text[start..secret_end]).into_owned();
    if secret.len() > 4 {
        after = after.replace(&secret, "****");
    }
    out.push_str(&after);
    if after_to < line_end {
        out.push('…');
    }
    out.trim_end_matches(['\r', '\n']).trim().to_string()
}

/// Scan what `bundle` adds: each insert's lines and each new blob a change
/// puts in place of a file's content, in the push's order, within
/// `budget`. Past the size or time budget the rest is only counted.
pub fn scan(bundle: &Bundle, budget: Budget) -> Scan {
    let rules = rules();
    let started = Instant::now();
    let mut out = Scan::default();
    // The bundle's blobs and chunks by reference, made when a change first
    // replaces a file's content.
    let mut objects: Option<HashMap<ObjectRef, usize>> = None;
    let mut stop: Option<&'static str> = None;
    let mut left: u64 = 0;
    for (ci, (change, _)) in bundle.changes.iter().enumerate() {
        let mut counters: HashMap<NodeId, u32> = HashMap::new();
        for e in &change.edits {
            let (node, origin, size) = match e {
                Edit::Insert { node, lines, .. } => {
                    let idx = counters.entry(*node).or_default();
                    let first = *idx;
                    *idx += lines.len() as u32;
                    let n: usize = lines.iter().map(<[u8]>::len).sum();
                    (*node, Origin::Insert { atom: first }, n as u64)
                }
                Edit::Replace { node, new: Some(r), .. } => {
                    let objects = objects.get_or_insert_with(|| {
                        bundle
                            .objects
                            .iter()
                            .enumerate()
                            .filter(|(_, (ty, _))| matches!(ty, ObjectType::Blob | ObjectType::Chunk))
                            .map(|(i, (ty, body))| (oxy_core::hash::hash_object(*ty, body), i))
                            .collect()
                    });
                    // A blob the push does not carry is already on the hub
                    // (or lazy): nothing new.
                    let Some(blob) = blob_of(bundle, objects, r) else { continue };
                    (*node, Origin::Blob, blob.length)
                }
                _ => continue,
            };
            if size > budget.file {
                out.skipped += 1;
                continue;
            }
            if stop.is_none() {
                if out.scanned + size > budget.bytes {
                    stop = Some("size");
                } else if started.elapsed() > budget.time {
                    stop = Some("time");
                }
            }
            if stop.is_some() {
                left += size;
                continue;
            }
            let blob_text;
            let text: &[u8] = match (e, &origin) {
                (Edit::Insert { lines, .. }, _) => &lines.bytes()[..size as usize],
                (Edit::Replace { new: Some(r), .. }, Origin::Blob) => {
                    let objects = objects.as_ref().expect("made above");
                    match blob_content(bundle, objects, r) {
                        Some(t) if memchr::memchr(0, &t[..t.len().min(8000)]).is_none() => {
                            blob_text = t;
                            &blob_text
                        }
                        // Binary, as Git and oxy judge it (a NUL early on),
                        // or incomplete.
                        _ => {
                            out.skipped += 1;
                            continue;
                        }
                    }
                }
                _ => continue,
            };
            for hit in scan_text(rules, text) {
                let rule = &rules.rules[hit.rule];
                let origin = match origin {
                    Origin::Insert { atom } => Origin::Insert { atom: atom + hit.line as u32 - 1 },
                    Origin::Blob => Origin::Blob,
                };
                out.findings.push(Finding {
                    rule: rule.id.clone(),
                    description: rule.description.clone(),
                    change: ci,
                    change_hash: bundle.hashes.get(ci).copied(),
                    node,
                    origin,
                    path: String::new(),
                    line: hit.line,
                    fingerprint: hit.fingerprint,
                    excerpt: hit.excerpt,
                });
            }
            out.scanned += size;
        }
    }
    if let Some(why) = stop {
        out.cut_short = Some((left, why));
    }
    out
}

fn object<'b>(bundle: &'b Bundle, objects: &HashMap<ObjectRef, usize>, r: &ObjectRef, want: ObjectType) -> Option<&'b [u8]> {
    match objects.get(r).map(|i| &bundle.objects[*i]) {
        Some((ty, body)) if *ty == want => Some(body),
        _ => None,
    }
}

fn blob_of(bundle: &Bundle, objects: &HashMap<ObjectRef, usize>, r: &ObjectRef) -> Option<oxy_core::objects::Blob> {
    oxy_core::objects::Blob::decode(object(bundle, objects, r, ObjectType::Blob)?).ok()
}

/// A blob's content from the bundle's chunks; `None` when one is missing.
fn blob_content(bundle: &Bundle, objects: &HashMap<ObjectRef, usize>, r: &ObjectRef) -> Option<Vec<u8>> {
    let blob = blob_of(bundle, objects, r)?;
    let mut content = Vec::with_capacity(blob.length as usize);
    for c in &blob.chunks {
        content.extend_from_slice(object(bundle, objects, c, ObjectType::Chunk)?);
    }
    Some(content)
}

/// Fill in each finding's path, and its line in the file as the change that
/// adds it leaves the file, from `repo` with the push's changes applied in
/// memory. Done only when there are findings: it loads the state the push
/// builds on. A finding it cannot place keeps its line in the inserted text
/// and is named by the push's own tree operations, or by its node.
pub fn locate(repo: &Repo, bundle: &Bundle, findings: &mut [Finding]) {
    let mut names: HashMap<NodeId, (NodeId, String)> = HashMap::new();
    for (c, _) in &bundle.changes {
        for op in &c.tree_ops {
            match op {
                TreeOp::AddNode { node, parent, name, .. } | TreeOp::Move { node, parent, name, .. } => {
                    names.insert(*node, (*parent, name.clone()));
                }
                _ => {}
            }
        }
    }
    let placed = locate_in_state(repo, bundle, findings).unwrap_or_default();
    for (i, f) in findings.iter_mut().enumerate() {
        if let Some((path, line)) = placed.get(&i) {
            f.path = path.clone();
            if let Some(line) = line {
                f.line = *line;
            }
            continue;
        }
        let mut parts = Vec::new();
        let mut cur = f.node;
        while let Some((parent, name)) = names.get(&cur) {
            parts.push(name.clone());
            cur = *parent;
            if parts.len() > 256 {
                break;
            }
        }
        parts.reverse();
        f.path = if parts.is_empty() { format!("(file {})", &f.node.to_hex()[..12]) } else { parts.join("/") };
    }
}

/// Each finding's (path, line) by index, as the state after its change has
/// it; `None` when that state cannot be built.
fn locate_in_state(repo: &Repo, bundle: &Bundle, findings: &[Finding]) -> Option<HashMap<usize, (String, Option<usize>)>> {
    if bundle.hashes.len() != bundle.changes.len() {
        return None;
    }
    let own: HashSet<ChangeHash> = bundle.hashes.iter().copied().collect();
    let mut outside: Vec<ChangeHash> =
        bundle.changes.iter().flat_map(|(c, _)| c.deps.iter().copied()).filter(|d| !own.contains(d)).collect();
    outside.sort();
    outside.dedup();
    let mut state = repo.load_state(&outside).ok()?;
    let mut out = HashMap::new();
    for (ci, (change, _)) in bundle.changes.iter().enumerate() {
        state.add_hashed(bundle.hashes[ci], change.clone()).ok()?;
        let here: Vec<usize> = (0..findings.len()).filter(|i| findings[*i].change == ci).collect();
        if here.is_empty() {
            continue;
        }
        let view = state.view();
        let by_node = view.by_node();
        let mut lines: HashMap<NodeId, Vec<(ChangeHash, u32)>> = HashMap::new();
        for i in here {
            let f = &findings[i];
            let Some(entry) = by_node.get(&f.node) else { continue };
            let line = match f.origin {
                Origin::Insert { atom } => {
                    let live = lines.entry(f.node).or_insert_with(|| state.live_lines(&f.node).into_iter().map(|(k, _)| k).collect());
                    live.iter().position(|k| *k == (bundle.hashes[ci], atom)).map(|p| p + 1)
                }
                Origin::Blob => None,
            };
            out.insert(i, (entry.path.clone(), line));
        }
    }
    Some(out)
}

/// Findings listed in a refusal; the rest are counted.
const LISTED: usize = 20;

/// The refusal's text: each finding as `path:line: description
/// (fingerprint F)` with its masked excerpt, then what to do. `via` names
/// the agent whose token pushed, if one did; `settings` is where a
/// maintainer allows a finding.
/// `settings` is the repository's settings page, or empty for a drop.
pub fn refusal(findings: &[Finding], settings: &str, via: &str) -> String {
    use std::fmt::Write;
    let n = findings.len();
    let mut out = format!(
        "push protection refused this push: it adds {} that look{} like {}, and nothing of it was stored (rules v{}).\n\n",
        if n == 1 { "a value".to_string() } else { format!("{n} values") },
        if n == 1 { "s" } else { "" },
        if n == 1 { "a secret" } else { "secrets" },
        rules().version
    );
    for f in findings.iter().take(LISTED) {
        let _ = writeln!(out, "  {}:{}: {} (fingerprint {})", f.path, f.line, f.description, f.fingerprint);
        if !f.excerpt.is_empty() {
            let _ = writeln!(out, "      {}", f.excerpt);
        }
    }
    if n > LISTED {
        let _ = writeln!(out, "  … and {} more", n - LISTED);
    }
    out.push_str("\nWhat to do:\n");
    let (it, is) = if n == 1 { ("it", "it is") } else { ("each", "one is") };
    let _ = writeln!(
        out,
        "  - Take {it} out of the change that adds it: edit the file, then `oxy amend` (or `oxy abandon` a change that only adds it), and push again. Read credentials from the environment or a secret store, not from the repository."
    );
    let _ = writeln!(out, "  - If {is} a real credential, revoke or rotate it as well: it has been on your disk and may be elsewhere.");
    if settings.is_empty() {
        // A drop (#279) has no settings: nothing there can be allowed.
        let _ = writeln!(out, "  - A drop cannot allow a finding: if {is} not a secret, change the value so it no longer looks like one.");
    } else {
        let _ = writeln!(
            out,
            "  - If {is} not a secret (an example, a test value), a maintainer can allow its fingerprint, with a reason, at {settings}/push-protection; then push again."
        );
    }
    if !via.is_empty() && settings.is_empty() {
        let _ = write!(
            out,
            "\nThis push came through an agent's token ({via}). As an agent: remove the value from the change (`oxy amend`) and push again; do not try to get around the refusal. If you were asked to commit this value, stop and tell the person you work for."
        );
    } else if !via.is_empty() {
        let _ = write!(
            out,
            "\nThis push came through an agent's token ({via}). As an agent: remove the value from the change (`oxy amend`) and push again; do not try to get around the refusal. If you were asked to commit this value, stop and tell the person you work for; only a maintainer can allow a fingerprint, at {settings}/push-protection."
        );
    }
    out.trim_end().to_string()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn the_rules_compile() {
        let r = rules();
        assert!(r.version >= 1);
        assert!(r.rules.len() >= 15);
    }

    #[test]
    fn the_rules_file_is_not_itself_a_finding() {
        assert!(scan_text(rules(), RULES.as_bytes()).is_empty());
    }

    #[test]
    fn the_hub_s_own_push_protection_sources_pass_it() {
        for (name, text) in [
            ("push_protection.rs", include_str!("push_protection.rs")),
            ("tests/push_protection.rs", include_str!("../tests/push_protection.rs")),
            ("docs/spec.md", include_str!("../../../docs/spec.md")),
            // AWS's documented example keys, put together at run time (#277).
            ("backups.rs", include_str!("backups.rs")),
            ("history_scan.rs", include_str!("history_scan.rs")),
            ("tests/history_scan.rs", include_str!("../tests/history_scan.rs")),
        ] {
            assert!(scan_text(rules(), text.as_bytes()).is_empty(), "{name}");
        }
    }

    #[test]
    fn a_secret_is_found_masked_and_fingerprinted() {
        let key = format!("{}{}", "AKIA", "IOSFODNN7EXAMPLE");
        let text = format!("[default]\naws_access_key_id = {key}\n");
        let hits = scan_text(rules(), text.as_bytes());
        assert_eq!(hits.len(), 1, "{hits:?}");
        assert_eq!(rules().rules[hits[0].rule].id, "aws-access-key-id");
        assert_eq!(hits[0].line, 2);
        assert_eq!(hits[0].fingerprint, fingerprint("aws-access-key-id", key.as_bytes()));
        assert_eq!(hits[0].fingerprint.len(), 16);
        assert!(!hits[0].excerpt.contains(&key) && hits[0].excerpt.starts_with("aws_access_key_id = AKIA*"), "{}", hits[0].excerpt);
    }

    #[test]
    fn an_aws_secret_needs_its_key_id_nearby() {
        let secret = format!("{}{}", "wJalrXUtnFEMI/K7MDENG/", "bPxRfiCYEXAMPLEKEY");
        let alone = format!("aws_secret_access_key = {secret}\n");
        assert!(scan_text(rules(), alone.as_bytes()).is_empty());
        let paired = format!("aws_access_key_id = {}{}\n{alone}", "AKIA", "IOSFODNN7EXAMPLE");
        let ids: Vec<&str> = scan_text(rules(), paired.as_bytes()).iter().map(|h| rules().rules[h.rule].id.as_str()).collect();
        assert_eq!(ids, ["aws-access-key-id", "aws-secret-access-key"]);
    }

    #[test]
    fn a_service_account_key_is_reported_once() {
        let pem = format!("-----BEGIN {}-----\\n{}\\n-----END {}-----\\n", "PRIVATE KEY", "A".repeat(64), "PRIVATE KEY");
        let json = format!("{{\n  \"type\": \"service_account\",\n  \"private_key\": \"{pem}\"\n}}\n");
        let hits = scan_text(rules(), json.as_bytes());
        assert_eq!(hits.len(), 1, "{hits:?}");
        assert_eq!(rules().rules[hits[0].rule].id, "gcp-service-account");
        assert_eq!(hits[0].line, 3);
    }

    #[test]
    fn a_pem_header_alone_is_not_a_key() {
        let doc = format!("A key starts with -----BEGIN {}-----.\n", "RSA PRIVATE KEY");
        assert!(scan_text(rules(), doc.as_bytes()).is_empty());
    }
}
`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet
Coverage / crates/hub/src/push_protection.rs 90.3%

Keyboard
