//! The history scan (oxygenhub#277, hub spec §15.4): a secret that reached

//! a repository before push protection existed, or while it was off, is

//! found in its history with the change, file and line; a maintainer

//! allows it or marks it resolved; an allowed fingerprint can be removed;

//! after a push only the new changes are scanned; a pass stops at its

//! budget and the next resumes there; the worker scans every repository

//! when it starts and each after a push.

//!

//! Every credential here is made up and put together at run time, so this

//! file is not itself a finding.

use axum::body::Body;

use axum::http::{header, Request, StatusCode};

use http_body_util::BodyExt;

use oxy_core::hash::ChangeHash;

use oxy_core::repo::Repo;

use oxygenhub::auth::random_hex;

use oxygenhub::config::Config;

use oxygenhub::db;

use oxygenhub::history_scan::{pass, Budget};

use oxygenhub::push_protection::fingerprint;

use oxygenhub::repos::RepoName;

use oxygenhub::Shared;

use std::path::Path;

use std::time::Duration;

use tower::ServiceExt;

fn hermetic() {

    static ONCE: std::sync::Once = std::sync::Once::new();

    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();

    ONCE.call_once(|| {

        let d = tempfile::tempdir().unwrap();

        std::env::set_var("OXY_HOME", d.path());

        DIR.set(d).ok();

    });

}

fn write(root: &Path, rel: &str, content: &[u8]) {

    let p = root.join(rel);

    std::fs::create_dir_all(p.parent().unwrap()).unwrap();

    std::fs::write(p, content).unwrap();

}

struct Hub {

    app: Shared,

    base: String,

    _dir: tempfile::TempDir,

}

/// A hub; with `worker` false the background scan is off and the tests

/// run its passes themselves.

async fn hub(worker: bool) -> Hub {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let mut config = Config::load(dir.path()).unwrap();

    config.registration.setup_code = "test-setup".into();

    config.registration.mode = "open".into();

    if !worker {

        config.limits.history_scan_limit = oxygenhub::config::ByteSize(0);

    }

    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();

    let port = listener.local_addr().unwrap().port();

    config.server.base_url = format!("http://127.0.0.1:{port}");

    let app = oxygenhub::build(config).await.unwrap();

    let router = oxygenhub::router(app.clone());

    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });

    Hub { app, base: format!("http://127.0.0.1:{port}"), _dir: dir }

}

struct Res {

    status: StatusCode,

    body: String,

    cookie: Option<String>,

}

async fn send(app: &Shared, method: &str, path: &str, cookie: Option<&str>, form: Option<&str>) -> Res {

    let mut req = Request::builder().method(method).uri(path);

    if let Some(c) = cookie {

        req = req.header(header::COOKIE, format!("oxyhub={c}"));

    }

    let body = match form {

        Some(f) => {

            req = req.header(header::CONTENT_TYPE, "application/x-www-form-urlencoded");

            Body::from(f.to_string())

        }

        None => Body::empty(),

    };

    let res = oxygenhub::router(app.clone()).oneshot(req.body(body).unwrap()).await.unwrap();

    let status = res.status();

    let cookie = res

        .headers()

        .get(header::SET_COOKIE)

        .and_then(|v| v.to_str().ok())

        .and_then(|v| v.split(';').next())

        .and_then(|kv| kv.split_once('='))

        .map(|(_, v)| v.to_string());

    let bytes = res.into_body().collect().await.unwrap().to_bytes();

    Res { status, body: String::from_utf8_lossy(&bytes).into_owned(), cookie }

}

async fn get(app: &Shared, path: &str, cookie: Option<&str>) -> Res {

    send(app, "GET", path, cookie, None).await

}

async fn post(app: &Shared, path: &str, cookie: Option<&str>, form: &str) -> Res {

    send(app, "POST", path, cookie, Some(form)).await

}

async fn register(app: &Shared, user: &str) -> String {

    let r = post(

        app,

        "/register",

        None,

        &format!("username={user}&email={user}%40example.com&password=correct-horse-battery&setup=test-setup"),

    )

    .await;

    assert_eq!(r.status, StatusCode::SEE_OTHER, "{}", r.body);

    r.cookie.unwrap()

}

fn csrf_of(page: &str) -> String {

    let i = page.find("name=\"_csrf\" value=\"").expect("csrf field") + "name=\"_csrf\" value=\"".len();

    page[i..].split('"').next().unwrap().to_string()

}

async fn token(app: &Shared, user: &str) -> String {

    let me = db::accounts::account_by_name(&app.db, user).await.unwrap().unwrap();

    let (secret, hash) = oxygenhub::auth::new_token();

    db::accounts::create_token(&app.db, me.id, "laptop", &hash, "repo:read,repo:write", "", None).await.unwrap();

    secret

}

/// andy/demo with a client checkout at `work` and its first change pushed.

async fn demo(h: &Hub, work: &Path, tok: &str) -> Repo {

    let repo = RepoName::parse("andy/demo").unwrap();

    h.app.repos.create(&repo, None).unwrap();

    db::repos::create(&h.app.db, "andy", "demo", "", hub_api::Visibility::Public).await.unwrap();

    let mut client = Repo::init(work).unwrap();

    client.create_identity("Andy McCall", "[email protected]", oxy_core::repo::IdentityScope::User).unwrap();

    client.config.set("remote.origin.url", &format!("{}/andy/demo", h.base)).unwrap();

    client.config.set("remote.origin.token", tok).unwrap();

    client.save_config().unwrap();

    write(work, "README.md", b"one\ntwo\nthree\n");

    client.snapshot("t").unwrap();

    client.commit("base", &[]).unwrap();

    assert!(client.push("origin", false).unwrap().iter().all(|r| r.ok));

    client

}

async fn row(app: &Shared) -> db::repos::RepoRow {

    db::repos::get(&app.db, "andy", "demo").await.unwrap().unwrap()

}

/// Commit `content` at `file` and push it with push protection off, as a

/// repository's history got its secrets before push protection existed.

async fn push_unprotected(app: &Shared, client: &Repo, work: &Path, file: &str, content: &str) -> ChangeHash {

    let id = row(app).await.id;

    db::push_protection::set_enabled(&app.db, id, false).await.unwrap();

    write(work, file, content.as_bytes());

    client.snapshot("t").unwrap();

    let h = client.commit(&format!("add {file}"), &[]).unwrap();

    assert!(client.push("origin", false).unwrap().iter().all(|r| r.ok));

    db::push_protection::set_enabled(&app.db, id, true).await.unwrap();

    h

}

fn stripe() -> String {

    format!("{}_{}_{}", "sk", "live", random_hex(12))

}

fn github() -> String {

    format!("{}_{}", "ghp", random_hex(18))

}

/// The whole flow: found with change, file and line; allowed, resolved, an

/// allowed fingerprint removed; incremental after a push; scanned again

/// on request without duplicates.

#[tokio::test(flavor = "multi_thread")]

async fn a_secret_in_history_is_found_and_judged() {

    let h = hub(false).await;

    let app = &h.app;

    let andy = register(app, "andy").await;

    let tok = token(app, "andy").await;

    let work = tempfile::tempdir().unwrap();

    let client = demo(&h, work.path(), &tok).await;

    let key = stripe();

    let leak = push_unprotected(app, &client, work.path(), "deploy/pay.env", &format!("# payments\nSTRIPE={key}\n")).await;

    let budget = Budget::default();

    // The first pass scans the whole history and finds it, placed.

    let p = pass(app, &row(app).await, budget).await.unwrap();

    assert_eq!((p.found, p.left, p.idle), (1, 0, false), "{p:?}");

    assert!(p.changes >= 2, "{p:?}");

    let all = p.changes;

    let found = db::push_protection::history(&app.db, row(app).await.id, 50).await.unwrap();

    assert_eq!(found.len(), 1, "{found:?}");

    let f = &found[0];

    let fp = fingerprint("stripe-live-key", key.as_bytes());

    assert_eq!((f.path.as_str(), f.line, f.fingerprint.as_str()), ("deploy/pay.env", 2, fp.as_str()));

    assert_eq!((f.change_hash.as_str(), f.bookmark.as_str()), (leak.to_hex().as_str(), "main"));

    assert!(!f.excerpt.contains(&key), "{}", f.excerpt);

    // Nothing new: the next pass has nothing to do.

    let p = pass(app, &row(app).await, budget).await.unwrap();

    assert!(p.idle && p.changes == 0, "{p:?}");

    // Maintainers see it under Found in history, never the secret.

    let page = get(app, "/andy/demo/settings", Some(&andy)).await;

    assert!(page.body.contains("Found in history") && page.body.contains(&fp), "{}", page.body);

    assert!(page.body.contains(&format!("/andy/demo/change/{}", leak.to_hex())) && page.body.contains("deploy/pay.env:2"));

    assert!(!page.body.contains(&key), "the settings page shows no secret");

    // This hub runs no scan of its own: the page says so.

    assert!(page.body.contains("The history scan is off on this hub"), "{}", page.body);

    let csrf = csrf_of(&page.body);

    // A push is scanned incrementally: only its change.

    write(work.path(), "README.md", b"one\ntwo\nthree\nfour\n");

    client.snapshot("t").unwrap();

    client.commit("four", &[]).unwrap();

    assert!(client.push("origin", false).unwrap().iter().all(|r| r.ok));

    let p = pass(app, &row(app).await, budget).await.unwrap();

    assert_eq!((p.changes, p.found, p.left), (1, 0, 0), "{p:?}");

    let ghp = github();

    let second = push_unprotected(app, &client, work.path(), "ci.env", &format!("GH_TOKEN={ghp}\n")).await;

    let p = pass(app, &row(app).await, budget).await.unwrap();

    assert_eq!((p.changes, p.found), (1, 1), "{p:?}");

    let gfp = fingerprint("github-token", ghp.as_bytes());

    let found = db::push_protection::history(&app.db, row(app).await.id, 50).await.unwrap();

    assert!(

        found.iter().any(|f| f.fingerprint == gfp && f.path == "ci.env" && f.line == 1 && f.change_hash == second.to_hex()),

        "{found:?}"

    );

    // A writer can neither resolve nor remove an allowed fingerprint.

    let bob = register(app, "bob").await;

    let r = post(app, "/andy/demo/settings/collaborators", Some(&andy), &format!("_csrf={csrf}&username=bob&role=write")).await;

    assert_eq!(r.status, StatusCode::OK, "{}", r.body);

    let bob_csrf = csrf_of(&get(app, "/andy/demo", Some(&bob)).await.body);

    let stripe_id = found.iter().find(|f| f.fingerprint == fp).unwrap().id;

    let r = post(app, "/andy/demo/settings/secret-resolve", Some(&bob), &format!("_csrf={bob_csrf}&finding={stripe_id}&reason=x")).await;

    assert_eq!(r.status, StatusCode::NOT_FOUND, "write is not maintain");

    let r = post(app, "/andy/demo/settings/secret-scan", Some(&bob), &format!("_csrf={bob_csrf}")).await;

    assert_eq!(r.status, StatusCode::NOT_FOUND, "write is not maintain");

    // Resolved: the key was revoked. A reason is needed; audited.

    let r = post(app, "/andy/demo/settings/secret-resolve", Some(&andy), &format!("_csrf={csrf}&finding={stripe_id}&reason=")).await;

    assert!(r.body.contains("Say how it was resolved"), "{}", r.body);

    let r = post(

        app,

        "/andy/demo/settings/secret-resolve",

        Some(&andy),

        &format!("_csrf={csrf}&finding={stripe_id}&reason=revoked+in+the+Stripe+dashboard"),

    )

    .await;

    assert_eq!(r.status, StatusCode::OK, "{}", r.body);

    assert!(r.body.contains("marked resolved") && r.body.contains("revoked in the Stripe dashboard"), "{}", r.body);

    let open = db::push_protection::history(&app.db, row(app).await.id, 50).await.unwrap();

    assert!(open.iter().all(|f| f.id != stripe_id), "{open:?}");

    let resolved = db::push_protection::resolved(&app.db, row(app).await.id, 50).await.unwrap();

    assert!(resolved.iter().any(|f| f.id == stripe_id && f.resolved_by == "andy" && f.resolved_ms.is_some()), "{resolved:?}");

    let r = post(app, "/andy/demo/settings/secret-resolve", Some(&andy), &format!("_csrf={csrf}&finding={stripe_id}&reason=again")).await;

    assert!(r.body.contains("not open here"), "only once");

    let audit = db::accounts::audit_rows(&app.db, 50).await.unwrap();

    assert!(

        audit.iter().any(|a| a.action == "secret.resolved" && a.detail.contains(&fp) && a.detail.contains("deploy/pay.env:2")),

        "{audit:?}"

    );

    // Allowed from its history row (a test token): no longer open.

    let r =

        post(app, "/andy/demo/settings/secret-bypass", Some(&andy), &format!("_csrf={csrf}&fingerprint={gfp}&reason=a+test+fixture")).await;

    assert_eq!(r.status, StatusCode::OK, "{}", r.body);

    assert!(db::push_protection::history(&app.db, row(app).await.id, 50).await.unwrap().is_empty());

    let audit = db::accounts::audit_rows(&app.db, 50).await.unwrap();

    assert!(audit.iter().any(|a| a.action == "secret.bypass" && a.detail.contains(&gfp) && a.detail.contains("github-token")), "{audit:?}");

    // The allowed fingerprint is removed again: a writer cannot; a

    // maintainer can, audited; the finding is open again and a push adding

    // that value is refused again.

    let r = post(app, "/andy/demo/settings/secret-bypass/remove", Some(&bob), &format!("_csrf={bob_csrf}&fingerprint={gfp}")).await;

    assert_eq!(r.status, StatusCode::NOT_FOUND, "write is not maintain");

    let r = post(app, "/andy/demo/settings/secret-bypass/remove", Some(&andy), &format!("_csrf={csrf}&fingerprint={gfp}")).await;

    assert_eq!(r.status, StatusCode::OK, "{}", r.body);

    assert!(r.body.contains(&format!("Fingerprint {gfp} is no longer allowed")), "{}", r.body);

    assert!(db::push_protection::allowed(&app.db, row(app).await.id).await.unwrap().is_empty());

    let audit = db::accounts::audit_rows(&app.db, 50).await.unwrap();

    assert!(

        audit.iter().any(|a| a.action == "secret.bypass_removed" && a.detail.contains(&gfp) && a.detail.contains("a test fixture")),

        "{audit:?}"

    );

    let open = db::push_protection::history(&app.db, row(app).await.id, 50).await.unwrap();

    assert!(open.iter().any(|f| f.fingerprint == gfp), "{open:?}");

    let r = post(app, "/andy/demo/settings/secret-bypass/remove", Some(&andy), &format!("_csrf={csrf}&fingerprint={gfp}")).await;

    assert!(r.body.contains("is not allowed here"), "{}", r.body);

    write(work.path(), "ci2.env", format!("GH_TOKEN={ghp}\n").as_bytes());

    client.snapshot("t").unwrap();

    let again = client.commit("again", &[]).unwrap();

    let err = client.push("origin", false).expect_err("refused again").to_string();

    assert!(err.contains(&format!("(fingerprint {gfp})")), "{err}");

    client.abandon(&again).unwrap();

    let _ = std::fs::remove_file(work.path().join("ci2.env"));

    client.snapshot("t").unwrap();

    // Scan history now: the whole history again, audited; nothing found

    // twice, and what was resolved stays resolved.

    let r = post(app, "/andy/demo/settings/secret-scan", Some(&andy), &format!("_csrf={csrf}")).await;

    assert_eq!(r.status, StatusCode::OK, "{}", r.body);

    assert!(r.body.contains("will be scanned again"), "{}", r.body);

    assert!(db::push_protection::scan_of(&app.db, row(app).await.id).await.unwrap().unwrap().requested_ms.is_some());

    let p = pass(app, &row(app).await, budget).await.unwrap();

    assert_eq!((p.changes, p.found, p.left), (all + 2, 0, 0), "{p:?}");

    let s = db::push_protection::scan_of(&app.db, row(app).await.id).await.unwrap().unwrap();

    assert!(s.requested_ms.is_none() && s.target_heads.is_empty() && s.finished_ms.is_some(), "{s:?}");

    assert_eq!(db::push_protection::resolved(&app.db, row(app).await.id, 50).await.unwrap().len(), 1);

    assert_eq!(db::push_protection::history(&app.db, row(app).await.id, 50).await.unwrap().len(), 1);

    let audit = db::accounts::audit_rows(&app.db, 50).await.unwrap();

    assert!(audit.iter().any(|a| a.action == "secret.history_scan"), "{audit:?}");

    // Findings of refused pushes stay apart from history's.

    let recent = db::push_protection::recent(&app.db, row(app).await.id, 50).await.unwrap();

    assert!(recent.iter().all(|f| f.fingerprint == gfp && f.path == "ci2.env"), "{recent:?}");

}

/// A pass stops at its budget and the next resumes where it stopped; a push

/// in the meantime waits for the scan under way, then is scanned alone.

#[tokio::test(flavor = "multi_thread")]

async fn the_scan_resumes_where_its_budget_stopped() {

    let h = hub(false).await;

    let app = &h.app;

    register(app, "andy").await;

    let tok = token(app, "andy").await;

    let work = tempfile::tempdir().unwrap();

    let client = demo(&h, work.path(), &tok).await;

    let key = stripe();

    for i in 0..5 {

        let text = if i == 3 { format!("n = {i}\nSTRIPE={key}\n") } else { format!("n = {i}\n") };

        push_unprotected(app, &client, work.path(), &format!("f{i}.txt"), &text).await;

    }

    // A byte: one change a pass.

    let budget = Budget { bytes: 1, ..Budget::default() };

    let first = pass(app, &row(app).await, budget).await.unwrap();

    assert_eq!(first.changes, 1, "{first:?}");

    let total = first.left + 1;

    assert!(total >= 6, "{first:?}");

    let s = db::push_protection::scan_of(&app.db, row(app).await.id).await.unwrap().unwrap();

    assert_eq!((s.position, s.pending), (1, total as i64), "{s:?}");

    assert!(!s.target_heads.is_empty() && s.done_heads.is_empty(), "{s:?}");

    let second = pass(app, &row(app).await, budget).await.unwrap();

    assert_eq!((second.changes, second.left), (1, total - 2), "{second:?}");

    // A push now does not restart the scan under way.

    write(work.path(), "late.txt", b"late\n");

    client.snapshot("t").unwrap();

    client.commit("late", &[]).unwrap();

    assert!(client.push("origin", false).unwrap().iter().all(|r| r.ok));

    let mut found = first.found + second.found;

    let mut passes = 2;

    loop {

        let p = pass(app, &row(app).await, budget).await.unwrap();

        assert_eq!(p.changes, 1, "{p:?}");

        found += p.found;

        passes += 1;

        if p.left == 0 {

            break;

        }

        assert!(passes < 50, "the scan never ends");

    }

    assert_eq!((passes, found), (total, 1), "each change once, the secret once");

    // Then the push, alone.

    let p = pass(app, &row(app).await, budget).await.unwrap();

    assert_eq!((p.changes, p.left, p.found), (1, 0, 0), "{p:?}");

    assert!(pass(app, &row(app).await, budget).await.unwrap().idle);

    let found = db::push_protection::history(&app.db, row(app).await.id, 50).await.unwrap();

    assert_eq!(found.iter().map(|f| (f.path.as_str(), f.line)).collect::<Vec<_>>(), [("f3.txt", 2)]);

}

/// The worker scans every repository when it starts (as on the first

/// deploy) and a repository again after a push, by itself.

#[tokio::test(flavor = "multi_thread")]

async fn the_worker_scans_on_start_and_after_a_push() {

    let h = hub(true).await;

    let app = &h.app;

    let andy = register(app, "andy").await;

    let tok = token(app, "andy").await;

    let work = tempfile::tempdir().unwrap();

    let client = demo(&h, work.path(), &tok).await;

    let key = stripe();

    push_unprotected(app, &client, work.path(), "old.env", &format!("STRIPE={key}\n")).await;

    let id = row(app).await.id;

    let until = |n: usize| async move {

        for _ in 0..300 {

            if db::push_protection::history(&app.db, id, 50).await.unwrap().len() >= n {