junit.rs 96.1%

396 of 412 lines run, 71 of 122 functions · run #464 on main · run missed
//! Test reports from OxygenFlow jobs (oxygenhub#114): a job's artifact
//! named `junit`, in JUnit XML as pytest `--junitxml`, cargo-nextest, Jest,
//! gotestsum and most runners write it, is parsed when it is uploaded and
//! kept per run: the counts, and every test case with its status and time.
//! A failure's message and output are kept, cut to a size a page can show.

use crate::db::{now_ms, Pool};
use crate::Shared;
use quick_xml::events::{BytesStart, Event};
use quick_xml::{Reader, XmlVersion};

/// The artifact name a job gives its JUnit report.
pub const ARTIFACT: &str = "junit";
/// Largest report read; a bigger one is noted in the job's log and skipped.
pub const MAX_REPORT: usize = 64 << 20;
/// Test cases kept from one report.
pub const MAX_CASES: usize = 50_000;
/// A failure's message, and its output, are cut to these many bytes.
const MAX_MESSAGE: usize = 2 << 10;
const MAX_OUTPUT: usize = 16 << 10;

/// One test case as the report gives it.
#[derive(Clone, Debug, PartialEq, Eq, sqlx::FromRow)]
pub struct Case {
    /// The enclosing `<testsuite>`'s name, `/`-joined when suites nest.
    pub suite: String,
    pub classname: String,
    pub name: String,
    /// `passed`, `failed`, `error` or `skipped`.
    pub status: String,
    pub time_ms: i64,
    /// The failure's, error's or skip's `message`, else its first line.
    pub message: String,
    /// The failure's or error's text, then the case's stdout and stderr;
    /// empty for a passing case.
    pub output: String,
}

impl Case {
    /// `classname::name`, or the name alone.
    pub fn title(&self) -> String {
        if self.classname.is_empty() {
            self.name.clone()
        } else {
            format!("{}::{}", self.classname, self.name)
        }
    }
}

/// Cut `s` to at most `max` bytes on a character boundary, saying so.
fn cut(mut s: String, max: usize) -> String {
    if s.len() > max {
        let mut i = max;
        while !s.is_char_boundary(i) {
            i -= 1;
        }
        s.truncate(i);
        s.push_str("\n… (cut)");
    }
    s
}

fn attr(e: &BytesStart, name: &str) -> String {
    e.try_get_attribute(name)
        .ok()
        .flatten()
        .and_then(|a| a.normalized_value(XmlVersion::Implicit1_0).ok().map(|v| v.into_owned()))
        .unwrap_or_default()
}

/// Seconds as JUnit writes them (`1.25`, sometimes `1,250.5`) to ms.
fn secs_ms(s: &str) -> i64 {
    s.replace(',', "").trim().parse::<f64>().map(|v| (v * 1000.0).round().max(0.0) as i64).unwrap_or(0)
}

/// What the parser is collecting text for.
#[derive(PartialEq)]
enum Collect {
    Nothing,
    Detail,
    Out,
}

/// Parse a JUnit XML report: every `<testcase>`, in order, wherever its
/// `<testsuite>` nests. Counts come from the cases, not the attributes.
pub fn parse(text: &str) -> Result<Vec<Case>, String> {
    let mut r = Reader::from_str(text);
    r.config_mut().trim_text(false);
    let mut suites: Vec<String> = Vec::new();
    let mut cases = Vec::new();
    let mut case: Option<Case> = None;
    let mut detail = String::new();
    let mut out = String::new();
    let mut into = Collect::Nothing;
    let mut seen_root = false;
    loop {
        let ev = r.read_event().map_err(|e| format!("not JUnit XML: {e} (at byte {})", r.error_position()))?;
        match ev {
            Event::Start(ref e) | Event::Empty(ref e)
                if e.local_name().as_ref() == "testsuites" || e.local_name().as_ref() == "testsuite" =>
            {
                seen_root = true;
                if e.local_name().as_ref() == "testsuite" {
                    suites.push(attr(e, "name"));
                }
                // An empty `<testsuite/>` has no end event.
                if matches!(ev, Event::Empty(_)) && e.local_name().as_ref() == "testsuite" {
                    suites.pop();
                }
            }
            Event::Start(ref e) | Event::Empty(ref e) if e.local_name().as_ref() == "testcase" => {
                seen_root = true;
                let c = Case {
                    suite: suites.iter().filter(|s| !s.is_empty()).cloned().collect::<Vec<_>>().join("/"),
                    classname: attr(e, "classname"),
                    name: attr(e, "name"),
                    status: "passed".into(),
                    time_ms: secs_ms(&attr(e, "time")),
                    message: String::new(),
                    output: String::new(),
                };
                if matches!(ev, Event::Empty(_)) {
                    cases.push(c);
                } else {
                    case = Some(c);
                    detail.clear();
                    out.clear();
                }
            }
            Event::Start(ref e) | Event::Empty(ref e) if case.is_some() => {
                let tag = e.local_name().as_ref().to_string();
                let c = case.as_mut().unwrap();
                let status = match tag.as_str() {
                    "failure" | "flakyFailure" if c.status == "passed" => Some("failed"),
                    "error" | "flakyError" if c.status == "passed" => Some("error"),
                    "skipped" if c.status == "passed" => Some("skipped"),
                    _ => None,
                };
                // nextest's flaky retries passed in the end: the case passed.
                let flaky = tag.starts_with("flaky") || tag.starts_with("rerun");
                if let Some(s) = status.filter(|_| !flaky) {
                    c.status = s.into();
                    c.message = attr(e, "message");
                    if matches!(ev, Event::Start(_)) {
                        into = Collect::Detail;
                    }
                } else if matches!(ev, Event::Start(_)) && (tag == "system-out" || tag == "system-err") {
                    into = Collect::Out;
                }
            }
            Event::Text(t) => push_text(&mut into, &mut detail, &mut out, &t.xml10_content()),
            Event::CData(t) => push_text(&mut into, &mut detail, &mut out, &t.xml10_content()),
            Event::GeneralRef(g) => {
                let s = match g.resolve_char_ref() {
                    Ok(Some(ch)) => ch.to_string(),
                    _ => {
                        let name = g.xml10_content();
                        quick_xml::escape::resolve_predefined_entity(&name).map(str::to_string).unwrap_or_else(|| format!("&{name};"))
                    }
                };
                push_text(&mut into, &mut detail, &mut out, &s);
            }
            Event::End(e) => match e.local_name().as_ref() {
                "testsuite" => {
                    suites.pop();
                }
                "testcase" => {
                    if let Some(mut c) = case.take() {
                        if c.status != "passed" {
                            if c.message.is_empty() {
                                c.message = detail.lines().map(str::trim).find(|l| !l.is_empty()).unwrap_or("").to_string();
                            }
                            let mut o = detail.trim().to_string();
                            if !out.trim().is_empty() && c.status != "skipped" {
                                if !o.is_empty() {
                                    o.push_str("\n\n");
                                }
                                o.push_str(out.trim());
                            }
                            c.output = cut(o, MAX_OUTPUT);
                        }
                        c.message = cut(c.message, MAX_MESSAGE);
                        cases.push(c);
                        if cases.len() > MAX_CASES {
                            return Err(format!("more than {MAX_CASES} test cases"));
                        }
                    }
                    into = Collect::Nothing;
                }
                "failure" | "error" | "skipped" | "system-out" | "system-err" | "flakyFailure" | "flakyError" | "rerunFailure"
                | "rerunError" => into = Collect::Nothing,
                _ => {}
            },
            Event::Eof => break,
            _ => {}
        }
    }
    if !seen_root {
        return Err("not JUnit XML: no <testsuites>, <testsuite> or <testcase>".into());
    }
    Ok(cases)
}

fn push_text(into: &mut Collect, detail: &mut String, out: &mut String, s: &str) {
    let (buf, max) = match into {
        Collect::Detail => (detail, MAX_OUTPUT),
        Collect::Out => (out, MAX_OUTPUT),
        Collect::Nothing => return,
    };
    // Keep a little past the cut so `cut` can say it cut.
    if buf.len() <= max {
        buf.push_str(s);
    }
}

/// A run's kept report.
#[derive(Clone, Debug, sqlx::FromRow)]
pub struct Report {
    pub id: i64,
    pub repo_id: i64,
    pub run_id: i64,
    pub job_id: i64,
    pub job: String,
    pub run_number: i64,
    pub bookmark: String,
    pub state: String,
    pub blob: String,
    pub tests: i64,
    pub passed: i64,
    pub failed: i64,
    pub errors: i64,
    pub skipped: i64,
    pub time_ms: i64,
    pub created_ms: i64,
}

impl Report {
    /// "630 tests: 2 failed, 3 skipped", or "630 passed".
    pub fn summary(&self) -> String {
        let mut bad = vec![];
        if self.failed > 0 {
            bad.push(format!("{} failed", self.failed));
        }
        if self.errors > 0 {
            bad.push(format!("{} errored", self.errors));
        }
        if self.skipped > 0 {
            bad.push(format!("{} skipped", self.skipped));
        }
        let tests = if self.tests == 1 { "1 test".to_string() } else { format!("{} tests", self.tests) };
        if bad.is_empty() {
            format!("{tests}, all passed")
        } else {
            format!("{tests}: {}", bad.join(", "))
        }
    }

    pub fn broken(&self) -> i64 {
        self.failed + self.errors
    }
}

/// What a run's report is kept with.
pub struct Origin<'a> {
    pub repo_id: i64,
    pub run_id: i64,
    pub run_number: i64,
    pub job_id: i64,
    pub job: &'a str,
    pub bookmark: &'a str,
    pub state: &'a str,
    pub blob: &'a str,
}

/// Keep a parsed report, replacing an earlier one of the same job.
pub async fn keep(app: &Shared, o: &Origin<'_>, cases: &[Case]) -> sqlx::Result<i64> {
    let count = |s: &str| cases.iter().filter(|c| c.status == s).count() as i64;
    let mut tx = crate::db::begin_write(&app.db).await?;
    sqlx::query("DELETE FROM test_reports WHERE job_id = $1").bind(o.job_id).execute(&mut *tx).await?;
    let id: i64 = sqlx::query_scalar(
        "INSERT INTO test_reports (repo_id, run_id, job_id, job, run_number, bookmark, state, blob, \
         tests, passed, failed, errors, skipped, time_ms, created_ms) \
         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id",
    )
    .bind(o.repo_id)
    .bind(o.run_id)
    .bind(o.job_id)
    .bind(o.job)
    .bind(o.run_number)
    .bind(o.bookmark)
    .bind(o.state)
    .bind(o.blob)
    .bind(cases.len() as i64)
    .bind(count("passed"))
    .bind(count("failed"))
    .bind(count("error"))
    .bind(count("skipped"))
    .bind(cases.iter().map(|c| c.time_ms).sum::<i64>())
    .bind(now_ms())
    .fetch_one(&mut *tx)
    .await?;
    for (i, c) in cases.iter().enumerate() {
        sqlx::query(
            "INSERT INTO test_cases (report_id, idx, suite, classname, name, status, time_ms, message, output) \
             VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
        )
        .bind(id)
        .bind(i as i64)
        .bind(&c.suite)
        .bind(&c.classname)
        .bind(&c.name)
        .bind(&c.status)
        .bind(c.time_ms)
        .bind(&c.message)
        .bind(&c.output)
        .execute(&mut *tx)
        .await?;
    }
    tx.commit().await?;
    mark(app, o.repo_id);
    Ok(id)
}

/// A run's reports, one per job that kept one.
pub async fn for_run(pool: &Pool, run_id: i64) -> sqlx::Result<Vec<Report>> {
    sqlx::query_as("SELECT * FROM test_reports WHERE run_id = $1 ORDER BY job").bind(run_id).fetch_all(pool).await
}

/// A report's cases in report order, all or only those with a status
/// among `statuses`.
pub async fn cases(pool: &Pool, report_id: i64, statuses: &[&str], limit: i64) -> sqlx::Result<Vec<Case>> {
    let filter = if statuses.is_empty() {
        String::new()
    } else {
        // $1 is the report; the statuses follow, then the limit.
        format!(" AND status IN ({})", (2..2 + statuses.len()).map(|i| format!("${i}")).collect::<Vec<_>>().join(", "))
    };
    let limit_at = 2 + statuses.len();
    let sql = format!(
        "SELECT suite, classname, name, status, time_ms, message, output FROM test_cases WHERE report_id = $1{filter} ORDER BY idx LIMIT ${limit_at}"
    );
    let mut q = sqlx::query_as(&sql).bind(report_id);
    for s in statuses {
        q = q.bind(*s);
    }
    q.bind(limit).fetch_all(pool).await
}

/// A report's failed and errored cases, in report order.
pub async fn failures(pool: &Pool, report_id: i64, limit: i64) -> sqlx::Result<Vec<Case>> {
    sqlx::query_as(
        "SELECT suite, classname, name, status, time_ms, message, output FROM test_cases \
         WHERE report_id = $1 AND status IN ('failed', 'error') ORDER BY idx LIMIT $2",
    )
    .bind(report_id)
    .bind(limit)
    .fetch_all(pool)
    .await
}

/// Which repositories have test reports, for the Tests tab; keyed by data
/// directory as well, for tests running several hubs in one process.
static HAS: std::sync::Mutex<Option<std::collections::HashSet<String>>> = std::sync::Mutex::new(None);

fn key(app: &Shared, repo_id: i64) -> String {
    format!("{}#{repo_id}", app.repos.root().display())
}

pub fn has_reports(app: &Shared, repo_id: i64) -> bool {
    HAS.lock().unwrap_or_else(|p| p.into_inner()).as_ref().is_some_and(|s| s.contains(&key(app, repo_id)))
}

fn mark(app: &Shared, repo_id: i64) {
    HAS.lock().unwrap_or_else(|p| p.into_inner()).get_or_insert_with(Default::default).insert(key(app, repo_id));
}

/// At start: which repositories have reports.
pub async fn load(app: &Shared) -> sqlx::Result<()> {
    let ids: Vec<(i64,)> = sqlx::query_as("SELECT DISTINCT repo_id FROM test_reports").fetch_all(&app.db).await?;
    for (id,) in ids {
        mark(app, id);
    }
    Ok(())
}

pub async fn report(pool: &Pool, repo_id: i64, id: i64) -> sqlx::Result<Option<Report>> {
    sqlx::query_as("SELECT * FROM test_reports WHERE id = $1 AND repo_id = $2").bind(id).bind(repo_id).fetch_optional(pool).await
}

/// The newest reports on a bookmark, newest first.
pub async fn on_bookmark(pool: &Pool, repo_id: i64, bookmark: &str, limit: i64) -> sqlx::Result<Vec<Report>> {
    sqlx::query_as("SELECT * FROM test_reports WHERE repo_id = $1 AND bookmark = $2 ORDER BY created_ms DESC, id DESC LIMIT $3")
        .bind(repo_id)
        .bind(bookmark)
        .bind(limit)
        .fetch_all(pool)
        .await
}

/// Bookmarks with reports, the most recently reported first.
pub async fn bookmarks(pool: &Pool, repo_id: i64) -> sqlx::Result<Vec<String>> {
    let rows: Vec<(String,)> =
        sqlx::query_as("SELECT bookmark FROM test_reports WHERE repo_id = $1 GROUP BY bookmark ORDER BY MAX(created_ms) DESC")
            .bind(repo_id)
            .fetch_all(pool)
            .await?;
    Ok(rows.into_iter().map(|(b,)| b).collect())
}

pub async fn first_ms(pool: &Pool, repo_id: i64) -> sqlx::Result<Option<i64>> {
    sqlx::query_scalar("SELECT MIN(created_ms) FROM test_reports WHERE repo_id = $1").bind(repo_id).fetch_one(pool).await
}

/// A report's slowest cases.
pub async fn slowest(pool: &Pool, report_id: i64, limit: i64) -> sqlx::Result<Vec<Case>> {
    sqlx::query_as(
        "SELECT suite, classname, name, status, time_ms, message, '' AS output FROM test_cases \
         WHERE report_id = $1 AND status != 'skipped' ORDER BY time_ms DESC, idx LIMIT $2",
    )
    .bind(report_id)
    .bind(limit)
    .fetch_all(pool)
    .await
}

/// Every case of a report as `(classname, name) -> status`.
pub async fn statuses(pool: &Pool, report_id: i64) -> sqlx::Result<std::collections::HashMap<(String, String), String>> {
    let rows: Vec<(String, String, String)> =
        sqlx::query_as("SELECT classname, name, status FROM test_cases WHERE report_id = $1").bind(report_id).fetch_all(pool).await?;
    Ok(rows.into_iter().map(|(c, n, s)| ((c, n), s)).collect())
}

/// One test across a bookmark's reports, newest first.
#[derive(Clone, Debug, sqlx::FromRow)]
pub struct Run {
    pub report_id: i64,
    pub run_number: i64,
    pub job: String,
    pub created_ms: i64,
    pub status: String,
    pub time_ms: i64,
    pub message: String,
}

pub async fn history(pool: &Pool, repo_id: i64, bookmark: &str, classname: &str, name: &str, limit: i64) -> sqlx::Result<Vec<Run>> {
    sqlx::query_as(
        "SELECT r.id AS report_id, r.run_number, r.job, r.created_ms, c.status, c.time_ms, c.message \
         FROM test_cases c JOIN test_reports r ON r.id = c.report_id \
         WHERE r.repo_id = $1 AND r.bookmark = $2 AND c.classname = $3 AND c.name = $4 \
         ORDER BY r.created_ms DESC, r.id DESC LIMIT $5",
    )
    .bind(repo_id)
    .bind(bookmark)
    .bind(classname)
    .bind(name)
    .bind(limit)
    .fetch_all(pool)
    .await
}

/// What a report is measured against: the same job's previous report on
/// the default bookmark for a run on it, its latest for any other.
pub async fn baseline(pool: &Pool, r: &Report, default_bookmark: &str) -> sqlx::Result<Option<Report>> {
    sqlx::query_as(
        "SELECT * FROM test_reports WHERE repo_id = $1 AND bookmark = $2 AND job = $3 AND id != $4 \
         AND (created_ms < $5 OR (created_ms = $6 AND id < $7)) ORDER BY created_ms DESC, id DESC LIMIT 1",
    )
    .bind(r.repo_id)
    .bind(default_bookmark)
    .bind(&r.job)
    .bind(r.id)
    .bind(if r.bookmark == default_bookmark { r.created_ms } else { i64::MAX })
    .bind(r.created_ms)
    .bind(if r.bookmark == default_bookmark { r.id } else { i64::MAX })
    .fetch_optional(pool)
    .await
}

/// Tests failing in `report` that passed in `base`, by title.
pub async fn newly_failing(pool: &Pool, report: i64, base: i64) -> sqlx::Result<Vec<String>> {
    let rows: Vec<(String, String)> = sqlx::query_as(
        "SELECT c.classname, c.name FROM test_cases c JOIN test_cases b \
         ON b.report_id = $1 AND b.classname = c.classname AND b.name = c.name AND b.status = 'passed' \
         WHERE c.report_id = $2 AND c.status IN ('failed', 'error') ORDER BY c.idx",
    )
    .bind(base)
    .bind(report)
    .fetch_all(pool)
    .await?;
    Ok(rows.into_iter().map(|(c, n)| if c.is_empty() { n } else { format!("{c}::{n}") }).collect())
}

/// A test that both failed and passed across some reports.
#[derive(Clone, Debug, sqlx::FromRow)]
pub struct Flaky {
    pub classname: String,
    pub name: String,
    pub failed: i64,
    pub passed: i64,
}

impl Flaky {
    pub fn title(&self) -> String {
        if self.classname.is_empty() {
            self.name.clone()
        } else {
            format!("{}::{}", self.classname, self.name)
        }
    }
}

/// Tests that failed in some of `reports` and passed in others, the most
/// often failing first: flaky, or broken and fixed again.
pub async fn flaky(pool: &Pool, reports: &[i64], limit: i64) -> sqlx::Result<Vec<Flaky>> {
    if reports.is_empty() {
        return Ok(vec![]);
    }
    let ids = reports.iter().map(|i| i.to_string()).collect::<Vec<_>>().join(",");
    sqlx::query_as(&format!(
        "SELECT classname, name, failed, passed FROM (\
           SELECT classname, name, SUM(CASE WHEN status IN ('failed', 'error') THEN 1 ELSE 0 END) AS failed, \
             SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) AS passed \
           FROM test_cases WHERE report_id IN ({ids}) GROUP BY classname, name) t \
         WHERE failed > 0 AND passed > 0 ORDER BY failed DESC, classname, name LIMIT $1"
    ))
    .bind(limit)
    .fetch_all(pool)
    .await
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn pytest_nextest_and_jest_shapes() {
        // pytest: one suite, a failure with a message and text, a skip, an error.
        let py = r#"<?xml version="1.0" encoding="utf-8"?>
<testsuites><testsuite name="pytest" errors="1" failures="1" skipped="1" tests="4" time="3.5">
<testcase classname="tests.test_a" name="test_ok" time="0.010"/>
<testcase classname="tests.test_a" name="test_bad" time="1.25"><failure message="assert 1 == 2">def test_bad():
&gt;       assert 1 == 2
E       assert 1 == 2</failure><system-out>printed &amp; kept</system-out></testcase>
<testcase classname="tests.test_b" name="test_skip" time="0"><skipped type="pytest.skip" message="no emulator"/></testcase>
<testcase classname="tests.test_b" name="test_err" time="0.5"><error message="fixture failed"><![CDATA[Traceback <here>]]></error></testcase>
</testsuite></testsuites>"#;
        let c = parse(py).unwrap();
        let st: Vec<(&str, &str)> = c.iter().map(|c| (c.name.as_str(), c.status.as_str())).collect();
        assert_eq!(st, [("test_ok", "passed"), ("test_bad", "failed"), ("test_skip", "skipped"), ("test_err", "error")]);
        assert_eq!(c[1].message, "assert 1 == 2");
        assert!(c[1].output.contains(">       assert 1 == 2") && c[1].output.contains("printed & kept"), "{}", c[1].output);
        assert_eq!((c[1].time_ms, c[0].time_ms), (1250, 10));
        assert_eq!(c[2].message, "no emulator");
        assert_eq!(c[3].output, "Traceback <here>");
        assert_eq!(c[1].suite, "pytest");
        assert_eq!(c[1].title(), "tests.test_a::test_bad");
        assert!(c[0].output.is_empty());

        // nextest: a flaky test that passed on retry is a pass.
        let nx = r#"<testsuites name="nextest-run"><testsuite name="oxy-core" tests="2">
<testcase name="repo::tests::a" classname="oxy-core::lib" time="0.2"><flakyFailure message="first try" type="test failure">boom</flakyFailure></testcase>
<testcase name="repo::tests::b" classname="oxy-core::lib" time="0.1"/></testsuite></testsuites>"#;
        let c = parse(nx).unwrap();
        assert_eq!(c.iter().filter(|c| c.status == "passed").count(), 2, "{c:?}");

        // Jest: nested suites, no wrapper on a single suite is fine too.
        let js = r#"<testsuite name="ui"><testsuite name="Button"><testcase classname="Button renders" name="renders" time="0.003"/></testsuite></testsuite>"#;
        let c = parse(js).unwrap();
        assert_eq!(c[0].suite, "ui/Button");

        assert!(parse("<html/>").is_err());
        assert!(parse("not xml <<<").is_err());
        assert!(parse("<testsuites/>").unwrap().is_empty());
    }

    #[test]
    fn long_output_is_cut() {
        let big = "x".repeat(MAX_OUTPUT * 3);
        let xml = format!(r#"<testsuite><testcase name="t"><failure>{big}</failure></testcase></testsuite>"#);
        let c = parse(&xml).unwrap();
        assert!(c[0].output.len() < MAX_OUTPUT + 32 && c[0].output.ends_with("(cut)"));
    }
}
`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet
Coverage / crates/hub/src/junit.rs 96.1%

Keyboard
