Coverage from a workstation: oh coverage upload, shown as uploaded 40dd3f86379a

-- Coverage uploaded from a workstation (oxygenhub#136): a report with no

-- Flow run or job behind it, made by `oh coverage upload`, naming who

-- uploaded it, so a runner's figure and an upload's cannot be mistaken

-- for each other. SQLite cannot drop NOT NULL, so both tables are rebuilt

-- (migrations run with foreign keys off).

CREATE TABLE coverage_reports_new (

    id               INTEGER PRIMARY KEY,

    repo_id          INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE,

    run_id           INTEGER REFERENCES flow_runs(id) ON DELETE CASCADE,

    job_id           INTEGER UNIQUE,

    job              TEXT NOT NULL,

    run_number       INTEGER NOT NULL,

    bookmark         TEXT NOT NULL,

    state            TEXT NOT NULL,

    heads            TEXT NOT NULL,

    blob             TEXT NOT NULL,

    format           TEXT NOT NULL,

    lines_found      INTEGER NOT NULL,

    lines_hit        INTEGER NOT NULL,

    functions_found  INTEGER NOT NULL,

    functions_hit    INTEGER NOT NULL,

    branches_found   INTEGER NOT NULL,

    branches_hit     INTEGER NOT NULL,

    hits             INTEGER NOT NULL DEFAULT 0,

    files            INTEGER NOT NULL,

    dropped          INTEGER NOT NULL,

    created_ms       INTEGER NOT NULL,

    -- Who uploaded it; empty for a Flow job's report.

    uploader         TEXT NOT NULL DEFAULT ''

);

INSERT INTO coverage_reports_new (id, repo_id, run_id, job_id, job, run_number, bookmark, state, heads, blob, format,

    lines_found, lines_hit, functions_found, functions_hit, branches_found, branches_hit, hits, files, dropped, created_ms)

SELECT id, repo_id, run_id, job_id, job, run_number, bookmark, state, heads, blob, format,

    lines_found, lines_hit, functions_found, functions_hit, branches_found, branches_hit, hits, files, dropped, created_ms

FROM coverage_reports;

CREATE TABLE coverage_files_new (

    report_id        INTEGER NOT NULL REFERENCES coverage_reports_new(id) ON DELETE CASCADE,

    path             TEXT NOT NULL,

    lines_found      INTEGER NOT NULL,

    lines_hit        INTEGER NOT NULL,

    functions_found  INTEGER NOT NULL,

    functions_hit    INTEGER NOT NULL,

    branches_found   INTEGER NOT NULL,

    branches_hit     INTEGER NOT NULL,

    PRIMARY KEY (report_id, path)

);

INSERT INTO coverage_files_new SELECT * FROM coverage_files;

DROP TABLE coverage_files;

DROP TABLE coverage_reports;

ALTER TABLE coverage_reports_new RENAME TO coverage_reports;

ALTER TABLE coverage_files_new RENAME TO coverage_files;

CREATE INDEX coverage_reports_bookmark ON coverage_reports (repo_id, bookmark, created_ms);

- **Coverage from a workstation (#136).** `oh coverage upload FILE [--bookmark B | --change HASH]` keeps an LCOV or Cobertura report measured off the hub as the coverage of that state, for suites a runner cannot run in full. The Coverage page, its history and the Reports card say "uploaded by NAME" where a run number would be, so a runner's figure and an upload's are never mistaken for each other, and an upload sets no check.

    /// Who uploaded it from a workstation (#136), when no Flow job made it;

    /// `run` and `job` are then 0 and "upload".

    #[serde(default, skip_serializing_if = "Option::is_none")]

    pub uploaded_by: Option<String>,

        .route(

            "/repos/{owner}/{name}/coverage",

            get(reports::coverage_list)

                .post(reports::coverage_upload)

                .layer(axum::extract::DefaultBodyLimit::max(crate::coverage::MAX_REPORT + (1 << 20))),

        )

        uploaded_by: r.uploaded().then(|| r.uploader.clone()),

#[derive(Deserialize)]

pub struct Upload {

    /// The bookmark whose current state it measured (default: the default

    /// bookmark), or `change`: the state whose only head is that change.

    #[serde(default)]

    bookmark: Option<String>,

    #[serde(default)]

    change: Option<String>,

    /// The report: LCOV or Cobertura XML.

    content: String,

}

/// `POST /repos/{o}/{n}/coverage` (repo:write, #136): coverage measured off

/// the hub, as a workstation's, kept for a state with the uploader's name.

/// It is read against that state's files as a job's artifact is, and sets

/// no check: no runner measured it.

pub async fn coverage_upload(

    State(app): State<Shared>,

    headers: HeaderMap,

    Path((owner, name)): Path<(String, String)>,

    Json(u): Json<Upload>,

) -> Result<(axum::http::StatusCode, Json<CoverageReport>), ApiError> {

    use axum::http::StatusCode;

    use oxy_core::hash::ChangeHash;

    let c = caller(&app, &headers).await?;

    let me = c.require("repo:write")?.clone();

    let (repo, row, _) = reach(&app, &c, &owner, &name, Access::Write).await?;

    if u.content.len() > coverage::MAX_REPORT {

        return Err(ApiError(StatusCode::PAYLOAD_TOO_LARGE, format!("at most {} MiB of coverage is read", coverage::MAX_REPORT >> 20)));

    }

    let bad = |m: String| ApiError(StatusCode::BAD_REQUEST, m);

    // The state it measured: a change's, or a bookmark's now.

    let (bookmark, heads) = match u.change.as_deref().map(str::trim).filter(|c| !c.is_empty()) {

        Some(ch) => {

            let h = ChangeHash::from_hex(ch.trim_start_matches('#')).ok_or_else(|| bad("give the full hash of a change".into()))?;

            (String::new(), vec![h])

        }

        None => {

            let b = u.bookmark.clone().filter(|b| !b.is_empty()).unwrap_or_else(|| row.default_bookmark.clone());

            let bb = b.clone();

            let heads = app

                .repos

                .read(&repo, move |r, _| match r.view()?.bookmarks.get(&bb) {

                    Some(st) => Ok(Some(r.store().get_state(st)?.heads)),

                    None => Ok(None),

                })

                .await

                .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?

                .ok_or_else(|| bad(format!("no bookmark {b}")))?;

            (b, heads)

        }

    };

    let state =

        oxy_core::hash::hash_object(oxy_core::hash::ObjectType::State, &oxy_core::objects::StateObject::new(heads.clone()).encode());

    let text = u.content;

    let (files, outside, format) = tokio::task::spawn_blocking({

        let t = text.clone();

        move || coverage::parse(&t)

    })

    .await

    .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?

    .map_err(|e| bad(format!("coverage: {e}")))?;

    let h = heads.clone();

    let (blob, in_state) = app

        .repos

        .write(&repo, move |r, cache| {

            let files: std::collections::HashSet<String> = cache.load(r, &h)?.snapshot.files.keys().cloned().collect();

            Ok((r.store_blob(text.as_bytes())?, files))

        })

        .await

        .map_err(|e| bad(format!("that state's files could not be read: {e}")))?;

    let hex: Vec<String> = heads.iter().map(|h| h.to_hex()).collect();

    let origin = coverage::Origin {

        repo_id: row.id,

        run_id: None,

        run_number: 0,

        job_id: None,

        uploader: &me.username,

        job: "upload",

        bookmark: &bookmark,

        state: &state.to_hex(),

        heads: &hex,

        blob: &blob.to_hex(),

        format,

    };

    let id = coverage::keep(&app, &origin, files, outside, &in_state).await?;

    let r = coverage::report(&app.db, row.id, id).await?.ok_or_else(|| not_found("the report was not kept"))?;

    crate::db::accounts::audit(&app.db, Some(&me), "coverage.upload", &format!("{owner}/{name}"), &r.source()).await.ok();

    Ok((StatusCode::CREATED, Json(coverage_of(&app, &format!("/{owner}/{name}"), &r))))

}

    /// The Flow run and job it came from; none for an upload (#136).

    pub run_id: Option<i64>,

    pub job_id: Option<i64>,

    /// Who uploaded it from a workstation (#136); empty for a job's report.

    pub uploader: String,

    /// Whether it was uploaded rather than measured by a Flow job.

    pub fn uploaded(&self) -> bool {

        !self.uploader.is_empty()

    }

    /// Where its figure came from: "run #61", or "uploaded by NAME".

    pub fn source(&self) -> String {

        if self.uploaded() {

            format!("uploaded by {}", self.uploader)

        } else {

            format!("run #{}", self.run_number)

        }

    }

    /// The Flow run and job; none for an upload, which names its uploader.

    pub run_id: Option<i64>,

    pub job_id: Option<i64>,

    pub uploader: &'a str,

    if let Some(job) = o.job_id {

        sqlx::query("DELETE FROM coverage_reports WHERE job_id = ?").bind(job).execute(&mut *tx).await?;

    }

         lines_found, lines_hit, functions_found, functions_hit, branches_found, branches_hit, hits, files, dropped, created_ms, uploader) \

         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id",

    .bind(o.uploader)

            run_id: Some(1),

            job_id: Some(1),

            uploader: String::new(),

        run_id: Some(run.id),

        job_id: Some(job.id),

        uploader: "",

            "{} · {} · {} of {} lines",

            r.source(),

        .replacen("run #", "Run #", 1)

            .find(|r| r.job_id == Some(j.id))

            sub: format!("{} of {} lines covered · {}", c.lines_hit, c.lines_found, c.source()),

/// oxygenhub#136: coverage measured on a workstation is uploaded for a

/// bookmark's state, kept as uploaded by its account, shown so, and sets

/// no check.

#[tokio::test(flavor = "multi_thread")]

async fn coverage_uploaded_from_a_workstation_says_so() {

    let h = hub().await;

    db::accounts::create_account(&h.app.db, "andy", "[email protected]", "x", "salt", true).await.unwrap();

    create(&h.app, "andy", "vk").await;

    let url = format!("{}/andy/vk", h.base);

    let work = tempfile::tempdir().unwrap();

    let c = client(&url, work.path());

    write(work.path(), "src/game.c", "int a;\nint b;\nint c;\nint d;\n");

    c.commit("Game", &[]).unwrap();

    assert!(c.push("origin", false).unwrap().iter().all(|r| r.ok));

    let lcov = "SF:src/game.c\nDA:1,3\nDA:2,1\nDA:3,0\nDA:4,5\nend_of_record\nSF:reference/rom.asm\nDA:1,1\nend_of_record\n";

    let body = serde_json::json!({ "bookmark": "main", "content": lcov }).to_string();

    let (st, out) = call(&h.app, "POST", "/api/v1/repos/andy/vk/coverage", Some("site"), Some(body)).await;

    assert_eq!(st, StatusCode::CREATED, "{out}");

    let v: serde_json::Value = serde_json::from_str(&out).unwrap();

    assert_eq!((v["uploaded_by"].as_str(), v["lines_hit"].as_i64(), v["lines_found"].as_i64()), (Some("andy"), Some(3), Some(4)));

    assert_eq!(v["dropped"], 1, "a file outside the state is left out");

    assert_eq!(v["bookmark"], "main");

    let (_, page) = call(&h.app, "GET", "/andy/vk/coverage", None, None).await;

    assert!(page.contains("uploaded by andy") && page.contains("75.0%"), "the Coverage page says where it came from");

    let (_, page) = call(&h.app, "GET", "/andy/vk/reports", None, None).await;

    assert!(page.contains("uploaded by andy"), "and the Reports card");

    let row = db::repos::get(&h.app.db, "andy", "vk").await.unwrap().unwrap();

    let state = v["state"].as_str().unwrap();

    let checks = db::hooks::checks_for_state(&h.app.db, row.id, state).await.unwrap();

    assert!(checks.iter().all(|c| c.name != "coverage"), "an upload sets no check: {checks:?}");

    // Readers cannot upload; bad reports and bookmarks are refused.

    let (st, _) = call(&h.app, "POST", "/api/v1/repos/andy/vk/coverage", None, Some(r#"{"content":"x"}"#.into())).await;

    assert_ne!(st, StatusCode::CREATED);

    let (st, _) = call(

        &h.app,

        "POST",

        "/api/v1/repos/andy/vk/coverage",

        Some("site"),

        Some(r#"{"bookmark":"nope","content":"SF:a\nend_of_record\n"}"#.into()),

    )

    .await;

    assert_eq!(st, StatusCode::BAD_REQUEST);

}

    /// Coverage: upload a report measured off the hub (oxygenhub#136)

    Coverage {

        #[command(subcommand)]

        cmd: CoverageCmd,

    },

}

#[derive(Subcommand)]

enum CoverageCmd {

    /// Upload an LCOV or Cobertura report measured on this machine, as the

    /// coverage of a bookmark's current state (default: the default

    /// bookmark) or of one change's. The hub shows it as uploaded by you,

    /// not measured by a runner, and sets no check from it

    Upload {

        file: std::path::PathBuf,

        #[arg(long)]

        bookmark: Option<String>,

        /// A change's full hash

        #[arg(long, conflicts_with = "bookmark")]

        change: Option<String>,

    },

        Cmd::Coverage { cmd: CoverageCmd::Upload { file, bookmark, change } } => {

            let content = std::fs::read_to_string(&file).with_context(|| format!("read {}", file.display()))?;

            let mut body = serde_json::json!({ "content": content });

            if let Some(b) = bookmark {

                body["bookmark"] = b.into();

            }

            if let Some(ch) = change {

                body["change"] = ch.into();

            }

            let base = t.repo_path()?;

            let r: hub_api::CoverageReport = c.send("POST", &format!("{base}/coverage"), &body)?;

            if json {

                println!("{}", serde_json::to_string(&r)?);

            } else {

                let pct = r.percent.map(|p| format!("{p:.1}%")).unwrap_or_else(|| "no lines".into());

                let on = if r.bookmark.is_empty() { format!("state {}", &r.state[..12.min(r.state.len())]) } else { r.bookmark.clone() };

                println!(

                    "{} uploaded coverage {pct} ({} of {} lines, {} files) for {on} {}",

                    ui::ok("✔"),

                    r.lines_hit,

                    r.lines_found,

                    r.files,

                    ui::dim(&r.url)

                );

                if r.dropped > 0 {

                    println!("  {}", ui::dim(&format!("{} files it named are not in that state and were left out", r.dropped)));

                }

            }

            Ok(0)

        }

JSON under `/api/v1`, bearer tokens, `application/json`, RFC 7807 problem responses (an unknown route, or a known one with the wrong method, too), cursor pagination (`?after=`), ETags on cacheable resources, and rate limits per token. Resources mirror the pages: `/repos/{owner}/{name}`, `/bookmarks`, `/changes/{hash}`, `/tree/{ref}/{path}`, `/file/{ref}/{path}`, `/blame`, `/history/{ref}` (`?path=`, `?offset=&limit=`, or `?r=REVSET` for the changes a revset selects, evaluated under a two-second budget and answering 400 when it does not parse or goes past it; Oxygen's docs/revset_query_language.md), `/compare`, `/conflicts`, `/proposals` (`?state=open|draft|landed|closed|all`, open ones and drafts by default; a source bookmark may hold `/`, and `owner/name:bookmark` naming the repository itself is its own bookmark), `/proposals/{n}/comments|reviews|checks|files`, `PATCH /proposals/{n}` (the title and description, for its author or a maintainer, and `labels`, the whole set by name, for triage; what is not given stays), `POST /proposals/{n}/land` (repo:write: what the Land button does, `409` with the blockers when it cannot; a landed proposal keeps the changes it landed and reports `landed` with who, how (`web`, or `api (token NAME)`) and when, which its event, the audit log and the push record also say, adding when nobody approved), `/proposals/{n}/content/{base|head}/{path}` (a changed file's bytes in the target's state or the projected state, for editor diff views), `/issues` (creating an issue can set its labels, assignees and milestone; `PATCH /issues/{n}` edits the title and body for the author or triage, and replaces labels, assignees or the milestone for triage, recording each on the timeline; unknown names are refused before anything changes), `/labels` (`POST` creates or changes one, `DELETE /labels/{name}` removes it; triage), `/milestones` (`POST` creates one or changes the description and due date of the one with that title, `PATCH /milestones/{title}` changes title, description, due date or state; triage), `/releases` (`PATCH /releases/{tag}` changes the title, notes or prerelease flag, repo:write), `/tags`, `/trust`, `/pushes`, `/hooks`, `/checks`, `/user`, `/notifications` (the caller's inbox and unread count; `/notifications/{id}/read` and `/notifications/read` mark read; scope `user:read`), `/users/{name}`, `/orgs/{name}`, `/checks/{state}` (the checks on a state, named by its hash, by its heads joined with `+`, by the one head of a single-head state, or by a bookmark), `/checks/required` (`GET`, and `PUT` of the whole list by a repository admin: the checks a proposal needs, passed, to land), `/projects/{owner}` (list, create) and `/projects/{owner}/{n}` (the board as the caller may see it, `PATCH` for title, description, privacy and closing, and `owner` to transfer it to an owner the caller may make boards under (#125: it takes the next number there, and its old address redirects), `/cards` to list them by column or add a reference or note to a column named by id or name, `/cards/{id}` to move or remove one; `repo:write` and the page's rights to change), `/coverage` and `/tests` (a bookmark's coverage and JUnit test reports from Flow jobs, newest first, `?bookmark=&limit=`), `POST /coverage` (repo:write, #136: `{"bookmark"}` or `{"change"}` and `content`, an LCOV or Cobertura report measured off the hub, kept for that state as uploaded by the caller, shown "uploaded by NAME" wherever a run number would be, and setting no check; `oh coverage upload FILE`), `/coverage/{id}` (a report with its files) and `/tests/{id}` (a report with its cases, `?status=failed` for the failed and errored ones), `/flow/runs` (`GET`: runs newest first, `?before=N&limit=`; `POST {"bookmark"}` or `{"change"}`, repo:write: run a bookmark's current state, the default bookmark when none is named, or one change's state; with `"jobs": [NAME..]` only those jobs and what they depend on, each one a manual run starts, else `409` naming the ones it does (#130); `oh run start --job NAME`), `/flow/runs/{n}` (a run with its jobs), `/flow/runs/{n}/jobs/{job}/log` (`?after=seq&limit=`) and `/flow/runs/{n}/rerun` (repo:write: a finished run again), both answering `201 {"number", "url"}` or `409` with why nothing started, `/flow/runs/{n}/cancel` (repo:write: a run still going ends as `cancelled`, its running jobs stopped when their runner next reports; `409` when it had already ended; a push to a bookmark cancels the push runs of it still going), `/search`. A change's author carries the account its key is linked to, or `null` and a `note` saying the key is not linked and where its owner links it; `via` names the delegate (an agent) that signed for the author when one did, with the delegation's scope and expiry and whether it has since been revoked, and the pages show the same as an `agent` badge, a `via` row under the author on the change page, and a count under the author in Contributors (the change page shows a "key not linked" tag, with a link to Settings → Keys for signed-in viewers). Shapes reuse `oxy --json` shapes wherever one exists (change, status, blame, diff) so tools written against the CLI work against the hub. `crates/hub-api` holds the types with `serde` derives and is what the runner and the VS Code extension will link.

        · {% if r.uploaded() %}<span class="tag warn" title="Uploaded with oh coverage upload: measured off the hub, not by a Flow job">uploaded by {{ r.uploader }}</span>{% else %}<a href="{{ repo.url }}/flow/{{ r.run_number }}">run #{{ r.run_number }}</a>{% endif %} · <time title="{{ r.created_ms|when_i }}">{{ r.created_ms|ago_i }}</time></div>

      {% if let Some(d) = delta %}<div class="cov-change {{ self.delta_class(d) }}" title="Against {% if let Some(p) = previous %}{{ p.source() }}{% endif %}">{{ self.signed(d) }} since the run before</div>{% else %}<div class="cov-change same">the first report here</div>{% endif %}

      {% if let Some((found, hit)) = self.lines_delta() %}<li>{% if *found >= 0 %}+{% endif %}{{ found }} relevant lines, {% if *hit >= 0 %}+{% endif %}{{ hit }} covered, against {% if let Some(p) = previous %}{{ p.source() }}{% endif %}</li>{% endif %}

      <li>{% if r.uploaded() %}Uploaded by <strong>{{ r.uploader }}</strong>, measured off the hub{% else %}Job <strong>{{ r.job }}</strong>{% endif %}{% if r.dropped > 0 %}; {{ r.dropped }} files the report named are not in this state and are left out{% endif %}</li>

    <td><a href="{{ repo.url }}/coverage?report={{ x.report.id }}">{% if x.report.uploaded() %}upload by {{ x.report.uploader }}{% else %}#{{ x.report.run_number }}{% endif %}</a></td>

<p class="small mute">{{ file.lines_hit }} of {{ file.lines_found }} lines run{% if file.functions_found > 0 %}, {{ file.functions_hit }} of {{ file.functions_found }} functions{% endif %}{% if file.branches_found > 0 %}, {{ file.branches_hit }} of {{ file.branches_found }} branches{% endif %} · {% if report.uploaded() %}uploaded by {{ report.uploader }}{% else %}<a href="{{ repo.url }}/flow/{{ report.run_number }}">run #{{ report.run_number }}</a>{% endif %} on {{ report.bookmark }} · <span class="cov-key cov-hit">run</span> <span class="cov-key cov-miss">missed</span></p></div>