Variables for pipelines at the site, organisation and repository 2a83f5303458

-- Variables for OxygenFlow jobs (hub spec §13.2, #102), per level: the

-- site (scope_id 0), an organisation or a repository. A job gets its

-- repository's effective set, inner levels overriding outer. Not secrets:

-- those are mounted by Flow (OxygenFlow spec §9), never put in the

-- environment.

CREATE TABLE flow_variables (

    scope       TEXT NOT NULL CHECK (scope IN ('site', 'org', 'repo')),

    scope_id    INTEGER NOT NULL DEFAULT 0,

    name        TEXT NOT NULL,

    value       TEXT NOT NULL,

    updated_ms  INTEGER NOT NULL,

    PRIMARY KEY (scope, scope_id, name)

);

//! The three levels settings live at, site, organisation and repository,

//! and how a repository sees the levels above it. Shared by AI assistants

//! (§13.4) and Flow variables (§13.2).

use crate::db::Pool;

/// Which level a setting belongs to.

#[derive(Clone, Copy, Debug, PartialEq, Eq)]

pub enum Scope {

    Site,

    Org(i64),

    Repo(i64),

}

impl Scope {

    pub fn kind(self) -> &'static str {

        match self {

            Scope::Site => "site",

            Scope::Org(_) => "org",

            Scope::Repo(_) => "repo",

        }

    }

    pub fn id(self) -> i64 {

        match self {

            Scope::Site => 0,

            Scope::Org(id) | Scope::Repo(id) => id,

        }

    }

}

/// A level as a settings page addresses it.

pub struct At {

    pub scope: Scope,

    /// The owner name, so a repository finds its organisation.

    pub owner: String,

    /// What the audit log names.

    pub label: String,

}

impl At {

    pub fn site() -> At {

        At { scope: Scope::Site, owner: String::new(), label: "site".into() }

    }

    pub fn org(org: &crate::db::accounts::Organisation) -> At {

        At { scope: Scope::Org(org.id), owner: org.name.clone(), label: org.name.clone() }

    }

    pub fn repo(row: &crate::db::repos::RepoRow) -> At {

        At { scope: Scope::Repo(row.id), owner: row.owner.clone(), label: format!("{}/{}", row.owner, row.name) }

    }

}

/// The levels above a scope, outermost first: the site, then the

/// organisation when the owner is one.

pub async fn above(pool: &Pool, scope: Scope, owner: &str) -> sqlx::Result<Vec<(Scope, String)>> {

    let mut out = vec![];

    match scope {

        Scope::Site => {}

        Scope::Org(_) => out.push((Scope::Site, "the site".to_string())),

        Scope::Repo(_) => {

            out.push((Scope::Site, "the site".to_string()));

            if let Some(org) = crate::db::accounts::org_by_name(pool, owner).await? {

                out.push((Scope::Org(org.id), format!("the {} organisation", org.name)));

            }

        }

    }

    Ok(out)

}

//! Variables for OxygenFlow jobs (hub spec §13.2, oxygenhub#102): plain

//! `NAME=value` pairs kept per site, organisation and repository. A job

//! gets its repository's effective set, inner levels overriding outer, as

//! the claim's `env`; Flow sets them under `FLOW_*` and the pipeline's

//! own `env`. They are not secrets and are shown in full on the settings

//! pages; secrets are mounted by Flow (OxygenFlow spec §9).

use crate::db::{now_ms, Pool};

use crate::levels::{self, Scope};

use sqlx::FromRow;

use std::collections::BTreeMap;

/// How many a level may hold.

pub const MAX_PER_LEVEL: usize = 100;

/// The longest value.

pub const MAX_VALUE: usize = 4096;

#[derive(Clone, Debug, FromRow)]

pub struct Variable {

    pub scope: String,

    pub scope_id: i64,

    pub name: String,

    pub value: String,

    pub updated_ms: i64,

}

/// A variable's name: a letter or `_`, then letters, digits and `_`, up

/// to 64; never `FLOW_*`, which Flow sets itself.

pub fn name_ok(name: &str) -> bool {

    let mut chars = name.chars();

    let first = chars.next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_');

    first && name.len() <= 64 && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') && !name.starts_with("FLOW_")

}

/// A value: up to 4096 characters, no control characters but newline and tab.

pub fn value_ok(value: &str) -> bool {

    value.len() <= MAX_VALUE && !value.chars().any(|c| c.is_control() && c != '\n' && c != '\t')

}

pub async fn list(pool: &Pool, scope: Scope) -> sqlx::Result<Vec<Variable>> {

    sqlx::query_as("SELECT * FROM flow_variables WHERE scope = ? AND scope_id = ? ORDER BY name")

        .bind(scope.kind())

        .bind(scope.id())

        .fetch_all(pool)

        .await

}

/// Set a variable at a level, new or changed.

pub async fn set(pool: &Pool, scope: Scope, name: &str, value: &str) -> sqlx::Result<()> {

    sqlx::query(

        "INSERT INTO flow_variables (scope, scope_id, name, value, updated_ms) VALUES (?, ?, ?, ?, ?) \

         ON CONFLICT (scope, scope_id, name) DO UPDATE SET value = excluded.value, updated_ms = excluded.updated_ms",

    )

    .bind(scope.kind())

    .bind(scope.id())

    .bind(name)

    .bind(value)

    .bind(now_ms())

    .execute(pool)

    .await?;

    Ok(())

}

/// Remove a variable from a level; false when it was not there.

pub async fn delete(pool: &Pool, scope: Scope, name: &str) -> sqlx::Result<bool> {

    let r = sqlx::query("DELETE FROM flow_variables WHERE scope = ? AND scope_id = ? AND name = ?")

        .bind(scope.kind())

        .bind(scope.id())

        .bind(name)

        .execute(pool)

        .await?;

    Ok(r.rows_affected() == 1)

}

/// Forget a level's variables, when the organisation or repository goes.

pub async fn delete_scope(pool: &Pool, scope: Scope) -> sqlx::Result<()> {

    sqlx::query("DELETE FROM flow_variables WHERE scope = ? AND scope_id = ?").bind(scope.kind()).bind(scope.id()).execute(pool).await?;

    Ok(())

}

/// A variable as a level sees it: where it comes from, and whether the

/// level's own value hides it.

#[derive(Clone, Debug)]

pub struct Seen {

    pub variable: Variable,

    /// `the site`, `the acme organisation`, or empty for the level's own.

    pub from: String,

    /// An inner level sets the same name, so this value is not the one used.

    pub overridden: bool,

}

/// Every variable a level sees, outermost first, inherited ones marked

/// overridden where an inner level sets the same name.

pub async fn seen(pool: &Pool, scope: Scope, owner: &str) -> sqlx::Result<Vec<Seen>> {

    let mut lvls = levels::above(pool, scope, owner).await?;

    lvls.push((scope, String::new()));

    let mut out: Vec<Seen> = vec![];

    for (level, from) in lvls {

        for v in list(pool, level).await? {

            for o in out.iter_mut().filter(|o| o.variable.name == v.name) {

                o.overridden = true;

            }

            out.push(Seen { variable: v, from: from.clone(), overridden: false });

        }

    }

    Ok(out)

}

/// The variables a repository's jobs get.

pub async fn env_for_repo(pool: &Pool, repo: &crate::db::repos::RepoRow) -> sqlx::Result<BTreeMap<String, String>> {

    Ok(seen(pool, Scope::Repo(repo.id), &repo.owner)

        .await?

        .into_iter()

        .filter(|s| !s.overridden)

        .map(|s| (s.variable.name, s.variable.value))

        .collect())

}

#[cfg(test)]

mod tests {

    use super::*;

    #[test]

    fn names_and_values_are_checked() {

        assert!(name_ok("CC") && name_ok("_x") && name_ok("RUST_LOG"));

        assert!(!name_ok("") && !name_ok("1A") && !name_ok("A-B") && !name_ok("FLOW_REPO") && !name_ok(&"A".repeat(65)));

        assert!(value_ok("clang") && value_ok("two\nlines\twith tab") && value_ok(""));

        assert!(!value_ok("a\u{0}b") && !value_ok(&"x".repeat(MAX_VALUE + 1)));

    }

}

//! The Flow variables panel (hub spec §13.2, oxygenhub#102) shared by

//! the administration, organisation and repository settings pages: the

//! level's own variables and the ones it inherits.

use crate::auth::Viewer;

use crate::db::accounts;

use crate::levels::At;

use crate::variables::{self, Seen};

use crate::Shared;

use serde::Deserialize;

#[derive(Deserialize)]

pub struct VariableForm {

    #[serde(rename = "_csrf")]

    pub csrf_token: String,

    #[serde(default)]

    action: String,

    #[serde(default)]

    name: String,

    #[serde(default)]

    value: String,

}

/// Everything the panel shows.

pub struct Panel {

    /// The level's own variables.

    pub own: Vec<Seen>,

    /// The ones the levels above provide, outermost first.

    pub inherited: Vec<Seen>,

}

pub async fn panel(app: &Shared, at: &At) -> sqlx::Result<Panel> {

    let (own, inherited): (Vec<_>, Vec<_>) =

        variables::seen(&app.db, at.scope, &at.owner).await?.into_iter().partition(|v| v.from.is_empty());

    Ok(Panel { own, inherited })

}

/// Apply the panel's form: `Ok(notice)` or `Err(message for the user)`.

pub async fn apply(app: &Shared, viewer: &Viewer, at: &At, f: VariableForm) -> sqlx::Result<Result<String, String>> {

    let name = f.name.trim();

    if !variables::name_ok(name) {

        return Ok(Err(

            "A variable's name is a letter or _, then letters, digits and _, up to 64, and not FLOW_… (Flow sets those).".into()

        ));

    }

    Ok(Ok(match f.action.as_str() {

        "set" => {

            let value = f.value.replace("\r\n", "\n");

            if !variables::value_ok(&value) {

                return Ok(Err(format!("A value is up to {} characters of text.", variables::MAX_VALUE)));

            }

            let own = variables::list(&app.db, at.scope).await?;

            if own.len() >= variables::MAX_PER_LEVEL && !own.iter().any(|v| v.name == name) {

                return Ok(Err(format!("Up to {} variables here.", variables::MAX_PER_LEVEL)));

            }

            variables::set(&app.db, at.scope, name, &value).await?;

            accounts::audit(&app.db, viewer.account(), "flow.variable.set", &at.label, name).await.ok();

            format!("{name} set.")

        }

        "delete" => {

            if !variables::delete(&app.db, at.scope, name).await? {

                return Ok(Err(format!("There is no variable {name} here.")));

            }

            accounts::audit(&app.db, viewer.account(), "flow.variable.deleted", &at.label, name).await.ok();

            format!("{name} removed.")

        }

        _ => return Ok(Err("Unknown action.".into())),

    }))

}

{% macro panel(base, v, csrf, what) %}

<section class="panel" id="variables">

  <h2>Variables</h2>

  <p class="small mute">{{ what }} Jobs get them in their environment under <code>FLOW_*</code> and the pipeline's own <code>env</code>; they are part of each job's receipt. Not for secrets: these are shown in full to everyone who can see this page.</p>

  {% if !v.inherited.is_empty() %}

  <h3>Inherited</h3>

  <div class="table"><table>

  <tr><th>Name</th><th>Value</th><th>From</th></tr>

  {% for i in v.inherited %}<tr{% if i.overridden %} class="mute"{% endif %}><td><code>{{ i.variable.name }}</code></td><td class="small"><code>{{ i.variable.value }}</code></td><td class="small">{{ i.from }}{% if i.overridden %} · <span title="A level below sets the same name">overridden</span>{% endif %}</td></tr>{% endfor %}

  </table></div>

  {% endif %}

  {% if v.own.is_empty() %}<p class="mute small">{% if v.inherited.is_empty() %}None.{% else %}None of its own.{% endif %}</p>{% else %}

  <div class="table"><table>

  <tr><th>Name</th><th>Value</th><th></th></tr>

  {% for o in v.own %}<tr><td><code>{{ o.variable.name }}</code></td><td class="small"><code>{{ o.variable.value }}</code></td>

  <td><form class="tiny" method="post" action="{{ base }}/variables"><input type="hidden" name="_csrf" value="{{ csrf }}"><input type="hidden" name="name" value="{{ o.variable.name }}"><button name="action" value="delete" class="danger-btn">remove</button></form></td></tr>{% endfor %}

  </table></div>{% endif %}

  <form class="card" method="post" action="{{ base }}/variables">

    <input type="hidden" name="_csrf" value="{{ csrf }}"><input type="hidden" name="action" value="set">

    <h3>Set a variable</h3>

    <label for="vname">Name</label><input type="text" id="vname" name="name" placeholder="RUSTFLAGS" required>

    <label for="vvalue">Value</label><textarea id="vvalue" name="value" rows="2"></textarea>

    <p class="small mute">An existing name takes the new value; a name set above is overridden here.</p>

    <button type="submit">SET</button>

  </form>

</section>

{% endmacro %}

- **Variables for pipelines (#102).** Administration, organisation and repository settings hold variables for OxygenFlow jobs, inner levels overriding outer, each page showing what it inherits; a claim carries the repository's effective set and Flow adds them to the job's environment under `FLOW_*` and the pipeline's own. Not for secrets, which Flow mounts.

pub use crate::levels::Scope;

    let mut levels = crate::levels::above(pool, scope, owner).await?;

    for (level, from) in crate::levels::above(pool, scope, owner).await? {

        // The site's, the owner's and the repository's variables (#102).

        env: crate::variables::env_for_repo(&app.db, &row).await?,

pub mod levels;

pub mod variables;

    /// The site's Flow variables (#102).

    variables: super::variables::Panel,

    let variables = super::variables::panel(app, &site_assistants()).await.map_err(e)?;

        variables,

fn site_assistants() -> crate::levels::At {

    crate::levels::At::site()

/// `POST /admin/variables`: the site's Flow variables (#102).

pub async fn variables(

    State(app): State<Shared>,

    viewer: Viewer,

    Form(f): Form<super::variables::VariableForm>,

) -> Result<Page<AdminPage>, Error> {

    let shell = Shell::new(&app, "/admin", &viewer);

    require_admin(&viewer, &shell)?;

    csrf(&viewer, &f.csrf_token, &shell)?;

    let outcome = super::variables::apply(&app, &viewer, &site_assistants(), f).await.map_err(|e| Error::from_db(shell.clone(), e))?;

    admin_page(&app, shell, Some(outcome.unwrap_or_else(|e| e))).await

}

use crate::assistants::{self, Assistant, Available, Fields, APIS, DEFAULT_BUDGET};

use crate::levels::At;

pub async fn panel(app: &Shared, at: &At) -> sqlx::Result<Panel> {

pub async fn apply(app: &Shared, viewer: &Viewer, at: &At, f: AssistantForm) -> sqlx::Result<Result<String, String>> {

mod variables;

        .route("/orgs/{name}/settings/variables", post(orgs::variables))

        .route("/admin/variables", post(admin::variables))

        .route("/{owner}/{name}/settings/variables", post(repo_settings::variables))

    /// The organisation's Flow variables (#102).

    variables: super::variables::Panel,

    let variables = super::variables::panel(app, &org_assistants(&org)).await.map_err(|e| Error::from_db(shell.clone(), e))?;

    Ok(Page(OrgSettingsPage { shell, org, members, teams, hooks, notice, error, runners, runner_token: None, assistants, variables }))

fn org_assistants(org: &accounts::Organisation) -> crate::levels::At {

    crate::levels::At::org(org)

/// `POST /orgs/{name}/settings/variables`: the organisation's Flow

/// variables (#102).

pub async fn variables(

    State(app): State<Shared>,

    viewer: Viewer,

    Path(name): Path<String>,

    Form(f): Form<super::variables::VariableForm>,

) -> Result<Page<OrgSettingsPage>, Error> {

    let shell = Shell::new(&app, &format!("/orgs/{name}/settings"), &viewer);

    let (_, org) = owned_org(&app, &shell, &viewer, &name).await?;

    csrf(&viewer, &f.csrf_token, &shell)?;

    match super::variables::apply(&app, &viewer, &org_assistants(&org), f).await.map_err(|e| Error::from_db(shell.clone(), e))? {

        Ok(notice) => org_page(&app, shell, org, Some(notice), None).await,

        Err(error) => org_page(&app, shell, org, None, Some(error)).await,

    }

}

    /// The repository's Flow variables (#102).

    variables: super::variables::Panel,

    let variables = super::variables::panel(app, &repo_assistants(&row)).await.map_err(|e| Error::from_db(shell.clone(), e))?;

        variables,

    crate::assistants::delete_scope(&app.db, crate::levels::Scope::Repo(row.id)).await.ok();

    crate::variables::delete_scope(&app.db, crate::levels::Scope::Repo(row.id)).await.ok();

fn repo_assistants(row: &db::repos::RepoRow) -> crate::levels::At {

    crate::levels::At::repo(row)

        Ok(notice) => settings_page(&app, shell, row, role, Some(notice), None).await,

        Err(error) => settings_page(&app, shell, row, role, None, Some(error)).await,

    }

}

/// `POST /{owner}/{name}/settings/variables`: the repository's Flow

/// variables (#102), for its admins.

pub async fn variables(

    State(app): State<Shared>,

    viewer: Viewer,

    Path((owner, name)): Path<(String, String)>,

    Form(f): Form<super::variables::VariableForm>,

) -> Result<Page<RepoSettingsPage>, Error> {

    let shell = Shell::new(&app, &format!("/{owner}/{name}/settings"), &viewer);

    let (_, row, role) = lookup_role(&app, &shell, &viewer, &owner, &name, Access::Admin).await?;

    csrf(&viewer, &f.csrf_token, &shell)?;

    match super::variables::apply(&app, &viewer, &repo_assistants(&row), f).await.map_err(|e| Error::from_db(shell.clone(), e))? {

    assert!(r.body.contains("smtps://hub:***@mail.example.org:465") && !r.body.contains("hub:secret"), "{}", r.body);

#[tokio::test(flavor = "multi_thread")]

async fn variables_from_three_levels_reach_a_job_inner_overriding_outer() {

    use oxygenhub::levels::Scope;

    use oxygenhub::variables;

    let h = hub().await;

    let app = &h.app;

    // Andy, the site's administrator, owns the acme organisation.

    let res = oxygenhub::router(app.clone())

        .oneshot(

            Request::post("/register")

                .header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")

                .body(Body::from("username=andy&email=a%40example.com&password=correct-horse-battery"))

                .unwrap(),

        )

        .await

        .unwrap();

    let cookie = res.headers().get(header::SET_COOKIE).unwrap().to_str().unwrap().split(';').next().unwrap().to_string();

    let andy = db::accounts::account_by_name(&app.db, "andy").await.unwrap().unwrap();

    db::accounts::create_org(&app.db, "acme", "Acme", andy.id).await.unwrap();

    create(app, "acme", "app").await;

    let page = |path: &str, form: Option<String>| {

        let mut req =

            Request::builder().method(if form.is_some() { "POST" } else { "GET" }).uri(path).header(header::COOKIE, cookie.clone());

        if form.is_some() {

            req = req.header(header::CONTENT_TYPE, "application/x-www-form-urlencoded");

        }

        let req = req.body(Body::from(form.unwrap_or_default())).unwrap();

        let app = app.clone();

        async move {

            let res = oxygenhub::router(app).oneshot(req).await.unwrap();

            let status = res.status();

            let body = String::from_utf8(res.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap();

            (status, body)

        }

    };

    let csrf_of = |page: &str| {

        let i = page.find("name=\"_csrf\" value=\"").expect("csrf field") + "name=\"_csrf\" value=\"".len();

        page[i..].split('"').next().unwrap().to_string()

    };

    // The site sets CC and LANG, the organisation its own CC, the

    // repository RUST_LOG from its settings page.

    variables::set(&app.db, Scope::Site, "CC", "gcc").await.unwrap();

    variables::set(&app.db, Scope::Site, "LANG", "C").await.unwrap();

    let org = db::accounts::org_by_name(&app.db, "acme").await.unwrap().unwrap();

    variables::set(&app.db, Scope::Org(org.id), "CC", "clang").await.unwrap();

    let (st, settings) = page("/acme/app/settings", None).await;

    assert_eq!(st, StatusCode::OK, "{settings}");

    assert!(settings.contains("Inherited") && settings.contains("the acme organisation") && settings.contains("overridden"), "{settings}");

    let csrf = csrf_of(&settings);

    let (_, body) = page("/acme/app/settings/variables", Some(format!("_csrf={csrf}&action=set&name=RUST_LOG&value=debug"))).await;

    assert!(body.contains("RUST_LOG set.") && body.contains("<code>debug</code>"), "{body}");

    let (_, body) = page("/acme/app/settings/variables", Some(format!("_csrf={csrf}&action=set&name=FLOW_REPO&value=x"))).await;

    assert!(body.contains("not FLOW_"), "{body}");

    let repo = db::repos::get(&app.db, "acme", "app").await.unwrap().unwrap();

    db::repos::set_flow(&app.db, repo.id, "on").await.unwrap();

    let env = variables::env_for_repo(&app.db, &repo).await.unwrap();

    assert_eq!(

        env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect::<Vec<_>>(),

        [("CC", "clang"), ("LANG", "C"), ("RUST_LOG", "debug")],

        "the organisation's CC wins over the site's"

    );

    // A claim carries them.

    let url = format!("{}/acme/app", h.base);

    let work = tempfile::tempdir().unwrap();

    let c = client(&url, work.path());

    write(work.path(), ".flow/pipeline.kdl", &pipeline(""));

    write(work.path(), "test.sh", "echo ok\n");

    c.snapshot("test").unwrap();

    c.commit("Add a pipeline", &[]).unwrap();

    assert!(c.push("origin", false).unwrap().iter().all(|r| r.ok));

    let runs = runs_of(app, "acme", "app").await;

    assert_eq!(runs.len(), 1, "{runs:?}");

    let got = claim(app).await.unwrap_or_else(|| panic!("a job: {:?}", runs[0]));

    assert_eq!(got["env"], serde_json::json!({ "CC": "clang", "LANG": "C", "RUST_LOG": "debug" }), "{got}");

    // Removing a site variable from the admin page takes effect at the next claim.

    let (_, admin) = page("/admin", None).await;

    assert!(admin.contains("Variables") && admin.contains("<code>gcc</code>"), "{admin}");

    let (_, body) = page("/admin/variables", Some(format!("_csrf={}&action=delete&name=LANG", csrf_of(&admin)))).await;

    assert!(body.contains("LANG removed."), "{body}");

    let got = claim(app).await.expect("another job");

    assert_eq!(got["env"], serde_json::json!({ "CC": "clang", "RUST_LOG": "debug" }), "{got}");

}

- **Variables** (oxygenhub#102): Administration, an organisation's settings and a repository's settings each hold `NAME=value` pairs (a name is a letter or `_` then letters, digits and `_`, up to 64, never `FLOW_*`; a value up to 4096 characters; up to 100 per level) under **Variables**, each page showing what it inherits and which inherited names it overrides. A claim carries the repository's effective set as `env`, inner levels overriding outer; Flow puts them in the job's environment under `FLOW_*` and the pipeline's own `env`, so they are part of the receipt. They are not secrets and are shown in full to whoever sees the page; secrets are mounted by Flow (OxygenFlow spec §9), never put in the environment. Changes are audited (`flow.variable.set`, `flow.variable.deleted`); `flow run --var NAME=VALUE` gives a local run the same.

{% import "variables_panel.html" as va %}

{% call va::panel("/admin", variables, shell.csrf, "Every repository's jobs get these, unless an organisation or repository sets the same name.") %}

{% import "variables_panel.html" as va %}

  <a href="#variables">Variables</a>

{% call va::panel("/orgs/" ~ org.name ~ "/settings", variables, shell.csrf, "Jobs of every repository of the organisation get these, beside the site's and each repository's own.") %}

{% import "variables_panel.html" as va %}

  <a href="#variables">Variables</a>

{% call va::panel(repo.url ~ "/settings", variables, shell.csrf, "This repository's jobs get these, beside the ones the site and its owner set.") %}