A repository can live in its owner's bucket 1f78655a0466

-- Repositories kept in their owner's bucket (oxygenhub#96, oxygen#68):

-- the store's URL, the credentials sealed under keys/secrets.key, and the

-- last check of the store.

CREATE TABLE repo_storage (

    repo_id      INTEGER PRIMARY KEY REFERENCES repositories(id) ON DELETE CASCADE,

    url          TEXT NOT NULL,

    credentials  BLOB NOT NULL,

    created_ms   INTEGER NOT NULL,

    checked_ms   INTEGER,

    check_result TEXT NOT NULL DEFAULT ''

);

//! Repositories kept in their owner's bucket (hub spec §4, oxygenhub#96,

//! oxygen#68). The repository's objects and operation log live in an S3,

//! Google Cloud Storage or Azure bucket the owner chose; the hub keeps a

//! bare repository directory with a cache in front (`storage.url` in its

//! config) and the credentials, sealed under `keys/secrets.key`. Stores

//! are opened by URL, and the credentials are handed to them through

//! oxy-strata's lookup, so nothing from the hub host's own environment or

//! cloud profiles is ever used for a repository.

use crate::db::{now_ms, Pool};

use crate::Shared;

use oxy_strata::Credentials;

use std::collections::HashMap;

use std::sync::RwLock;

#[derive(Clone, Debug, sqlx::FromRow)]

pub struct Storage {

    pub repo_id: i64,

    pub url: String,

    /// Sealed; `App.secret_key` opens it.

    pub credentials: Vec<u8>,

    pub created_ms: i64,

    pub checked_ms: Option<i64>,

    pub check_result: String,

}

/// What the stores opened by URL get: each repository's credentials by

/// its store URL, and the URL by repository id for pages.

#[derive(Default)]

struct Known {

    credentials: HashMap<String, Credentials>,

    /// By the repository's directory, unique even between hubs in one process.

    urls: HashMap<std::path::PathBuf, String>,

}

static KNOWN: RwLock<Option<Known>> = RwLock::new(None);

fn lookup(url: &str) -> Option<Credentials> {

    KNOWN.read().ok()?.as_ref()?.credentials.get(url).cloned()

}

fn remember(dir: std::path::PathBuf, url: &str, credentials: Credentials) {

    let mut known = KNOWN.write().unwrap_or_else(|p| p.into_inner());

    let known = known.get_or_insert_with(Known::default);

    known.credentials.insert(url.to_string(), credentials);

    known.urls.insert(dir, url.to_string());

}

fn forget(dir: &std::path::Path) {

    let mut known = KNOWN.write().unwrap_or_else(|p| p.into_inner());

    if let Some(k) = known.as_mut() {

        if let Some(url) = k.urls.remove(dir) {

            k.credentials.remove(&url);

        }

    }

}

/// Where a repository's data lives, when it is not on this hub's disk.

pub fn location(app: &Shared, owner: &str, name: &str) -> Option<String> {

    let dir = app.repos.dir(&crate::repos::RepoName::new(owner, name)?);

    KNOWN.read().ok()?.as_ref()?.urls.get(&dir).cloned()

}

/// The directory of the repository with row `repo_id`.

async fn dir_of(app: &Shared, repo_id: i64) -> Option<std::path::PathBuf> {

    let row = crate::db::repos::get_by_id(&app.db, repo_id).await.ok()??;

    Some(app.repos.dir(&crate::repos::RepoName::new(&row.owner, &row.name)?))

}

/// Hand the cloud stores to oxy-core, and every repository's credentials

/// to them. At start.

pub async fn load(app: &Shared) -> sqlx::Result<()> {

    oxy_strata::register();

    oxy_strata::set_credentials_lookup(lookup);

    for s in all(&app.db).await? {

        match app.secret_key.open(&s.credentials).map(|j| Credentials::from_json(&j)) {

            Some(Ok(c)) => match dir_of(app, s.repo_id).await {

                Some(dir) => remember(dir, &s.url, c),

                None => tracing::warn!("storage: repository {} has a store but no row", s.repo_id),

            },

            _ => tracing::warn!("storage: the credentials of repository {} do not open under keys/secrets.key", s.repo_id),

        }

    }

    Ok(())

}

pub async fn all(pool: &Pool) -> sqlx::Result<Vec<Storage>> {

    sqlx::query_as("SELECT * FROM repo_storage").fetch_all(pool).await

}

pub async fn of(pool: &Pool, repo_id: i64) -> sqlx::Result<Option<Storage>> {

    sqlx::query_as("SELECT * FROM repo_storage WHERE repo_id = ?").bind(repo_id).fetch_optional(pool).await

}

/// Check a store URL a user gave: a bucket on S3, Google Cloud Storage or

/// Azure, with no setting but `region`. A profile, a key file or an

/// endpoint would make the hub use its own host's credentials or reach

/// addresses on its network, and `dir://` its own disk.

pub fn check_url(url: &str) -> Result<oxy_strata::StoreUrl, String> {

    let scheme = url.trim().split_once("://").map(|(s, _)| s).unwrap_or("");

    if !matches!(scheme, "s3" | "gs" | "az") {

        return Err("a store is s3://BUCKET/PREFIX, gs://BUCKET/PREFIX or az://ACCOUNT/CONTAINER/PREFIX".into());

    }

    let parsed = oxy_strata::StoreUrl::parse(url.trim()).map_err(|e| e.to_string())?;

    if parsed.scheme == "az" && parsed.path.is_empty() {

        return Err("an Azure store names the account and the container: az://ACCOUNT/CONTAINER/PREFIX".into());

    }

    if let Some(k) = parsed.params.keys().find(|k| k.as_str() != "region") {

        return Err(format!("?{k}= is not allowed here: the only setting a hub accepts is ?region="));

    }

    Ok(parsed)

}

/// The credentials a form gave for a store of `scheme`.

pub fn credentials_from(

    scheme: &str,

    access_key_id: &str,

    secret_access_key: &str,

    service_account_key: &str,

    tenant_id: &str,

    client_id: &str,

    client_secret: &str,

) -> Result<Credentials, String> {

    let json = match scheme {

        "s3" => serde_json::json!({ "kind": "aws", "access_key_id": access_key_id, "secret_access_key": secret_access_key }),

        "gs" => serde_json::json!({ "kind": "gcp", "service_account_key": service_account_key }),

        "az" => serde_json::json!({ "kind": "azure", "tenant_id": tenant_id, "client_id": client_id, "client_secret": client_secret }),

        _ => return Err("unknown kind of store".into()),

    };

    Credentials::from_json(&json.to_string()).map_err(|e| e.to_string())

}

/// Run the conformance check against a store under a probe prefix of its

/// own, which it leaves empty: what passed, or what failed. With

/// `must_be_empty` the store must hold nothing yet (a new repository's).

pub async fn probe(url: &str, credentials: &Credentials, must_be_empty: bool) -> Result<String, String> {

    let (url, credentials) = (url.to_string(), credentials.clone());

    tokio::task::spawn_blocking(move || {

        use oxy_core::strata::Store as _;

        let store = oxy_strata::CloudStore::open_with(&url, Some(&credentials)).map_err(|e| e.to_string())?;

        if must_be_empty {

            let found = store.list("").map_err(|e| e.to_string())?;

            if !found.is_empty() {

                return Err(format!(

                    "{} already holds {} object(s): a new repository needs an empty bucket or prefix",

                    store.describe(),

                    found.len()

                ));

            }

        }

        let mut nonce = [0u8; 6];

        rand::RngCore::fill_bytes(&mut rand::rngs::OsRng, &mut nonce);

        let prefix = format!("oxy-probe-{}", hex::encode(nonce));

        let passed = oxy_core::strata::conformance(&store, &prefix).map_err(|e| {

            for e in store.list(&prefix).unwrap_or_default() {

                let _ = store.delete(&e.key);

            }

            e.to_string()

        })?;

        Ok(format!("{} checks passed", passed.len()))

    })

    .await

    .map_err(|e| e.to_string())?

}

/// Create repository `name` (row `repo_id`, already in the database) in

/// the store at `url`: checked first, then the credentials sealed and

/// kept, then the bare repository made over the store.

pub async fn create(app: &Shared, name: &crate::repos::RepoName, repo_id: i64, url: &str, credentials: Credentials) -> Result<(), String> {

    let result = probe(url, &credentials, true).await?;

    let sealed = app.secret_key.seal(&credentials.to_json());

    let dir = app.repos.dir(name);

    remember(dir.clone(), url, credentials);

    let made = {

        let (app, name, url) = (app.clone(), name.clone(), url.to_string());

        tokio::task::spawn_blocking(move || app.repos.create_on(&name, &url).map(drop).map_err(|e| e.to_string()))

            .await

            .map_err(|e| e.to_string())

            .and_then(|r| r)

    };

    if let Err(e) = made {

        forget(&dir);

        return Err(format!("the repository could not be made in {url}: {e}"));

    }

    let now = now_ms();

    sqlx::query("INSERT INTO repo_storage (repo_id, url, credentials, created_ms, checked_ms, check_result) VALUES (?, ?, ?, ?, ?, ?)")

        .bind(repo_id)

        .bind(url)

        .bind(sealed)

        .bind(now)

        .bind(now)

        .bind(&result)

        .execute(&app.db)

        .await

        .map_err(|e| e.to_string())?;

    Ok(())

}

/// Check a repository's store again and record the result.

pub async fn check(app: &Shared, s: &Storage) -> Result<String, String> {

    let json = app.secret_key.open(&s.credentials).ok_or("the credentials do not open under this hub's key")?;

    let credentials = Credentials::from_json(&json).map_err(|e| e.to_string())?;

    let result = probe(&s.url, &credentials, false).await;

    let text = match &result {

        Ok(r) => r.clone(),

        Err(e) => format!("failed: {e}"),

    };

    sqlx::query("UPDATE repo_storage SET checked_ms = ?, check_result = ? WHERE repo_id = ?")

        .bind(now_ms())

        .bind(&text)

        .bind(s.repo_id)

        .execute(&app.db)

        .await

        .map_err(|e| e.to_string())?;

    result

}

/// Replace a repository's credentials (a rotated key), once they pass the

/// check against its store.

pub async fn set_credentials(app: &Shared, s: &Storage, credentials: Credentials) -> Result<String, String> {

    let result = probe(&s.url, &credentials, false).await?;

    sqlx::query("UPDATE repo_storage SET credentials = ?, checked_ms = ?, check_result = ? WHERE repo_id = ?")

        .bind(app.secret_key.seal(&credentials.to_json()))

        .bind(now_ms())

        .bind(&result)

        .bind(s.repo_id)

        .execute(&app.db)

        .await

        .map_err(|e| e.to_string())?;

    if let Some(dir) = dir_of(app, s.repo_id).await {

        remember(dir, &s.url, credentials);

    }

    Ok(result)

}

/// A repository is gone (its store row with it): forget its credentials.

pub fn removed(app: &Shared, name: &crate::repos::RepoName) {

    forget(&app.repos.dir(name));

}

/// A repository moved to another name (a transfer): its store goes with it.

pub fn renamed(app: &Shared, from: &crate::repos::RepoName, to: &crate::repos::RepoName) {

    let mut known = KNOWN.write().unwrap_or_else(|p| p.into_inner());

    if let Some(k) = known.as_mut() {

        if let Some(url) = k.urls.remove(&app.repos.dir(from)) {

            k.urls.insert(app.repos.dir(to), url);

        }

    }

}

//! Repositories kept in their owner's bucket (oxygenhub#96): the New

//! repository form refuses what the hub must not accept and creates

//! nothing then; with a real S3 test bucket configured (`OXY_TEST_S3` and

//! `OXY_TEST_S3_PROFILE`, keys read from that profile and handed over), a

//! repository is made in the bucket, takes a push, shows where it lives,

//! passes Settings → Storage → Check, and the test removes what it wrote.

use axum::body::Body;

use axum::http::{header, Request, StatusCode};

use http_body_util::BodyExt;

use oxy_core::repo::Repo;

use oxygenhub::config::Config;

use oxygenhub::repos::RepoName;

use oxygenhub::{db, Shared};

use tower::ServiceExt;

fn hermetic() {

    static ONCE: std::sync::Once = std::sync::Once::new();

    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();

    ONCE.call_once(|| {

        let d = tempfile::tempdir().unwrap();

        std::env::set_var("OXY_HOME", d.path());

        DIR.set(d).ok();

    });

}

struct Hub {

    app: Shared,

    base: String,

    _dir: tempfile::TempDir,

}

async fn hub() -> Hub {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let mut config = Config::load(dir.path()).unwrap();

    config.auth.push_tokens = vec!["site".into()];

    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();

    let port = listener.local_addr().unwrap().port();

    config.server.base_url = format!("http://127.0.0.1:{port}");

    let app = oxygenhub::build(config).await.unwrap();

    let router = oxygenhub::router(app.clone());

    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });

    Hub { app, base: format!("http://127.0.0.1:{port}"), _dir: dir }

}

async fn send(app: &Shared, method: &str, path: &str, cookie: Option<&str>, form: Option<&str>) -> (StatusCode, String, Option<String>) {

    let mut req = Request::builder().method(method).uri(path);

    if let Some(c) = cookie {

        req = req.header(header::COOKIE, format!("oxyhub={c}"));

    }

    let body = match form {

        Some(f) => {

            req = req.header(header::CONTENT_TYPE, "application/x-www-form-urlencoded");

            Body::from(f.to_string())

        }

        None => Body::empty(),

    };

    let res = oxygenhub::router(app.clone()).oneshot(req.body(body).unwrap()).await.unwrap();

    let status = res.status();

    let cookie = res

        .headers()

        .get_all(header::SET_COOKIE)

        .iter()

        .filter_map(|v| v.to_str().ok())

        .filter_map(|c| c.split(';').next())

        .filter_map(|kv| kv.strip_prefix("oxyhub="))

        .map(String::from)

        .next();

    let bytes = res.into_body().collect().await.unwrap().to_bytes();

    (status, String::from_utf8_lossy(&bytes).into_owned(), cookie)

}

fn csrf_of(page: &str) -> String {

    let i = page.find("name=\"_csrf\" value=\"").expect("csrf field") + "name=\"_csrf\" value=\"".len();

    page[i..].split('"').next().unwrap().to_string()

}

fn enc(s: &str) -> String {

    s.bytes()

        .map(|b| if b.is_ascii_alphanumeric() || b"-_.~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") })

        .collect()

}

async fn andy(app: &Shared) -> String {

    let (st, body, cookie) =

        send(app, "POST", "/register", None, Some("username=andy&email=andy%40example.com&password=correct-horse-battery&invite=")).await;

    assert_eq!(st, StatusCode::SEE_OTHER, "{body}");

    cookie.expect("session")

}

async fn new_repo(app: &Shared, me: &str, fields: &str) -> (StatusCode, String) {

    let csrf = csrf_of(&send(app, "GET", "/new", Some(me), None).await.1);

    let (st, body, _) = send(app, "POST", "/new", Some(me), Some(&format!("_csrf={csrf}&owner=andy&{fields}"))).await;

    (st, body)

}

#[tokio::test(flavor = "multi_thread")]

async fn a_bucket_the_hub_must_not_use_is_refused_and_nothing_is_made() {

    let h = hub().await;

    let me = andy(&h.app).await;

    for (url, why) in [

        ("s3://b/p?profile=default", "?profile= is not allowed"),

        ("s3://b/p?endpoint=http%3A%2F%2F169.254.169.254", "?endpoint= is not allowed"),

        ("gs://b/p?credentials=%2Fetc%2Fkey.json", "?credentials= is not allowed"),

        ("dir:///var/lib/oxygenhub", "a store is s3://"),

        ("az://account", "names the account and the container"),

    ] {

        let (st, body) =

            new_repo(&h.app, &me, &format!("name=cloudy&storage=bucket&store_url={}&access_key_id=a&secret_access_key=b", enc(url))).await;

        assert_eq!(st, StatusCode::OK, "{url}");

        assert!(body.contains(why), "{url}: {body}");

    }

    let (_, body) = new_repo(&h.app, &me, &format!("name=cloudy&storage=bucket&store_url={}&access_key_id=a", enc("s3://b/p"))).await;

    assert!(body.contains("Credentials: credentials need secret_access_key"), "{body}");

    let name = RepoName::new("andy", "cloudy").unwrap();

    assert!(!h.app.repos.exists(&name));

    assert!(db::repos::get(&h.app.db, "andy", "cloudy").await.unwrap().is_none());

    // The hub's own disk stays the default.

    let (st, body) = new_repo(&h.app, &me, "name=local").await;

    assert_eq!(st, StatusCode::SEE_OTHER, "{body}");

    let page = send(&h.app, "GET", "/andy/local", Some(&me), None).await.1;

    assert!(!page.contains(">bucket</span>"));

}

/// The S3 test bucket's URL with only its region kept, and the profile's keys.

fn s3_test() -> Option<(String, String, String)> {

    let url = std::env::var("OXY_TEST_S3").ok().filter(|u| !u.is_empty())?;

    let profile = std::env::var("OXY_TEST_S3_PROFILE").ok().filter(|p| !p.is_empty())?;

    let (base, query) = url.split_once('?').unwrap_or((&url, ""));

    let region: Vec<&str> = query.split('&').filter(|p| p.starts_with("region=")).collect();

    let home = std::path::PathBuf::from(std::env::var("HOME").ok()?);

    let text = std::fs::read_to_string(home.join(".aws/credentials")).ok()?;

    let (mut section, mut id, mut secret) = (String::new(), None, None);

    for line in text.lines().map(str::trim) {

        if let Some(n) = line.strip_prefix('[').and_then(|l| l.strip_suffix(']')) {

            section = n.to_string();

        } else if let Some((k, v)) = line.split_once('=') {

            if section == profile {

                match k.trim() {

                    "aws_access_key_id" => id = Some(v.trim().to_string()),

                    "aws_secret_access_key" => secret = Some(v.trim().to_string()),

                    _ => {}

                }

            }

        }

    }

    let base = format!("{base}/hub-{}", std::process::id());

    let url = if region.is_empty() { base } else { format!("{base}?{}", region.join("&")) };

    Some((url, id?, secret?))

}

#[tokio::test(flavor = "multi_thread")]

async fn a_repository_in_a_real_bucket_takes_a_push_and_passes_its_check() {

    let Some((url, id, secret)) = s3_test() else {

        eprintln!("skipped: OXY_TEST_S3 and OXY_TEST_S3_PROFILE are not set");

        return;

    };

    let h = hub().await;

    let me = andy(&h.app).await;

    let fields = format!(

        "name=cloudy&visibility=private&storage=bucket&store_url={}&access_key_id={}&secret_access_key={}",

        enc(&url),

        enc(&id),

        enc(&secret)

    );

    let (st, body) = new_repo(&h.app, &me, &fields).await;

    let credentials = oxy_strata::Credentials::Aws { access_key_id: id, secret_access_key: secret, session_token: None };

    let store = oxy_strata::CloudStore::open_with(&url, Some(&credentials)).unwrap();

    let cleanup = || {

        use oxy_core::strata::Store as _;

        for e in store.list("").unwrap() {

            store.delete(&e.key).unwrap();

        }

    };

    if st != StatusCode::SEE_OTHER {

        cleanup();

        panic!("{body}");

    }

    let result = std::panic::AssertUnwindSafe(async {

        // A push from a client lands in the bucket.

        let work = tempfile::tempdir().unwrap();

        let mut c = Repo::init(work.path()).unwrap();

        c.config.set("remote.origin.url", &format!("{}/andy/cloudy", h.base)).unwrap();

        c.config.set("remote.origin.token", "site").unwrap();

        c.save_config().unwrap();

        std::fs::write(work.path().join("a.txt"), "kept in a bucket\n").unwrap();

        let head = c.commit("First", &[]).unwrap();

        let pushed = tokio::task::spawn_blocking(move || c.push("origin", false).unwrap()).await.unwrap();

        assert!(pushed.iter().all(|r| r.ok), "{pushed:?}");

        let hosted = h.app.repos.open(&RepoName::new("andy", "cloudy").unwrap()).unwrap();

        assert_eq!(hosted.bookmark_heads().unwrap(), vec![head]);

        {

            use oxy_core::strata::Store as _;

            assert!(store.read_ref("ops/head").unwrap().is_some(), "the operation log is in the bucket");

        }

        // The pages say where it lives; Settings → Storage checks the bucket.

        let page = send(&h.app, "GET", "/andy/cloudy", Some(&me), None).await.1;

        assert!(page.contains(">bucket</span>") && page.contains(url.split('?').next().unwrap()), "{page}");

        let settings = send(&h.app, "GET", "/andy/cloudy/settings", Some(&me), None).await.1;

        assert!(settings.contains("id=\"storage\""), "{settings}");

        let csrf = csrf_of(&settings);

        let (_, after, _) =

            send(&h.app, "POST", "/andy/cloudy/settings/storage", Some(&me), Some(&format!("_csrf={csrf}&action=check"))).await;

        assert!(after.contains("The bucket passed the check: 8 checks passed."), "{after}");

        // A second repository cannot start in the same place.

        let (st, body) = new_repo(&h.app, &me, &fields.replace("name=cloudy", "name=again")).await;

        assert_eq!(st, StatusCode::OK);

        assert!(body.contains("already holds"), "{body}");

    });

    let outcome = futures_catch(result).await;

    cleanup();

    if let Err(p) = outcome {

        std::panic::resume_unwind(p);

    }

}

async fn futures_catch<F: std::future::Future<Output = ()>>(f: std::panic::AssertUnwindSafe<F>) -> std::thread::Result<()> {

    use futures::FutureExt;

    f.catch_unwind().await

}

- **A repository can live in its owner's bucket (#96).** The New repository form has a Storage section: this server's disk, or your own Amazon S3, Google Cloud Storage or Azure bucket, with credentials for it. The bucket is checked before anything is made. The credentials are sealed on the server and used only for that repository. The hub keeps a cache in front. The header shows a *bucket* tag, and Settings › Storage checks the bucket again and replaces the credentials. Tested with a push into a real S3 bucket.

name = "android_system_properties"

version = "0.1.6"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"

dependencies = [

 "libc",

]

[[package]]

name = "chrono"

version = "0.4.45"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"

dependencies = [

 "iana-time-zone",

 "num-traits",

 "serde",

 "windows-link",

]

[[package]]

name = "crc-fast"

version = "1.10.0"

dependencies = [

 "digest 0.10.7",

 "spin 0.10.1",

]

[[package]]

 "spin 0.9.9",

name = "humantime"

version = "2.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"

[[package]]

]

[[package]]

name = "iana-time-zone"

version = "0.1.65"

dependencies = [

 "android_system_properties",

 "iana-time-zone-haiku",

 "js-sys",

 "log",

 "wasm-bindgen",

 "windows-core",

]

[[package]]

name = "iana-time-zone-haiku"

version = "0.1.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"

dependencies = [

 "cc",

name = "itertools"

version = "0.15.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"

dependencies = [

 "either",

]

[[package]]

 "spin 0.9.9",

]

[[package]]

name = "md-5"

version = "0.11.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"

dependencies = [

 "cfg-if",

 "digest 0.11.3",

 "spin 0.9.9",

]

[[package]]

name = "object_store"

version = "0.14.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94"

dependencies = [

 "async-trait",

 "base64 0.23.1",

 "bytes",

 "chrono",

 "crc-fast",

 "form_urlencoded",

 "futures-channel",

 "futures-core",

 "futures-util",

 "http",

 "http-body-util",

 "httparse",

 "humantime",

 "hyper",

 "itertools",

 "md-5 0.11.0",

 "parking_lot",

 "percent-encoding",

 "quick-xml 0.41.0",

 "rand 0.10.3",

 "ring",

 "rustls-pki-types",

 "serde",

 "serde_json",

 "serde_urlencoded",

 "thiserror 2.0.21",

 "tokio",

 "tracing",

 "url",

 "wasm-bindgen-futures",

 "web-time",

name = "oxy-strata"

version = "0.4.0"

dependencies = [

 "async-trait",

 "bytes",

 "futures",

 "http",

 "http-body-util",

 "object_store",

 "oxy-core",

 "rustls",

 "serde_json",

 "tokio",

 "ureq",

]

[[package]]

 "md-5 0.10.6",

 "oxy-strata",

 "quick-xml 0.42.0",

[[package]]

name = "quick-xml"

version = "0.41.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"

dependencies = [

 "memchr",

 "serde",

]

[[package]]

name = "spin"

version = "0.10.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"

 "md-5 0.10.6",

 "md-5 0.10.6",

name = "windows-core"

version = "0.62.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"

dependencies = [

 "windows-implement",

 "windows-interface",

 "windows-link",

 "windows-result",

 "windows-strings",

]

[[package]]

name = "windows-implement"

version = "0.60.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"

dependencies = [

 "proc-macro2",

 "quote",

 "syn 2.0.119",

]

[[package]]

name = "windows-interface"

version = "0.59.3"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"

dependencies = [

 "proc-macro2",

 "quote",

 "syn 2.0.119",

]

[[package]]

[[package]]

name = "windows-result"

version = "0.4.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"

dependencies = [

 "windows-link",

]

[[package]]

name = "windows-strings"

version = "0.5.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"

dependencies = [

 "windows-link",

]

oxy-strata = { path = "deps/oxygenflow/deps/oxygen/crates/oxy-strata" }

# The cloud stores' two patched dependencies, as oxygen patches them so

# the macOS cross build links (see their PATCHED.md in the pinned oxygen).

[patch.crates-io]

crc-fast = { path = "deps/oxygenflow/deps/oxygen/vendor/crc-fast" }

iana-time-zone = { path = "deps/oxygenflow/deps/oxygen/vendor/iana-time-zone" }

oxy-strata.workspace = true

pub mod storage;

    storage::load(&app).await?;

    /// Create an empty, unbound bare repository whose objects and

    /// operation log are kept in the store at `url` (oxygenhub#96); the

    /// store's credentials must already be known to the opener.

    pub fn create_on(&self, name: &RepoName, url: &str) -> Result<Repo, RepoError> {

        let dir = self.dir(name);

        if dir.exists() {

            return Err(RepoError::Exists(name.to_string()));

        }

        let made = (|| -> Result<Repo, RepoError> {

            drop(Repo::init_bare_on(&dir, [0; 16], Some(url))?);

            // Unbound as `create` makes them: oxy-core's marker file in

            // place of the id, so the first push gives the id.

            let oxy = dir.join(oxy_core::repo::OXY_DIR);

            std::fs::remove_file(oxy.join("id"))?;

            std::fs::write(oxy.join("unbound"), b"")?;

            Ok(Repo::open(&dir)?)

        })();

        if made.is_err() {

            let _ = std::fs::remove_dir_all(&dir);

        }

        made

    }

        }

        if Repo::open(&from).is_ok_and(|r| r.config.storage_url.is_some()) {

            return Err(RepoError::Io(std::io::Error::other(

                "a repository kept in a bucket cannot be forked yet: clone it and push the clone to a new repository",

            )));

        .route("/{owner}/{name}/settings/storage", post(repo_settings::storage))

    /// The bucket its data lives in, when not this hub's disk (#96).

    pub stored_in: Option<String>,

            stored_in: crate::storage::location(app, &row.owner, &row.name),

    /// `hub` (this server's disk, the default) or `bucket` (oxygenhub#96).

    #[serde(default)]

    storage: String,

    #[serde(flatten)]

    bucket: BucketForm,

}

/// A bucket and the credentials for it, as the New repository form and

/// Settings → Storage give them; only the kind the URL names is read.

#[derive(Deserialize, Default)]

pub struct BucketForm {

    #[serde(default)]

    store_url: String,

    #[serde(default)]

    access_key_id: String,

    #[serde(default)]

    secret_access_key: String,

    #[serde(default)]

    service_account_key: String,

    #[serde(default)]

    tenant_id: String,

    #[serde(default)]

    client_id: String,

    #[serde(default)]

    client_secret: String,

impl BucketForm {

    fn credentials(&self, scheme: &str) -> Result<oxy_strata::Credentials, String> {

        crate::storage::credentials_from(

            scheme,

            &self.access_key_id,

            &self.secret_access_key,

            &self.service_account_key,

            &self.tenant_id,

            &self.client_id,

            &self.client_secret,

        )

    }

}

    }

    if f.storage == "bucket" {

        // Kept in the owner's bucket (oxygenhub#96): checked before

        // anything is made, and nothing is left behind when it fails.

        let url = f.bucket.store_url.trim();

        let parsed = match crate::storage::check_url(url) {

            Ok(p) => p,

            Err(e) => return Ok(fail(&format!("Store: {e}."))),

        };

        let credentials = match f.bucket.credentials(&parsed.scheme) {

            Ok(c) => c,

            Err(e) => return Ok(fail(&format!("Credentials: {e}."))),

        };

        let row = db::repos::create(&app.db, &name.owner, &name.name, f.description.trim(), visibility)

            .await

            .map_err(|e| Error::from_db(shell.clone(), e))?;

        if let Err(e) = crate::storage::create(&app, &name, row.id, url, credentials).await {

            db::repos::delete(&app.db, row.id).await.ok();

            return Ok(fail(&format!("The bucket did not pass the check, so nothing was created: {e}")));

        }

        accounts::audit(

            &app.db,

            Some(&me.account),

            "repo.created",

            &name.to_string(),

            &format!("{} in {}", visibility.as_str(), parsed.plain()),

        )

        .await

        .ok();

        return Ok(Redirect::to(&name.url()).into_response());

    /// Where its data lives, when in a bucket (oxygenhub#96).

    storage: Option<crate::storage::Storage>,

    let storage = crate::storage::of(&app.db, row.id).await.map_err(|e| Error::from_db(shell.clone(), e))?;

        storage,

        crate::storage::renamed(&app, &repo, &dest);

    // A repository kept in a bucket leaves its data there: the owner's.

    crate::storage::removed(&app, &repo);

}

#[derive(Deserialize)]

pub struct StorageForm {

    #[serde(rename = "_csrf")]

    csrf_token: String,

    /// `check`, or `credentials` (with the new ones).

    action: String,

    #[serde(flatten)]

    bucket: BucketForm,

}

/// Settings → Storage (oxygenhub#96): check the bucket again, or replace

/// its credentials once the new ones pass the check.

pub async fn storage(

    State(app): State<Shared>,

    viewer: Viewer,

    Path((owner, name)): Path<(String, String)>,

    Form(f): Form<StorageForm>,

) -> Result<Page<RepoSettingsPage>, Error> {

    let shell = Shell::new(&app, &format!("/{owner}/{name}/settings"), &viewer);

    let (_, row, role) = lookup_role(&app, &shell, &viewer, &owner, &name, Access::Admin).await?;

    csrf(&viewer, &f.csrf_token, &shell)?;

    let Some(s) = crate::storage::of(&app.db, row.id).await.map_err(|e| Error::from_db(shell.clone(), e))? else {

        return settings_page(&app, shell, row, role, None, Some("This repository is kept on this server's disk.".into())).await;

    };

    let scheme = s.url.split("://").next().unwrap_or("").to_string();

    let outcome = match f.action.as_str() {

        "credentials" => match f.bucket.credentials(&scheme) {

            Ok(c) => crate::storage::set_credentials(&app, &s, c).await.map(|r| format!("New credentials saved: {r}.")),

            Err(e) => Err(format!("Credentials: {e}")),

        },

        _ => crate::storage::check(&app, &s).await.map(|r| format!("The bucket passed the check: {r}.")),

    };

    match outcome {

        Ok(notice) => {

            accounts::audit(&app.db, viewer.account(), "repo.storage", &format!("{owner}/{name}"), &f.action).await.ok();

            settings_page(&app, shell, row, role, Some(notice), None).await

        }

        Err(e) => settings_page(&app, shell, row, role, None, Some(format!("Storage: {e}"))).await,

    }

### 4.1a Repositories in the owner's bucket (oxygenhub#96)

A repository can keep its data in its owner's bucket instead of on the hub's disk.

- **Creating:** the New repository form's Storage section takes a store URL (`s3://BUCKET/PREFIX`, `gs://BUCKET/PREFIX` or `az://ACCOUNT/CONTAINER/PREFIX`, with `?region=` the only setting allowed) and the credentials for it: an S3 access key, a Google service account key, or an Azure service principal. The hub refuses a profile, a key file, an endpoint or `dir://`, which would make it use its own host's credentials, reach its own network or write to its own disk.

- **The check:** before anything is made, the bucket or prefix must be empty and pass the store conformance check under a probe prefix. Nothing is created when either fails.

- **What the hub keeps:** a bare repository directory whose `storage.url` names the store, with a cache in front (oxygen#68), and a `repo_storage` row holding the URL and the credentials, sealed under `keys/secrets.key`. At start the hub hands every repository's credentials to the cloud stores by URL, so nothing from its host's environment or cloud profiles is used for a repository.

- **Pages:** the header shows a *bucket* tag; its title names the URL to the repository's admins only. Settings › Storage shows the store and the last check, checks it again, and replaces the credentials once new ones pass the check. Both are audited as `repo.storage`.

- **Not yet:** forking such a repository is refused (clone and push instead). Deleting one leaves its data in the bucket, which is the owner's. An organisation storage policy, the Sync status and runs on references moved outside the hub are still to come.

  <label>Storage</label>

  <div class="choice">

    <label><input type="radio" name="storage" value="hub" checked> this server's disk</label>

    <label><input type="radio" name="storage" value="bucket"> my own bucket (Amazon S3, Google Cloud Storage or Azure)</label>

  </div>

  <fieldset class="storage-bucket">

    <legend>Your bucket <small class="mute">(only when kept in your own bucket)</small></legend>

    <p class="small mute">The repository's data lives in the bucket and this server keeps a cache. The bucket, or the prefix in it, must be empty. It is checked before anything is created. The credentials are sealed on this server and used only for this repository; give them read, write, list and delete under the prefix and nothing more.</p>

    <label for="store_url">Store URL</label><input type="text" id="store_url" name="store_url" placeholder="s3://my-bucket/repos/name?region=eu-west-2, gs://my-bucket/name, az://account/container/name" autocomplete="off">

    <p class="small mute">Amazon S3: an access key.</p>

    <div class="row">

      <div><label for="access_key_id">Access key ID</label><input type="text" id="access_key_id" name="access_key_id" autocomplete="off"></div>

      <div><label for="secret_access_key">Secret access key</label><input type="password" id="secret_access_key" name="secret_access_key" autocomplete="new-password"></div>

    </div>

    <p class="small mute">Google Cloud Storage: a service account key, the JSON file Google gives you.</p>

    <label for="service_account_key">Service account key</label><textarea id="service_account_key" name="service_account_key" rows="3" autocomplete="off" spellcheck="false"></textarea>

    <p class="small mute">Azure: a service principal with a role on the container.</p>

    <div class="row">

      <div><label for="tenant_id">Tenant ID</label><input type="text" id="tenant_id" name="tenant_id" autocomplete="off"></div>

      <div><label for="client_id">Client ID</label><input type="text" id="client_id" name="client_id" autocomplete="off"></div>

      <div><label for="client_secret">Client secret</label><input type="password" id="client_secret" name="client_secret" autocomplete="new-password"></div>

    </div>

  </fieldset>

  {% if storage.is_some() %}<a href="#storage">Storage</a>{% endif %}

{% if role == crate::auth::Role::Admin %}{% if let Some(st) = storage %}<section class="panel" id="storage">

  <h2>Storage</h2>

  <p>This repository's data lives in <code>{{ st.url }}</code>, your bucket. This server keeps a cache and the credentials, sealed.</p>

  <dl class="facts">

    <dt>Since</dt><dd><time title="{{ st.created_ms|when_i }}">{{ st.created_ms|ago_i }}</time></dd>

    <dt>Last check</dt><dd>{% if let Some(t) = st.checked_ms %}<time title="{{ t|when_i }}">{{ t|ago_i }}</time>: {{ st.check_result }}{% else %}<span class="mute">never</span>{% endif %}</dd>

  </dl>

  <form class="inline" method="post" action="{{ repo.url }}/settings/storage">

    <input type="hidden" name="_csrf" value="{{ shell.csrf }}">

    <input type="hidden" name="action" value="check">

    <button type="submit" title="Write, read, list and delete under a probe prefix of the bucket, and compare-and-swap a record">CHECK THE BUCKET</button>

  </form>

  <details>

    <summary>Replace the credentials</summary>

    <form class="card" method="post" action="{{ repo.url }}/settings/storage">

      <input type="hidden" name="_csrf" value="{{ shell.csrf }}">

      <input type="hidden" name="action" value="credentials">

      <p class="small mute">The new credentials must pass the check before they replace the old ones. Fill the fields for your provider.</p>

      {% if st.url.starts_with("s3://") %}<div class="row">

        <div><label for="sak">Access key ID</label><input type="text" id="sak" name="access_key_id" autocomplete="off" required></div>

        <div><label for="ssk">Secret access key</label><input type="password" id="ssk" name="secret_access_key" autocomplete="new-password" required></div>

      </div>{% else if st.url.starts_with("gs://") %}<label for="ssa">Service account key</label><textarea id="ssa" name="service_account_key" rows="3" required spellcheck="false"></textarea>{% else %}<div class="row">

        <div><label for="stn">Tenant ID</label><input type="text" id="stn" name="tenant_id" autocomplete="off" required></div>

        <div><label for="sci">Client ID</label><input type="text" id="sci" name="client_id" autocomplete="off" required></div>

        <div><label for="scs">Client secret</label><input type="password" id="scs" name="client_secret" autocomplete="new-password" required></div>

      </div>{% endif %}

      <button type="submit">SAVE</button>

    </form>

  </details>

</section>{% endif %}{% endif %}

    <span class="crumb"><a href="/{{ repo.owner }}">{{ repo.owner }}</a> / <a href="{{ repo.url }}"><strong>{{ repo.name }}</strong></a>{% if repo.archived %} <span class="tag warn" title="This repository is archived: read-only">archived</span>{% endif %} <span class="vis vis-{{ repo.visibility }}" title="{% if repo.visibility == "public" %}Anyone can see this repository{% else if repo.visibility == "internal" %}Anyone signed in to this hub can see this repository{% else %}Only people given access can see this repository{% endif %}"><span aria-hidden="true">{% if repo.visibility == "public" %}◯{% else if repo.visibility == "internal" %}◐{% else %}●{% endif %}</span> {{ repo.visibility }}</span>{% if let Some(b) = repo.stored_in %} <span class="tag" title="This repository's data lives in {% if repo.can(crate::auth::Access::Admin) %}{{ b }}, {% endif %}its owner's bucket">bucket</span>{% endif %}</span>