OXYGENHUB

Build against oxygenflow's pinned oxygen (deps/oxygenflow/deps/oxygen) instead of a second pin, so there is one oxy-core; OxygenFlow claims follow subrepos of subrepos, parents first, up to three deep 25091e5c3bba

refs #19
Author
Andy McCall @andymccall <[email protected]>
When
· 2026-09-28 19:20 UTC
Signature
✔ signed by Andy McCall
Hash
25091e5c3bba728b43c3f8214df1b0d811896789b7acbf5ca9c271a8347837d8
Depends on
370035a46397 380996db84e5 42fa95540468 d23464c0ac60 fd3db644df22
Depended on by
5d54cf746e00 180dc43ad0e0 f677d6270298 1f78655a0466 4d2e44d79b03 afd0be382257 046bd71a180f 5c6e92fd8d71 76bc5a058351
In bookmarks
api-unknown-fields cancel-landed-runs clear-waiting close-proposal-140 diffusion-pin-bookmark draft-ready image-check job-waits main oh-comment-body-file proposal-list-fast run-one-job wiki-says-so
In tags
v0.1.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.5.0 v0.5.1 v0.5.2 v0.6.0 v0.6.1 v0.7.0 v0.8.0 v0.9.0 v0.9.1 v0.9.11 v0.9.15 v0.9.16 v0.9.19 v0.9.2 v0.9.20 v0.9.21 v0.9.22 v0.9.23 v0.9.24 v0.9.3 v0.9.4 v0.9.5 v0.9.6 v0.9.7 v0.9.8 v0.9.9
Edits
21 atoms, 1 tree operation, 8 files

Browse the tree at this change

removed deps/oxygen +0 −0

No line changes.

modified .oxymodules +0 −4
1 -
[deps/oxygen]
2 -
url = https://earth.andymccall.xyz:7343/andymccall/oxygen
3 -
bookmark = main
4 -
5 1
[deps/oxygenflow]
6 2
url = https://earth.andymccall.xyz:7343/andymccall/oxygenflow
7 3
bookmark = main
…
1 unchanged lines
modified Cargo.toml +5 −4
1 1
[workspace]
2 2
members = ["crates/hub", "crates/hub-api"]
3 -
# Pinned subrepos (oxy submodule): oxygen and oxygenflow at the states this
4 -
# hub builds against. They are workspaces of their own.
3 +
# oxygenflow as a pinned subrepo (oxy submodule), and oxygen as oxygenflow's
4 +
# own pinned subrepo inside it: one oxygen, the one oxygenflow was tested
5 +
# with. They are workspaces of their own.
5 6
exclude = ["deps"]
6 7
resolver = "2"
7 8
…
5 unchanged lines
13 14
repository = "https://earth.andymccall.xyz:7343/andymccall/oxygenhub"
14 15
15 16
[workspace.dependencies]
16 -
oxy-core = { path = "deps/oxygen/crates/oxy-core" }
17 -
oxy-server = { path = "deps/oxygen/crates/oxy-server" }
17 +
oxy-core = { path = "deps/oxygenflow/deps/oxygen/crates/oxy-core" }
18 +
oxy-server = { path = "deps/oxygenflow/deps/oxygen/crates/oxy-server" }
18 19
hub-api = { path = "crates/hub-api" }
19 20
flow-core = { path = "deps/oxygenflow/crates/flow-core" }
20 21
…
44 unchanged lines
modified Makefile +4 −3
…
25 unchanged lines
26 26
	rsync -a --delete --exclude target --exclude node_modules --exclude .oxy/objects/loose ../oxygen ../oxygenflow ../oxygenhub claude@earth:src/
27 27
	ssh claude@earth 'source ~/.cargo/env && cd src/oxygenhub && cargo build --release && sudo install -m 0755 target/release/oxygenhub /usr/local/bin/oxygenhub && sudo systemctl restart oxygenhub && systemctl is-active oxygenhub'
28 28
29 -
# Move the pinned subrepos (deps/oxygen, deps/oxygenflow) to their bookmark on
30 -
# Earth; the next commit records the new pins. Push oxygen and oxygenflow first.
29 +
# Move the pinned oxygenflow (deps/oxygenflow) to its main on Earth, with the
30 +
# oxygen it pins; the next commit records the new pin. Push oxygen, then
31 +
# oxygenflow (with `make pins` there), first.
31 32
.PHONY: pins
32 33
pins:
33 -
	for d in deps/oxygen deps/oxygenflow; do (cd $$d && OXY_CA=$(HOME)/.config/oxy/earth-ca.pem oxy --plain pull origin) || exit 1; done
34 +
	cd deps/oxygenflow && OXY_CA=$(HOME)/.config/oxy/earth-ca.pem oxy --plain pull origin && OXY_CA=$(HOME)/.config/oxy/earth-ca.pem oxy --plain submodule update --init
modified README.md +6 −2
…
63 unchanged lines
64 64
The tests use a real `oxy-core` client against a hub on a loopback port and
65 65
never touch your own Oxygen configuration.
66 66
67 -
## oxygen and oxygenflow as pinned subrepos
67 +
## oxygenflow and oxygen as pinned subrepos
68 +
69 +
The hub builds against `deps/oxygenflow`, a subrepo (`oxy submodule`) pinned to an exact state, and against the oxygen that oxygenflow pins in turn, `deps/oxygenflow/deps/oxygen`. So there is one oxygen, the one oxygenflow was built and tested with, every build says which it used, and OxygenFlow can build the hub without network.
68 70
69 -
The hub builds against `deps/oxygen` and `deps/oxygenflow`, subrepos (`oxy submodule`) pinned to exact states, so every build says which oxygen it used and OxygenFlow can build it without network. `make pins` moves both to their `main` on Earth (push those repositories first); the next commit records the new pins. Format only this workspace, `cargo fmt -p oxygenhub -p hub-api`: `--all` also reaches the path dependencies in `deps/`, which have their own formatting. After a fresh clone, `oxy submodule update --init` fetches them.
71 +
- **Newer code:** push oxygen, run `make pins` in oxygenflow and push it, then run `make pins` here. The next commit records the new pin.
72 +
- **Formatting:** format only this workspace, with `cargo fmt -p oxygenhub -p hub-api`. `--all` also reaches the path dependencies in `deps/`, which have their own formatting.
73 +
- **Fresh clone:** `oxy submodule update --init` here and in `deps/oxygenflow` fetches them.
modified crates/hub/src/flow.rs +50 −31
…
158 unchanged lines
159 159
    Ok(Json(claim).into_response())
160 160
}
161 161
162 -
/// The Oxygen subrepos the run's state pins that live on this hub and that
163 -
/// its pusher may read: what the job's workspace needs beside its own tree.
164 -
/// A pin the hub cannot serve is left out; the job then fails on the missing
165 -
/// files with an ordinary error.
166 -
async fn subrepo_inputs(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>) {
167 -
    let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![]) };
168 -
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
169 -
    let pins = app
170 -
        .repos
162 +
/// The Oxygen subrepo pins in `name`'s state `heads`: (path, repository id, pinned heads).
163 +
async fn pins_in(
164 +
    app: &Shared,
165 +
    name: crate::repos::RepoName,
166 +
    heads: Vec<oxy_core::hash::ChangeHash>,
167 +
) -> Vec<(String, [u8; 16], Vec<oxy_core::hash::ChangeHash>)> {
168 +
    app.repos
171 169
        .read(&name, move |repo, cache| {
172 170
            let loaded = cache.load(repo, &heads)?;
173 171
            let mut out = Vec::new();
…
16 unchanged lines
190 188
            Ok(out)
191 189
        })
192 190
        .await
193 -
        .unwrap_or_default();
194 -
    if pins.is_empty() {
195 -
        return (vec![], vec![]);
196 -
    }
191 +
        .unwrap_or_default()
192 +
}
193 +
194 +
/// How deep subrepos of subrepos are followed.
195 +
const MAX_SUBREPO_DEPTH: usize = 3;
196 +
197 +
/// The Oxygen subrepos the run's state pins, and those their states pin in
198 +
/// turn (parents first, paths joined), that live on this hub and that the
199 +
/// run's pusher may read: what the job's workspace needs beside its own
200 +
/// tree. A pin the hub cannot serve is left out, with what it pins; the job
201 +
/// then fails on the missing files with an ordinary error.
202 +
async fn subrepo_inputs(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>) {
203 +
    let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![]) };
204 +
    let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect();
197 205
    let pusher = db::accounts::account_by_name(&app.db, &run.actor).await.ok().flatten();
206 +
    let rows = db::repos::list(&app.db).await.unwrap_or_default();
198 207
    let mut inputs = Vec::new();
199 208
    let mut ids = Vec::new();
200 -
    let rows = db::repos::list(&app.db).await.unwrap_or_default();
201 -
    for (path, id, heads) in pins {
202 -
        for other in &rows {
203 -
            let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue };
204 -
            let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false);
205 -
            if !matches {
206 -
                continue;
207 -
            }
208 -
            let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten();
209 -
            if crate::auth::roles::allows(role, crate::auth::Access::Read) {
210 -
                inputs.push(flow_core::api::SubrepoInput {
211 -
                    path: path.clone(),
212 -
                    repo: format!("{}/{}", other.owner, other.name),
213 -
                    clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name),
214 -
                    heads: heads.iter().map(|h| h.to_hex()).collect(),
215 -
                });
216 -
                ids.push(other.id);
209 +
    // (path prefix, repository, heads, depth), parents before children.
210 +
    let mut queue = std::collections::VecDeque::from([(String::new(), name, heads, 0usize)]);
211 +
    while let Some((prefix, name, heads, depth)) = queue.pop_front() {
212 +
        if depth == MAX_SUBREPO_DEPTH {
213 +
            continue;
214 +
        }
215 +
        for (path, id, pinned) in pins_in(app, name, heads).await {
216 +
            let path = if prefix.is_empty() { path } else { format!("{prefix}/{path}") };
217 +
            for other in &rows {
218 +
                let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue };
219 +
                let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false);
220 +
                if !matches {
221 +
                    continue;
222 +
                }
223 +
                let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten();
224 +
                if crate::auth::roles::allows(role, crate::auth::Access::Read) {
225 +
                    inputs.push(flow_core::api::SubrepoInput {
226 +
                        path: path.clone(),
227 +
                        repo: format!("{}/{}", other.owner, other.name),
228 +
                        clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name),
229 +
                        heads: pinned.iter().map(|h| h.to_hex()).collect(),
230 +
                    });
231 +
                    if !ids.contains(&other.id) {
232 +
                        ids.push(other.id);
233 +
                    }
234 +
                    queue.push_back((path.clone(), other_name, pinned.clone(), depth + 1));
235 +
                }
236 +
                break;
217 237
            }
218 -
            break;
219 238
        }
220 239
    }
221 240
    (inputs, ids)
…
101 unchanged lines
modified crates/hub/tests/flow.rs +15 −4
…
344 unchanged lines
345 345
#[tokio::test(flavor = "multi_thread")]
346 346
async fn claims_carry_pinned_subrepos_and_the_job_token_reads_only_those() {
347 347
    let h = hub().await;
348 -
    for n in ["lib", "vault", "other", "app"] {
348 +
    for n in ["base", "lib", "vault", "other", "app"] {
349 349
        create(&h.app, "andy", n).await;
350 350
    }
351 351
    let push = |name: &str, file: &str| {
…
6 unchanged lines
358 358
        let heads = c.bookmark_heads().unwrap();
359 359
        (work, heads)
360 360
    };
361 -
    let (_l, lib_heads) = push("lib", "lib.rs");
361 +
    let (_b, base_heads) = push("base", "base.rs");
362 +
    // lib pins base in turn: a subrepo of a subrepo.
363 +
    let lib_work = tempfile::tempdir().unwrap();
364 +
    let lib = client(&format!("{}/andy/lib", h.base), lib_work.path());
365 +
    lib.submodule_add(&format!("{}/andy/base", h.base), "deps/base", Some("main")).unwrap();
366 +
    write(lib_work.path(), "lib.rs", "x\n");
367 +
    lib.snapshot("t").unwrap();
368 +
    lib.commit("lib on base", &[]).unwrap();
369 +
    assert!(lib.push("origin", false).unwrap().iter().all(|r| r.ok));
370 +
    let lib_heads = lib.bookmark_heads().unwrap();
362 371
    let (_v, _) = push("vault", "secret.rs");
363 372
    let (_o, _) = push("other", "o.rs");
364 373
…
17 unchanged lines
382 391
383 392
    let job = claim(&h.app).await.unwrap();
384 393
    let subs = job["subrepos"].as_array().cloned().unwrap_or_default();
385 -
    assert_eq!(subs.len(), 1, "vault is private and the pusher has no account: {subs:?}");
394 +
    assert_eq!(subs.len(), 2, "lib and what lib pins; vault is private and the pusher has no account: {subs:?}");
395 +
    assert_eq!(subs[1]["path"], "deps/lib/deps/base", "nested under its parent, after it");
396 +
    assert_eq!(subs[1]["heads"], serde_json::json!(base_heads.iter().map(|h| h.to_hex()).collect::<Vec<_>>()));
386 397
    let _ = std::fs::remove_dir_all(work.path().join("deps/vault2"));
387 398
    assert_eq!(subs[0]["path"], "deps/lib");
388 399
    assert_eq!(subs[0]["repo"], "andy/lib");
…
6 unchanged lines
395 406
        let opts = RemoteOptions { token: Some(token.clone()), ..Default::default() };
396 407
        open_remote(&format!("{}/andy/{name}", h.base), &opts).and_then(|r| r.refs()).is_ok()
397 408
    };
398 -
    assert!(read("app") && read("lib"));
409 +
    assert!(read("app") && read("lib") && read("base"));
399 410
    assert!(!read("vault"), "not pinned for this pusher");
400 411
    assert!(!read("other"), "public, but not the job's to read");
401 412
}
replaced deps/oxygenflow +0 −0

No line changes.