Build against oxygenflow's pinned oxygen (deps/oxygenflow/deps/oxygen) instead of a second pin, so there is one oxy-core; OxygenFlow claims follow subrepos of subrepos, parents first, up to three deep 25091e5c3bba
refs #19
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-28 19:20 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
25091e5c3bba728b43c3f8214df1b0d811896789b7acbf5ca9c271a8347837d8- Depends on
- 370035a46397 380996db84e5 42fa95540468 d23464c0ac60 fd3db644df22
- Depended on by
- 5d54cf746e00 180dc43ad0e0 f677d6270298 1f78655a0466 4d2e44d79b03 afd0be382257 046bd71a180f 5c6e92fd8d71 76bc5a058351
- In bookmarks
- api-unknown-fields cancel-landed-runs clear-waiting close-proposal-140 diffusion-pin-bookmark draft-ready image-check job-waits main oh-comment-body-file proposal-list-fast run-one-job wiki-says-so
- In tags
- v0.1.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.5.0 v0.5.1 v0.5.2 v0.6.0 v0.6.1 v0.7.0 v0.8.0 v0.9.0 v0.9.1 v0.9.11 v0.9.15 v0.9.16 v0.9.19 v0.9.2 v0.9.20 v0.9.21 v0.9.22 v0.9.23 v0.9.24 v0.9.3 v0.9.4 v0.9.5 v0.9.6 v0.9.7 v0.9.8 v0.9.9
- Edits
- 21 atoms, 1 tree operation, 8 files
Browse the tree at this change
removed deps/oxygen +0 −0
No line changes.
modified .oxymodules +0 −4
| 1 | - | [deps/oxygen] |
|
| 2 | - | url = https://earth.andymccall.xyz:7343/andymccall/oxygen |
|
| 3 | - | bookmark = main |
|
| 4 | - | ||
| 5 | 1 | [deps/oxygenflow] |
|
| 6 | 2 | url = https://earth.andymccall.xyz:7343/andymccall/oxygenflow |
|
| 7 | 3 | bookmark = main |
|
| … | 1 unchanged lines |
modified Cargo.toml +5 −4
| 1 | 1 | [workspace] |
|
| 2 | 2 | members = ["crates/hub", "crates/hub-api"] |
|
| 3 | - | # Pinned subrepos (oxy submodule): oxygen and oxygenflow at the states this |
|
| 4 | - | # hub builds against. They are workspaces of their own. |
|
| 3 | + | # oxygenflow as a pinned subrepo (oxy submodule), and oxygen as oxygenflow's |
|
| 4 | + | # own pinned subrepo inside it: one oxygen, the one oxygenflow was tested |
|
| 5 | + | # with. They are workspaces of their own. |
|
| 5 | 6 | exclude = ["deps"] |
|
| 6 | 7 | resolver = "2" |
|
| 7 | 8 | ||
| … | 5 unchanged lines |
||
| 13 | 14 | repository = "https://earth.andymccall.xyz:7343/andymccall/oxygenhub" |
|
| 14 | 15 | ||
| 15 | 16 | [workspace.dependencies] |
|
| 16 | - | oxy-core = { path = "deps/oxygen/crates/oxy-core" } |
|
| 17 | - | oxy-server = { path = "deps/oxygen/crates/oxy-server" } |
|
| 17 | + | oxy-core = { path = "deps/oxygenflow/deps/oxygen/crates/oxy-core" } |
|
| 18 | + | oxy-server = { path = "deps/oxygenflow/deps/oxygen/crates/oxy-server" } |
|
| 18 | 19 | hub-api = { path = "crates/hub-api" } |
|
| 19 | 20 | flow-core = { path = "deps/oxygenflow/crates/flow-core" } |
|
| 20 | 21 | ||
| … | 44 unchanged lines |
modified Makefile +4 −3
| … | 25 unchanged lines |
||
| 26 | 26 | rsync -a --delete --exclude target --exclude node_modules --exclude .oxy/objects/loose ../oxygen ../oxygenflow ../oxygenhub claude@earth:src/ |
|
| 27 | 27 | ssh claude@earth 'source ~/.cargo/env && cd src/oxygenhub && cargo build --release && sudo install -m 0755 target/release/oxygenhub /usr/local/bin/oxygenhub && sudo systemctl restart oxygenhub && systemctl is-active oxygenhub' |
|
| 28 | 28 | ||
| 29 | - | # Move the pinned subrepos (deps/oxygen, deps/oxygenflow) to their bookmark on |
|
| 30 | - | # Earth; the next commit records the new pins. Push oxygen and oxygenflow first. |
|
| 29 | + | # Move the pinned oxygenflow (deps/oxygenflow) to its main on Earth, with the |
|
| 30 | + | # oxygen it pins; the next commit records the new pin. Push oxygen, then |
|
| 31 | + | # oxygenflow (with `make pins` there), first. |
|
| 31 | 32 | .PHONY: pins |
|
| 32 | 33 | pins: |
|
| 33 | - | for d in deps/oxygen deps/oxygenflow; do (cd $$d && OXY_CA=$(HOME)/.config/oxy/earth-ca.pem oxy --plain pull origin) || exit 1; done |
|
| 34 | + | cd deps/oxygenflow && OXY_CA=$(HOME)/.config/oxy/earth-ca.pem oxy --plain pull origin && OXY_CA=$(HOME)/.config/oxy/earth-ca.pem oxy --plain submodule update --init |
modified README.md +6 −2
| … | 63 unchanged lines |
||
| 64 | 64 | The tests use a real `oxy-core` client against a hub on a loopback port and |
|
| 65 | 65 | never touch your own Oxygen configuration. |
|
| 66 | 66 | ||
| 67 | - | ## oxygen and oxygenflow as pinned subrepos |
|
| 67 | + | ## oxygenflow and oxygen as pinned subrepos |
|
| 68 | + | ||
| 69 | + | The hub builds against `deps/oxygenflow`, a subrepo (`oxy submodule`) pinned to an exact state, and against the oxygen that oxygenflow pins in turn, `deps/oxygenflow/deps/oxygen`. So there is one oxygen, the one oxygenflow was built and tested with, every build says which it used, and OxygenFlow can build the hub without network. |
|
| 68 | 70 | ||
| 69 | - | The hub builds against `deps/oxygen` and `deps/oxygenflow`, subrepos (`oxy submodule`) pinned to exact states, so every build says which oxygen it used and OxygenFlow can build it without network. `make pins` moves both to their `main` on Earth (push those repositories first); the next commit records the new pins. Format only this workspace, `cargo fmt -p oxygenhub -p hub-api`: `--all` also reaches the path dependencies in `deps/`, which have their own formatting. After a fresh clone, `oxy submodule update --init` fetches them. |
|
| 71 | + | - **Newer code:** push oxygen, run `make pins` in oxygenflow and push it, then run `make pins` here. The next commit records the new pin. |
|
| 72 | + | - **Formatting:** format only this workspace, with `cargo fmt -p oxygenhub -p hub-api`. `--all` also reaches the path dependencies in `deps/`, which have their own formatting. |
|
| 73 | + | - **Fresh clone:** `oxy submodule update --init` here and in `deps/oxygenflow` fetches them. |
modified crates/hub/src/flow.rs +50 −31
| … | 158 unchanged lines |
||
| 159 | 159 | Ok(Json(claim).into_response()) |
|
| 160 | 160 | } |
|
| 161 | 161 | ||
| 162 | - | /// The Oxygen subrepos the run's state pins that live on this hub and that |
|
| 163 | - | /// its pusher may read: what the job's workspace needs beside its own tree. |
|
| 164 | - | /// A pin the hub cannot serve is left out; the job then fails on the missing |
|
| 165 | - | /// files with an ordinary error. |
|
| 166 | - | async fn subrepo_inputs(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>) { |
|
| 167 | - | let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![]) }; |
|
| 168 | - | let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect(); |
|
| 169 | - | let pins = app |
|
| 170 | - | .repos |
|
| 162 | + | /// The Oxygen subrepo pins in `name`'s state `heads`: (path, repository id, pinned heads). |
|
| 163 | + | async fn pins_in( |
|
| 164 | + | app: &Shared, |
|
| 165 | + | name: crate::repos::RepoName, |
|
| 166 | + | heads: Vec<oxy_core::hash::ChangeHash>, |
|
| 167 | + | ) -> Vec<(String, [u8; 16], Vec<oxy_core::hash::ChangeHash>)> { |
|
| 168 | + | app.repos |
|
| 171 | 169 | .read(&name, move |repo, cache| { |
|
| 172 | 170 | let loaded = cache.load(repo, &heads)?; |
|
| 173 | 171 | let mut out = Vec::new(); |
|
| … | 16 unchanged lines |
||
| 190 | 188 | Ok(out) |
|
| 191 | 189 | }) |
|
| 192 | 190 | .await |
|
| 193 | - | .unwrap_or_default(); |
|
| 194 | - | if pins.is_empty() { |
|
| 195 | - | return (vec![], vec![]); |
|
| 196 | - | } |
|
| 191 | + | .unwrap_or_default() |
|
| 192 | + | } |
|
| 193 | + | ||
| 194 | + | /// How deep subrepos of subrepos are followed. |
|
| 195 | + | const MAX_SUBREPO_DEPTH: usize = 3; |
|
| 196 | + | ||
| 197 | + | /// The Oxygen subrepos the run's state pins, and those their states pin in |
|
| 198 | + | /// turn (parents first, paths joined), that live on this hub and that the |
|
| 199 | + | /// run's pusher may read: what the job's workspace needs beside its own |
|
| 200 | + | /// tree. A pin the hub cannot serve is left out, with what it pins; the job |
|
| 201 | + | /// then fails on the missing files with an ordinary error. |
|
| 202 | + | async fn subrepo_inputs(app: &Shared, row: &db::repos::RepoRow, run: &fdb::Run) -> (Vec<flow_core::api::SubrepoInput>, Vec<i64>) { |
|
| 203 | + | let Some(name) = crate::repos::RepoName::new(&row.owner, &row.name) else { return (vec![], vec![]) }; |
|
| 204 | + | let heads: Vec<oxy_core::hash::ChangeHash> = run.head_list().into_iter().filter_map(oxy_core::hash::ChangeHash::from_hex).collect(); |
|
| 197 | 205 | let pusher = db::accounts::account_by_name(&app.db, &run.actor).await.ok().flatten(); |
|
| 206 | + | let rows = db::repos::list(&app.db).await.unwrap_or_default(); |
|
| 198 | 207 | let mut inputs = Vec::new(); |
|
| 199 | 208 | let mut ids = Vec::new(); |
|
| 200 | - | let rows = db::repos::list(&app.db).await.unwrap_or_default(); |
|
| 201 | - | for (path, id, heads) in pins { |
|
| 202 | - | for other in &rows { |
|
| 203 | - | let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue }; |
|
| 204 | - | let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false); |
|
| 205 | - | if !matches { |
|
| 206 | - | continue; |
|
| 207 | - | } |
|
| 208 | - | let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten(); |
|
| 209 | - | if crate::auth::roles::allows(role, crate::auth::Access::Read) { |
|
| 210 | - | inputs.push(flow_core::api::SubrepoInput { |
|
| 211 | - | path: path.clone(), |
|
| 212 | - | repo: format!("{}/{}", other.owner, other.name), |
|
| 213 | - | clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name), |
|
| 214 | - | heads: heads.iter().map(|h| h.to_hex()).collect(), |
|
| 215 | - | }); |
|
| 216 | - | ids.push(other.id); |
|
| 209 | + | // (path prefix, repository, heads, depth), parents before children. |
|
| 210 | + | let mut queue = std::collections::VecDeque::from([(String::new(), name, heads, 0usize)]); |
|
| 211 | + | while let Some((prefix, name, heads, depth)) = queue.pop_front() { |
|
| 212 | + | if depth == MAX_SUBREPO_DEPTH { |
|
| 213 | + | continue; |
|
| 214 | + | } |
|
| 215 | + | for (path, id, pinned) in pins_in(app, name, heads).await { |
|
| 216 | + | let path = if prefix.is_empty() { path } else { format!("{prefix}/{path}") }; |
|
| 217 | + | for other in &rows { |
|
| 218 | + | let Some(other_name) = crate::repos::RepoName::new(&other.owner, &other.name) else { continue }; |
|
| 219 | + | let matches = app.repos.read(&other_name, move |r, _| Ok(r.id().ok() == Some(id))).await.unwrap_or(false); |
|
| 220 | + | if !matches { |
|
| 221 | + | continue; |
|
| 222 | + | } |
|
| 223 | + | let role = crate::auth::roles::role_for(&app.db, pusher.as_ref(), other).await.ok().flatten(); |
|
| 224 | + | if crate::auth::roles::allows(role, crate::auth::Access::Read) { |
|
| 225 | + | inputs.push(flow_core::api::SubrepoInput { |
|
| 226 | + | path: path.clone(), |
|
| 227 | + | repo: format!("{}/{}", other.owner, other.name), |
|
| 228 | + | clone_url: format!("{}/{}/{}", app.config.base_url(), other.owner, other.name), |
|
| 229 | + | heads: pinned.iter().map(|h| h.to_hex()).collect(), |
|
| 230 | + | }); |
|
| 231 | + | if !ids.contains(&other.id) { |
|
| 232 | + | ids.push(other.id); |
|
| 233 | + | } |
|
| 234 | + | queue.push_back((path.clone(), other_name, pinned.clone(), depth + 1)); |
|
| 235 | + | } |
|
| 236 | + | break; |
|
| 217 | 237 | } |
|
| 218 | - | break; |
|
| 219 | 238 | } |
|
| 220 | 239 | } |
|
| 221 | 240 | (inputs, ids) |
|
| … | 101 unchanged lines |
modified crates/hub/tests/flow.rs +15 −4
| … | 344 unchanged lines |
||
| 345 | 345 | #[tokio::test(flavor = "multi_thread")] |
|
| 346 | 346 | async fn claims_carry_pinned_subrepos_and_the_job_token_reads_only_those() { |
|
| 347 | 347 | let h = hub().await; |
|
| 348 | - | for n in ["lib", "vault", "other", "app"] { |
|
| 348 | + | for n in ["base", "lib", "vault", "other", "app"] { |
|
| 349 | 349 | create(&h.app, "andy", n).await; |
|
| 350 | 350 | } |
|
| 351 | 351 | let push = |name: &str, file: &str| { |
|
| … | 6 unchanged lines |
||
| 358 | 358 | let heads = c.bookmark_heads().unwrap(); |
|
| 359 | 359 | (work, heads) |
|
| 360 | 360 | }; |
|
| 361 | - | let (_l, lib_heads) = push("lib", "lib.rs"); |
|
| 361 | + | let (_b, base_heads) = push("base", "base.rs"); |
|
| 362 | + | // lib pins base in turn: a subrepo of a subrepo. |
|
| 363 | + | let lib_work = tempfile::tempdir().unwrap(); |
|
| 364 | + | let lib = client(&format!("{}/andy/lib", h.base), lib_work.path()); |
|
| 365 | + | lib.submodule_add(&format!("{}/andy/base", h.base), "deps/base", Some("main")).unwrap(); |
|
| 366 | + | write(lib_work.path(), "lib.rs", "x\n"); |
|
| 367 | + | lib.snapshot("t").unwrap(); |
|
| 368 | + | lib.commit("lib on base", &[]).unwrap(); |
|
| 369 | + | assert!(lib.push("origin", false).unwrap().iter().all(|r| r.ok)); |
|
| 370 | + | let lib_heads = lib.bookmark_heads().unwrap(); |
|
| 362 | 371 | let (_v, _) = push("vault", "secret.rs"); |
|
| 363 | 372 | let (_o, _) = push("other", "o.rs"); |
|
| 364 | 373 | ||
| … | 17 unchanged lines |
||
| 382 | 391 | ||
| 383 | 392 | let job = claim(&h.app).await.unwrap(); |
|
| 384 | 393 | let subs = job["subrepos"].as_array().cloned().unwrap_or_default(); |
|
| 385 | - | assert_eq!(subs.len(), 1, "vault is private and the pusher has no account: {subs:?}"); |
|
| 394 | + | assert_eq!(subs.len(), 2, "lib and what lib pins; vault is private and the pusher has no account: {subs:?}"); |
|
| 395 | + | assert_eq!(subs[1]["path"], "deps/lib/deps/base", "nested under its parent, after it"); |
|
| 396 | + | assert_eq!(subs[1]["heads"], serde_json::json!(base_heads.iter().map(|h| h.to_hex()).collect::<Vec<_>>())); |
|
| 386 | 397 | let _ = std::fs::remove_dir_all(work.path().join("deps/vault2")); |
|
| 387 | 398 | assert_eq!(subs[0]["path"], "deps/lib"); |
|
| 388 | 399 | assert_eq!(subs[0]["repo"], "andy/lib"); |
|
| … | 6 unchanged lines |
||
| 395 | 406 | let opts = RemoteOptions { token: Some(token.clone()), ..Default::default() }; |
|
| 396 | 407 | open_remote(&format!("{}/andy/{name}", h.base), &opts).and_then(|r| r.refs()).is_ok() |
|
| 397 | 408 | }; |
|
| 398 | - | assert!(read("app") && read("lib")); |
|
| 409 | + | assert!(read("app") && read("lib") && read("base")); |
|
| 399 | 410 | assert!(!read("vault"), "not pinned for this pusher"); |
|
| 400 | 411 | assert!(!read("other"), "public, but not the job's to read"); |
|
| 401 | 412 | } |
replaced deps/oxygenflow +0 −0
No line changes.