OXYGENHUB

Pinned subrepos in job workspaces: the claim lists them, the runner fetches each with the job token and exports its pinned state at its path, inside the scope hash c23d519168d8

closes #4
Author
Andy McCall @andymccall <[email protected]>
When
· 2026-09-28 18:40 UTC
Signature
✔ signed by Andy McCall
Hash
c23d519168d8a746cbf1851e8789b5a1475ba1ed34f170f84ef3704f8774b42a
Depends on
e0918b887850
Depended on by
ed4aef8b70ca f53e6d47b56a e2c125610f43 c4285535b03a
In bookmarks
main newer-pipeline push-uses-login
In tags
v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.5.0 v0.5.1 v0.5.2 v0.5.3
Edits
10 atoms, 0 tree operations, 2 files

Browse the tree at this change

modified crates/flow-core/src/api.rs +14 −0
…
13 unchanged lines
14 14
    pub runtimes: Vec<String>,
15 15
}
16 16
17 +
/// A pinned subrepo to put in the workspace: `repo`'s state `heads`, at `path`.
18 +
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
19 +
pub struct SubrepoInput {
20 +
    pub path: String,
21 +
    /// `owner/name` on the hub.
22 +
    pub repo: String,
23 +
    pub clone_url: String,
24 +
    pub heads: Vec<String>,
25 +
}
26 +
17 27
/// A claimed job: everything the runner needs, nothing more.
18 28
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
19 29
pub struct Claim {
…
13 unchanged lines
33 43
    pub sandbox: Sandbox,
34 44
    /// The hub's limit for this job.
35 45
    pub timeout_s: u64,
46 +
    /// Subrepos the state pins (Oxygen spec §7.8) that the job may read,
47 +
    /// fetched with the same token and written at their paths.
48 +
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
49 +
    pub subrepos: Vec<SubrepoInput>,
36 50
}
37 51
38 52
/// One line of output (docs/m0.md §2).
…
30 unchanged lines
modified crates/flow/src/runner.rs +36 −8
…
133 unchanged lines
134 134
    let _ = std::fs::remove_dir_all(&scratch);
135 135
    std::fs::create_dir_all(scratch.join("tree"))?;
136 136
    let result = (|| -> Result<Finish> {
137 -
        let files = workspace::export_state(&repo, &heads, &scratch.join("tree"))?;
137 +
        let mut files = workspace::export_state(&repo, &heads, &scratch.join("tree"))?;
138 +
        // Pinned subrepos (Oxygen spec §7.8), at their paths, as part of the
139 +
        // workspace and of the scope hash.
140 +
        for sub in &claim.subrepos {
141 +
            if sub.path.split('/').any(|p| p.is_empty() || p == "." || p == ".." || p == ".oxy") {
142 +
                bail!("subrepo path {:?} is not a clean relative path", sub.path);
143 +
            }
144 +
            note(
145 +
                &mut sink,
146 +
                format!(
147 +
                    "fetching subrepo {} at {} for {}",
148 +
                    sub.repo,
149 +
                    sub.heads.iter().map(|h| &h[..12]).collect::<Vec<_>>().join("+"),
150 +
                    sub.path
151 +
                ),
152 +
            );
153 +
            let sub_repo = cached_repo_at(home, cfg, &sub.repo, &sub.clone_url, &claim.token)?;
154 +
            let sub_heads: Vec<ChangeHash> =
155 +
                sub.heads.iter().map(|h| ChangeHash::from_hex(h).context("bad subrepo head")).collect::<Result<_>>()?;
156 +
            let dir = scratch.join("tree").join(&sub.path);
157 +
            std::fs::create_dir_all(&dir)?;
158 +
            let sub_files = workspace::export_state(&sub_repo, &sub_heads, &dir)?;
159 +
            files.extend(sub_files.into_iter().map(|(p, h)| (format!("{}/{p}", sub.path), h)));
160 +
        }
138 161
        let scope_hash = scope_tree_hash(&claim.job.scope, files.iter().map(|(p, h)| (p.as_str(), *h)));
139 162
        workspace::image(&scratch.join("tree"), &scratch.join("workspace.ext4"))?;
140 163
        std::fs::remove_dir_all(scratch.join("tree"))?;
…
49 unchanged lines
190 213
191 214
/// The runner's copy of a repository, fetched up to date with the job's token.
192 215
fn cached_repo(home: &Home, cfg: &RunnerConfig, claim: &Claim) -> Result<Repo> {
193 -
    let dir: PathBuf = home.root.join("repos").join(claim.repo.replace('/', "__"));
216 +
    cached_repo_at(home, cfg, &claim.repo, &claim.clone_url, &claim.token)
217 +
}
218 +
219 +
/// The runner's copy of `repo`, cloned or fetched with `token`.
220 +
fn cached_repo_at(home: &Home, cfg: &RunnerConfig, repo_name: &str, clone_url: &str, token: &str) -> Result<Repo> {
221 +
    let dir: PathBuf = home.root.join("repos").join(repo_name.replace('/', "__"));
194 222
    let set_remote = |repo: &mut Repo| -> Result<()> {
195 -
        repo.config.set("remote.origin.url", &claim.clone_url).map_err(|e| anyhow::anyhow!("{e}"))?;
196 -
        repo.config.set("remote.origin.token", &claim.token).map_err(|e| anyhow::anyhow!("{e}"))?;
223 +
        repo.config.set("remote.origin.url", clone_url).map_err(|e| anyhow::anyhow!("{e}"))?;
224 +
        repo.config.set("remote.origin.token", token).map_err(|e| anyhow::anyhow!("{e}"))?;
197 225
        if let Some(ca) = &cfg.ca {
198 226
            repo.config.set("remote.origin.ca", &ca.to_string_lossy()).map_err(|e| anyhow::anyhow!("{e}"))?;
199 227
        }
…
2 unchanged lines
202 230
    if dir.join(".oxy").is_dir() {
203 231
        let mut repo = Repo::open(&dir).map_err(|e| anyhow::anyhow!("{e}"))?;
204 232
        set_remote(&mut repo)?;
205 -
        repo.fetch("origin").map_err(|e| anyhow::anyhow!("fetch {}: {e}", claim.clone_url))?;
233 +
        repo.fetch("origin").map_err(|e| anyhow::anyhow!("fetch {clone_url}: {e}"))?;
206 234
        return Ok(repo);
207 235
    }
208 236
    // First job for this repository: clone with the job's token.
209 -
    std::env::set_var("OXY_TOKEN", &claim.token);
237 +
    std::env::set_var("OXY_TOKEN", token);
210 238
    if let Some(ca) = &cfg.ca {
211 239
        std::env::set_var("OXY_CA", ca);
212 240
    }
213 -
    let cloned = Repo::clone(&claim.clone_url, &dir);
241 +
    let cloned = Repo::clone(clone_url, &dir);
214 242
    std::env::remove_var("OXY_TOKEN");
215 243
    std::env::remove_var("OXY_CA");
216 244
    let mut repo = cloned.map_err(|e| {
217 245
        let _ = std::fs::remove_dir_all(&dir);
218 -
        anyhow::anyhow!("clone {}: {e}", claim.clone_url)
246 +
        anyhow::anyhow!("clone {}: {e}", clone_url)
219 247
    })?;
220 248
    set_remote(&mut repo)?;
221 249
    Ok(repo)
…
1 unchanged lines