OXYGENHUB
# a083cc802892 oxygenflow / crates / flow / src / runner.rs
8.6 KiB · Rust · last changed OxygenFlow M0: pipeline parser and canonical job graph, receipts, the runner API types, flow-init, the Firecracker tier, flow check, run, rootfs import and runner 1 month ago source blame history raw
e0918b88 Andy McCall 1
//! `flow runner` (docs/m0.md §5): claim a job from OxygenHub, fetch the
2
//! repository into a cache with the job's token, write the state into a
3
//! workspace, run it in its tier, stream the log, report the receipt.
4
//! One job at a time.
5
6
use crate::home::{Home, RunnerConfig};
7
use crate::{microvm, workspace};
8
use anyhow::{bail, Context, Result};
9
use flow_core::api::{Claim, ClaimRequest, Finish, LogBatch, Record, MAX_RECORD_TEXT};
10
use flow_core::{scope_tree_hash, Receipt};
11
use oxy_core::hash::ChangeHash;
12
use oxy_core::repo::Repo;
13
use std::path::PathBuf;
14
use std::time::{Duration, Instant};
15
16
pub struct Hub {
17
    base: String,
18
    token: String,
19
    agent: ureq::Agent,
20
}
21
22
impl Hub {
23
    pub fn new(cfg: &RunnerConfig) -> Result<Hub> {
24
        let tls = oxy_core::protocol::tls_client_config(cfg.ca.as_deref()).map_err(|e| anyhow::anyhow!("{e}"))?;
25
        let agent = ureq::AgentBuilder::new().tls_config(tls).timeout(Duration::from_secs(60)).build();
26
        Ok(Hub { base: cfg.hub.trim_end_matches('/').to_string(), token: cfg.token.clone(), agent })
27
    }
28
29
    fn post(&self, path: &str, body: &impl serde::Serialize) -> Result<Option<ureq::Response>> {
30
        let url = format!("{}/api/v1{path}", self.base);
31
        match self.agent.post(&url).set("Authorization", &format!("Bearer {}", self.token)).send_json(body) {
32
            Ok(r) if r.status() == 204 => Ok(None),
33
            Ok(r) => Ok(Some(r)),
34
            Err(ureq::Error::Status(code, r)) => bail!("{url}: HTTP {code}: {}", r.into_string().unwrap_or_default()),
35
            Err(e) => bail!("{url}: {e}"),
36
        }
37
    }
38
39
    pub fn claim(&self, req: &ClaimRequest) -> Result<Option<Claim>> {
40
        match self.post("/flow/claim", req)? {
41
            Some(r) => Ok(Some(r.into_json()?)),
42
            None => Ok(None),
43
        }
44
    }
45
46
    pub fn log(&self, job: i64, records: Vec<Record>) -> Result<()> {
47
        self.post(&format!("/flow/jobs/{job}/log"), &LogBatch { records })?;
48
        Ok(())
49
    }
50
51
    pub fn finish(&self, job: i64, f: &Finish) -> Result<()> {
52
        self.post(&format!("/flow/jobs/{job}/finish"), f)?;
53
        Ok(())
54
    }
55
}
56
57
pub fn serve(home: &Home) -> Result<()> {
58
    let cfg = home.config.runner.clone().context("flow.toml has no [runner] section")?;
59
    let hub = Hub::new(&cfg)?;
60
    let name = cfg.name.clone().unwrap_or_else(|| "flow".into());
61
    let req = ClaimRequest { runner: name.clone(), runtimes: vec!["microvm".into()] };
62
    eprintln!("flow runner {name}: polling {}", cfg.hub);
63
    let mut idle = Duration::from_secs(2);
64
    loop {
65
        match hub.claim(&req) {
66
            Ok(Some(claim)) => {
67
                idle = Duration::from_secs(2);
68
                let id = claim.job_id;
69
                eprintln!("job {id}: {} {} on {}", claim.repo, claim.job.name, claim.heads.first().map(|h| &h[..12]).unwrap_or("-"));
70
                let finish = run_claim(home, &hub, &cfg, &claim);
71
                let finish =
72
                    finish.unwrap_or_else(|e| Finish { exit_code: None, duration_ms: 0, receipt: None, error: Some(format!("{e:#}")) });
73
                eprintln!(
74
                    "job {id}: exit {:?}{}",
75
                    finish.exit_code,
76
                    finish.error.as_deref().map(|e| format!(" ({e})")).unwrap_or_default()
77
                );
78
                if let Err(e) = hub.finish(id, &finish) {
79
                    eprintln!("job {id}: reporting failed: {e:#}");
80
                }
81
            }
82
            Ok(None) => std::thread::sleep(idle),
83
            Err(e) => {
84
                eprintln!("claim: {e:#}");
85
                std::thread::sleep(idle);
86
                idle = (idle * 2).min(Duration::from_secs(60));
87
            }
88
        }
89
    }
90
}
91
92
/// Batches records to the hub every second (or every 200 lines).
93
struct LogSink<'a> {
94
    hub: &'a Hub,
95
    job: i64,
96
    buf: Vec<Record>,
97
    last: Instant,
98
}
99
100
impl LogSink<'_> {
101
    fn push(&mut self, mut r: Record) {
102
        if r.text.len() > MAX_RECORD_TEXT {
103
            let mut cut = MAX_RECORD_TEXT;
104
            while !r.text.is_char_boundary(cut) {
105
                cut -= 1;
106
            }
107
            r.text.truncate(cut);
108
        }
109
        self.buf.push(r);
110
        if self.buf.len() >= 200 || self.last.elapsed() >= Duration::from_secs(1) {
111
            self.flush();
112
        }
113
    }
114
115
    fn flush(&mut self) {
116
        if !self.buf.is_empty() {
117
            if let Err(e) = self.hub.log(self.job, std::mem::take(&mut self.buf)) {
118
                eprintln!("job {}: log: {e:#}", self.job);
119
            }
120
        }
121
        self.last = Instant::now();
122
    }
123
}
124
125
fn run_claim(home: &Home, hub: &Hub, cfg: &RunnerConfig, claim: &Claim) -> Result<Finish> {
126
    let started = Instant::now();
127
    let mut sink = LogSink { hub, job: claim.job_id, buf: Vec::new(), last: Instant::now() };
128
    let note =
129
        |sink: &mut LogSink, text: String| sink.push(Record { t: started.elapsed().as_millis() as u64, stream: "flow".into(), text });
130
    let heads: Vec<ChangeHash> = claim.heads.iter().map(|h| ChangeHash::from_hex(h).context("bad head")).collect::<Result<_>>()?;
131
    note(&mut sink, format!("fetching {} for state {}", claim.repo, claim.heads.iter().map(|h| &h[..12]).collect::<Vec<_>>().join("+")));
132
    let repo = cached_repo(home, cfg, claim)?;
133
    let scratch = home.root.join("work").join(format!("job-{}", claim.job_id));
134
    let _ = std::fs::remove_dir_all(&scratch);
135
    std::fs::create_dir_all(scratch.join("tree"))?;
136
    let result = (|| -> Result<Finish> {
137
        let files = workspace::export_state(&repo, &heads, &scratch.join("tree"))?;
138
        let scope_hash = scope_tree_hash(&claim.job.scope, files.iter().map(|(p, h)| (p.as_str(), *h)));
139
        workspace::image(&scratch.join("tree"), &scratch.join("workspace.ext4"))?;
140
        std::fs::remove_dir_all(scratch.join("tree"))?;
141
        note(
142
            &mut sink,
143
            format!(
144
                "booting a microVM: {} vCPU, {} MiB, rootfs {}",
145
                claim.sandbox.cpus,
146
                claim.sandbox.memory_mib,
147
                &claim.sandbox.rootfs[..12]
148
            ),
149
        );
150
        let outcome = microvm::run(
151
            home,
152
            &claim.sandbox,
153
            &claim.job,
154
            &scratch.join("workspace.ext4"),
155
            &scratch,
156
            Duration::from_secs(claim.timeout_s),
157
            &mut |r| sink.push(r),
158
        )?;
159
        let receipt = outcome.exit_code.map(|code| Receipt {
160
            job_name: claim.job.name.clone(),
161
            pipeline_hash: hex32(&claim.pipeline_hash),
162
            scope_tree_hash: scope_hash,
163
            inputs: vec![],
164
            environment: outcome.environment.hash(),
165
            outputs: vec![],
166
            exit_code: code as u32,
167
            duration_ms: outcome.duration_ms,
168
        });
169
        Ok(Finish {
170
            exit_code: outcome.exit_code,
171
            duration_ms: started.elapsed().as_millis() as u64,
172
            receipt,
173
            error: outcome.timed_out.then(|| "stopped at the time limit".to_string()),
174
        })
175
    })();
176
    let _ = std::fs::remove_dir_all(&scratch);
177
    sink.flush();
178
    result
179
}
180
181
fn hex32(s: &str) -> [u8; 32] {
182
    let mut out = [0; 32];
183
    if let Ok(v) = hex::decode(s) {
184
        if v.len() == 32 {
185
            out.copy_from_slice(&v);
186
        }
187
    }
188
    out
189
}
190
191
/// The runner's copy of a repository, fetched up to date with the job's token.
192
fn cached_repo(home: &Home, cfg: &RunnerConfig, claim: &Claim) -> Result<Repo> {
193
    let dir: PathBuf = home.root.join("repos").join(claim.repo.replace('/', "__"));
194
    let set_remote = |repo: &mut Repo| -> Result<()> {
195
        repo.config.set("remote.origin.url", &claim.clone_url).map_err(|e| anyhow::anyhow!("{e}"))?;
196
        repo.config.set("remote.origin.token", &claim.token).map_err(|e| anyhow::anyhow!("{e}"))?;
197
        if let Some(ca) = &cfg.ca {
198
            repo.config.set("remote.origin.ca", &ca.to_string_lossy()).map_err(|e| anyhow::anyhow!("{e}"))?;
199
        }
200
        repo.save_config().map_err(|e| anyhow::anyhow!("{e}"))
201
    };
202
    if dir.join(".oxy").is_dir() {
203
        let mut repo = Repo::open(&dir).map_err(|e| anyhow::anyhow!("{e}"))?;
204
        set_remote(&mut repo)?;
205
        repo.fetch("origin").map_err(|e| anyhow::anyhow!("fetch {}: {e}", claim.clone_url))?;
206
        return Ok(repo);
207
    }
208
    // First job for this repository: clone with the job's token.
209
    std::env::set_var("OXY_TOKEN", &claim.token);
210
    if let Some(ca) = &cfg.ca {
211
        std::env::set_var("OXY_CA", ca);
212
    }
213
    let cloned = Repo::clone(&claim.clone_url, &dir);
214
    std::env::remove_var("OXY_TOKEN");
215
    std::env::remove_var("OXY_CA");
216
    let mut repo = cloned.map_err(|e| {
217
        let _ = std::fs::remove_dir_all(&dir);
218
        anyhow::anyhow!("clone {}: {e}", claim.clone_url)
219
    })?;
220
    set_remote(&mut repo)?;
221
    Ok(repo)
222
}