|
|
… |
104 unchanged lines |
| 105 |
105 |
|
/// Say this runner is there while it claims nothing: its slots are full, |
| 106 |
106 |
|
/// or it is draining (oxygenhub#243). Once, no retries: the next comes |
| 107 |
107 |
|
/// soon. A hub without the endpoint (404) is not an error. |
|
108 |
+ |
/// A ten-minute certificate for `identity`, asked with a job's token |
|
109 |
+ |
/// (keyless, oxygenhub#241), and the identity of the hub's key that |
|
110 |
+ |
/// issued it, from `/.well-known/oxyhub-key`. |
|
111 |
+ |
pub fn certify( |
|
112 |
+ |
&self, |
|
113 |
+ |
job_token: &str, |
|
114 |
+ |
identity: &oxy_core::identity::Identity, |
|
115 |
+ |
proof: &oxy_core::identity::Signature, |
|
116 |
+ |
) -> Result<(oxy_core::cert::SignedCertificate, oxy_core::identity::Identity)> { |
|
117 |
+ |
use oxy_core::dsse::{base64, base64_decode}; |
|
118 |
+ |
let body = serde_json::json!({ |
|
119 |
+ |
"identity": base64(&identity.encode()), |
|
120 |
+ |
"proof": base64(&oxy_core::cbor::encode(&proof.to_value())), |
|
121 |
+ |
"scope": "seal", |
|
122 |
+ |
}); |
|
123 |
+ |
let url = format!("{}/api/v1/keyless/certify", self.base); |
|
124 |
+ |
let r = self |
|
125 |
+ |
.agent |
|
126 |
+ |
.post(&url) |
|
127 |
+ |
.set("Authorization", &format!("Bearer {job_token}")) |
|
128 |
+ |
.send_json(body) |
|
129 |
+ |
.map_err(|e| anyhow::anyhow!("asking the hub to certify the key: {e}"))?; |
|
130 |
+ |
let v: serde_json::Value = r.into_json()?; |
|
131 |
+ |
let cert = base64_decode(v["certificate"].as_str().unwrap_or("")) |
|
132 |
+ |
.context("the hub's answer has no certificate") |
|
133 |
+ |
.and_then(|b| oxy_core::cert::SignedCertificate::decode(&b).map_err(|e| anyhow::anyhow!("{e}")))?; |
|
134 |
+ |
let mut bytes = Vec::new(); |
|
135 |
+ |
std::io::Read::read_to_end( |
|
136 |
+ |
&mut self.agent.get(&format!("{}/.well-known/oxyhub-key", self.base)).call()?.into_reader(), |
|
137 |
+ |
&mut bytes, |
|
138 |
+ |
)?; |
|
139 |
+ |
let issuer = oxy_core::identity::Identity::decode(&bytes)?; |
|
140 |
+ |
if issuer.hash() != cert.cert.issuer || cert.verify(&issuer).is_err() { |
|
141 |
+ |
bail!("the certificate is not signed by the hub's published key"); |
|
142 |
+ |
} |
|
143 |
+ |
if cert.cert.key_id() != identity.hash() { |
|
144 |
+ |
bail!("the hub certified another key"); |
|
145 |
+ |
} |
|
146 |
+ |
Ok((cert, issuer)) |
|
147 |
+ |
} |
|
148 |
+ |
|
| 108 |
149 |
|
pub fn heartbeat(&self) -> Result<()> { |
| 109 |
150 |
|
match self.post_with("/flow/runners/heartbeat", &serde_json::json!({}), &[]) { |
| 110 |
151 |
|
Ok(_) => Ok(()), |
|
|
… |
873 unchanged lines |
| 984 |
1025 |
|
duration_ms: outcome.duration_ms, |
| 985 |
1026 |
|
}); |
| 986 |
1027 |
|
let signed_receipt = match &receipt { |
| 987 |
|
- |
Some(r) => Some(hex::encode(sign_receipt(&repo, &heads, cfg, key, claim, r)?.encode())), |
|
1028 |
+ |
Some(r) => { |
|
1029 |
+ |
let mut signed = sign_receipt(&repo, &heads, cfg, key, claim, r)?; |
|
1030 |
+ |
if cfg.keyless { |
|
1031 |
+ |
match sign_keyless(&repo, &heads, cfg, hub, claim, r) { |
|
1032 |
+ |
Ok(k) => signed = k, |
|
1033 |
+ |
Err(e) => { |
|
1034 |
+ |
note(&mut sink, format!("receipt: signed with the runner's key; a short-lived key was not certified: {e:#}")) |
|
1035 |
+ |
} |
|
1036 |
+ |
} |
|
1037 |
+ |
} |
|
1038 |
+ |
Some(hex::encode(signed.encode())) |
|
1039 |
+ |
} |
| 988 |
1040 |
|
None => None, |
| 989 |
1041 |
|
}; |
| 990 |
1042 |
|
// Measured before a cached disk leaves the scratch directory. |
|
|
… |
64 unchanged lines |
| 1055 |
1107 |
|
Ok(oxy_core::receipt::SignedReceipt::sign(receipt.to_oxy(), repo.id()?, state, &run, &key.identity, &key.secret)) |
| 1056 |
1108 |
|
} |
| 1057 |
1109 |
|
|
|
1110 |
+ |
/// Sign `receipt` with a fresh key the hub certifies for this job now, |
|
1111 |
+ |
/// just before signing, so the certificate covers the moment the hub |
|
1112 |
+ |
/// accepts it (keyless, oxygenhub#241). The key is dropped with the |
|
1113 |
+ |
/// receipt signed: nothing long-lived signs. |
|
1114 |
+ |
fn sign_keyless( |
|
1115 |
+ |
repo: &Repo, |
|
1116 |
+ |
heads: &[ChangeHash], |
|
1117 |
+ |
cfg: &RunnerConfig, |
|
1118 |
+ |
hub: &Hub, |
|
1119 |
+ |
claim: &Claim, |
|
1120 |
+ |
receipt: &Receipt, |
|
1121 |
+ |
) -> Result<oxy_core::receipt::SignedReceipt> { |
|
1122 |
+ |
let sk = oxy_core::identity::SecretKey::generate(); |
|
1123 |
+ |
let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0); |
|
1124 |
+ |
let name = format!("{} job {}", cfg.name.as_deref().unwrap_or("runner"), claim.job_id); |
|
1125 |
+ |
let id = sk.identity(&name, "", now); |
|
1126 |
+ |
let proof = sk.sign_challenge(&id.hash(), &oxy_core::cert::certify_challenge(&claim.token)).map_err(|e| anyhow::anyhow!("{e}"))?; |
|
1127 |
+ |
let (cert, issuer) = hub.certify(&claim.token, &id, &proof)?; |
|
1128 |
+ |
let state = state_of(heads); |
|
1129 |
+ |
let run = format!("{}/{}/flow/{}", cfg.hub.trim_end_matches('/'), claim.repo, claim.run_number); |
|
1130 |
+ |
Ok(oxy_core::receipt::SignedReceipt::sign(receipt.to_oxy(), repo.id()?, state, &run, &id, &sk).with_certificate(cert, issuer)) |
|
1131 |
+ |
} |
|
1132 |
+ |
|
| 1058 |
1133 |
|
/// The ObjectRef of the state whose heads are `heads` (Oxygen spec §10.1). |
| 1059 |
1134 |
|
pub fn state_of(heads: &[ChangeHash]) -> oxy_core::hash::ObjectRef { |
| 1060 |
1135 |
|
use oxy_core::hash::{hash_object, ObjectType}; |
|
|
… |
238 unchanged lines |
| 1299 |
1374 |
|
let cfg = again.config.runner.unwrap(); |
| 1300 |
1375 |
|
assert_eq!((cfg.hub.as_str(), cfg.token.as_str(), cfg.name.as_deref()), (hub.as_str(), "flowrunner_secret", Some("plex"))); |
| 1301 |
1376 |
|
assert_eq!(cfg.labels, ["gpu"]); |
|
1377 |
+ |
} |
|
1378 |
+ |
|
|
1379 |
+ |
/// A hub that answers certify with a certificate `sign_as` signs for |
|
1380 |
+ |
/// `cert_key`, and publishes `published` as its key: two requests. |
|
1381 |
+ |
fn fake_certify_hub( |
|
1382 |
+ |
sign_as: (oxy_core::identity::Identity, oxy_core::identity::SecretKey), |
|
1383 |
+ |
cert_key: oxy_core::identity::Identity, |
|
1384 |
+ |
published: oxy_core::identity::Identity, |
|
1385 |
+ |
) -> String { |
|
1386 |
+ |
use std::io::{Read, Write}; |
|
1387 |
+ |
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); |
|
1388 |
+ |
let addr = listener.local_addr().unwrap(); |
|
1389 |
+ |
std::thread::spawn(move || { |
|
1390 |
+ |
for _ in 0..2 { |
|
1391 |
+ |
let Ok((mut s, _)) = listener.accept() else { return }; |
|
1392 |
+ |
let mut req = Vec::new(); |
|
1393 |
+ |
let mut buf = [0u8; 65536]; |
|
1394 |
+ |
loop { |
|
1395 |
+ |
let n = s.read(&mut buf).unwrap_or(0); |
|
1396 |
+ |
if n == 0 { |
|
1397 |
+ |
break; |
|
1398 |
+ |
} |
|
1399 |
+ |
req.extend_from_slice(&buf[..n]); |
|
1400 |
+ |
let text = String::from_utf8_lossy(&req); |
|
1401 |
+ |
if let Some(end) = text.find("\r\n\r\n") { |
|
1402 |
+ |
let len = text[..end] |
|
1403 |
+ |
.lines() |
|
1404 |
+ |
.find_map(|l| { |
|
1405 |
+ |
l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap_or(0)) |
|
1406 |
+ |
}) |
|
1407 |
+ |
.unwrap_or(0); |
|
1408 |
+ |
if req.len() >= end + 4 + len { |
|
1409 |
+ |
break; |
|
1410 |
+ |
} |
|
1411 |
+ |
} |
|
1412 |
+ |
} |
|
1413 |
+ |
let line = String::from_utf8_lossy(&req).lines().next().unwrap_or("").to_string(); |
|
1414 |
+ |
let body: Vec<u8> = if line.contains("/keyless/certify") { |
|
1415 |
+ |
let cert = oxy_core::cert::SignedCertificate::sign( |
|
1416 |
+ |
oxy_core::cert::Certificate { |
|
1417 |
+ |
key: cert_key.clone(), |
|
1418 |
+ |
issuer: sign_as.0.hash(), |
|
1419 |
+ |
subject: "repo:andy/demo:bookmark:main".into(), |
|
1420 |
+ |
claims: vec![], |
|
1421 |
+ |
scope: oxy_core::identity::Scope::SEAL, |
|
1422 |
+ |
not_before: 1, |
|
1423 |
+ |
expires: 600_001, |
|
1424 |
+ |
}, |
|
1425 |
+ |
&sign_as.0, |
|
1426 |
+ |
&sign_as.1, |
|
1427 |
+ |
); |
|
1428 |
+ |
serde_json::json!({ "certificate": oxy_core::dsse::base64(&cert.encode()) }).to_string().into_bytes() |
|
1429 |
+ |
} else { |
|
1430 |
+ |
published.encode() |
|
1431 |
+ |
}; |
|
1432 |
+ |
let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); |
|
1433 |
+ |
let _ = s.write_all(&body); |
|
1434 |
+ |
} |
|
1435 |
+ |
}); |
|
1436 |
+ |
format!("http://{addr}") |
|
1437 |
+ |
} |
|
1438 |
+ |
|
|
1439 |
+ |
/// keyless (oxygenhub#241): the runner takes a certificate only when it |
|
1440 |
+ |
/// is for its fresh key and signed by the key the hub publishes. |
|
1441 |
+ |
#[test] |
|
1442 |
+ |
fn a_certificate_is_taken_only_for_this_key_from_the_hubs_key() { |
|
1443 |
+ |
use oxy_core::identity::SecretKey; |
|
1444 |
+ |
let hub_sk = SecretKey::generate(); |
|
1445 |
+ |
let hub_id = hub_sk.identity("hub", "", 1); |
|
1446 |
+ |
let sk = SecretKey::generate(); |
|
1447 |
+ |
let id = sk.identity("earth-1 job 7", "", 1); |
|
1448 |
+ |
let proof = sk.sign_challenge(&id.hash(), &oxy_core::cert::certify_challenge("flowjob_x")).unwrap(); |
|
1449 |
+ |
|
|
1450 |
+ |
let base = fake_certify_hub((hub_id.clone(), hub_sk.clone()), id.clone(), hub_id.clone()); |
|
1451 |
+ |
let (cert, issuer) = hub_at(base).certify("flowjob_x", &id, &proof).unwrap(); |
|
1452 |
+ |
assert_eq!((cert.cert.key_id(), issuer.hash()), (id.hash(), hub_id.hash())); |
|
1453 |
+ |
|
|
1454 |
+ |
let mallory_sk = SecretKey::generate(); |
|
1455 |
+ |
let mallory = mallory_sk.identity("mallory", "", 1); |
|
1456 |
+ |
let base = fake_certify_hub((mallory.clone(), mallory_sk), id.clone(), hub_id.clone()); |
|
1457 |
+ |
let e = hub_at(base).certify("flowjob_x", &id, &proof).unwrap_err().to_string(); |
|
1458 |
+ |
assert!(e.contains("not signed by the hub's published key"), "{e}"); |
|
1459 |
+ |
|
|
1460 |
+ |
let other = SecretKey::generate().identity("other", "", 1); |
|
1461 |
+ |
let base = fake_certify_hub((hub_id.clone(), hub_sk), other, hub_id); |
|
1462 |
+ |
let e = hub_at(base).certify("flowjob_x", &id, &proof).unwrap_err().to_string(); |
|
1463 |
+ |
assert!(e.contains("another key"), "{e}"); |
| 1302 |
1464 |
|
} |
| 1303 |
1465 |
|
|
| 1304 |
1466 |
|
fn hub_at(base: String) -> Hub { |
|
|
… |
147 unchanged lines |