| ea85398a Andy McCall ↶ |
1 |
|
| 2 |
|
| 3 |
|
| 4 |
|
| 5 |
|
| 6 |
|
| 7 |
use crate::{db, Shared}; |
| 8 |
use axum::extract::{Path, State}; |
| 9 |
use axum::http::{header, StatusCode}; |
| 10 |
use axum::response::{IntoResponse, Response}; |
| 11 |
use sha2::Digest; |
| 12 |
use std::collections::{BTreeMap, HashMap}; |
| 13 |
use std::io::Read; |
| 14 |
use std::sync::{Arc, Mutex, OnceLock}; |
| 15 |
|
| 16 |
|
| 17 |
#[derive(Clone, Debug)] |
| 18 |
struct Entry { |
| 19 |
|
| 20 |
control: String, |
| 21 |
package: String, |
| 22 |
architecture: String, |
| 23 |
|
| 24 |
file: String, |
| 25 |
size: u64, |
| 26 |
md5: String, |
| 27 |
sha256: String, |
| 28 |
} |
| 29 |
|
| 30 |
|
| 31 |
struct Index { |
| 32 |
|
| 33 |
key: String, |
| 34 |
|
| 35 |
files: BTreeMap<String, Vec<u8>>, |
| 36 |
release: String, |
| 37 |
in_release: Vec<u8>, |
| 38 |
release_gpg: Vec<u8>, |
| 39 |
|
| 40 |
pool: HashMap<String, String>, |
| 41 |
} |
| 42 |
|
| 43 |
|
| 44 |
fn parsed() -> &'static Mutex<HashMap<String, Entry>> { |
| 45 |
static P: OnceLock<Mutex<HashMap<String, Entry>>> = OnceLock::new(); |
| 46 |
P.get_or_init(Default::default) |
| 47 |
} |
| 48 |
|
| 49 |
fn built() -> &'static Mutex<Option<Arc<Index>>> { |
| 50 |
static B: OnceLock<Mutex<Option<Arc<Index>>>> = OnceLock::new(); |
| 51 |
B.get_or_init(Default::default) |
| 52 |
} |
| 53 |
|
| 54 |
|
| 55 |
|
| 56 |
|
| 57 |
fn ar_members(bytes: &[u8]) -> Result<Vec<(String, &[u8])>, String> { |
| 58 |
let body = bytes.strip_prefix(b"!<arch>\n").ok_or("not an ar archive")?; |
| 59 |
let mut out = Vec::new(); |
| 60 |
let mut at = 0; |
| 61 |
while at + 60 <= body.len() { |
| 62 |
let head = &body[at..at + 60]; |
| 63 |
let name = String::from_utf8_lossy(&head[..16]).trim().trim_end_matches('/').to_string(); |
| 64 |
let size: usize = String::from_utf8_lossy(&head[48..58]).trim().parse().map_err(|_| "bad ar member size")?; |
| 65 |
let start = at + 60; |
| 66 |
let end = start.checked_add(size).filter(|e| *e <= body.len()).ok_or("truncated ar archive")?; |
| 67 |
out.push((name, &body[start..end])); |
| 68 |
at = end + (size & 1); |
| 69 |
} |
| 70 |
Ok(out) |
| 71 |
} |
| 72 |
|
| 73 |
|
| 74 |
pub fn deb_control(bytes: &[u8]) -> Result<String, String> { |
| 75 |
let members = ar_members(bytes)?; |
| 76 |
let (name, data) = members.into_iter().find(|(n, _)| n.starts_with("control.tar")).ok_or("no control.tar member")?; |
| 77 |
let mut tar_bytes = Vec::new(); |
| 78 |
match name.as_str() { |
| 79 |
"control.tar" => tar_bytes.extend_from_slice(data), |
| 80 |
"control.tar.gz" => { |
| 81 |
flate2::read::GzDecoder::new(data).read_to_end(&mut tar_bytes).map_err(|e| e.to_string())?; |
| 82 |
} |
| 83 |
"control.tar.xz" => lzma_rs::xz_decompress(&mut std::io::BufReader::new(data), &mut tar_bytes).map_err(|e| e.to_string())?, |
| 84 |
"control.tar.zst" => tar_bytes = zstd::decode_all(data).map_err(|e| e.to_string())?, |
| 85 |
other => return Err(format!("{other}: a compression this hub does not read")), |
| 86 |
} |
| 87 |
let mut archive = tar::Archive::new(tar_bytes.as_slice()); |
| 88 |
for entry in archive.entries().map_err(|e| e.to_string())? { |
| 89 |
let mut entry = entry.map_err(|e| e.to_string())?; |
| 90 |
let path = entry.path().map_err(|e| e.to_string())?.to_string_lossy().to_string(); |
| 91 |
if path.trim_start_matches("./") == "control" { |
| 92 |
let mut text = String::new(); |
| 93 |
entry.read_to_string(&mut text).map_err(|e| e.to_string())?; |
| 94 |
return Ok(text); |
| 95 |
} |
| 96 |
} |
| 97 |
Err("no control file".into()) |
| 98 |
} |
| 99 |
|
| 100 |
|
| 101 |
fn field<'a>(control: &'a str, name: &str) -> Option<&'a str> { |
| 102 |
control.lines().find_map(|l| l.strip_prefix(name).and_then(|r| r.strip_prefix(':')).map(str::trim)) |
| 103 |
} |
| 104 |
|
| 105 |
fn entry_of(file: &str, bytes: &[u8]) -> Result<Entry, String> { |
| 106 |
let control = deb_control(bytes)?; |
| 107 |
let package = field(&control, "Package").ok_or("no Package field")?.to_string(); |
| 108 |
let architecture = field(&control, "Architecture").ok_or("no Architecture field")?.to_string(); |
| 109 |
field(&control, "Version").ok_or("no Version field")?; |
| 110 |
Ok(Entry { |
| 111 |
control: control.trim_end().to_string(), |
| 112 |
package, |
| 113 |
architecture, |
| 114 |
file: file.to_string(), |
| 115 |
size: bytes.len() as u64, |
| 116 |
md5: hex::encode(md5::Md5::digest(bytes)), |
| 117 |
sha256: hex::encode(sha2::Sha256::digest(bytes)), |
| 118 |
}) |
| 119 |
} |
| 120 |
|
| 121 |
|
| 122 |
|
| 123 |
|
| 124 |
|
| 125 |
async fn debs(app: &Shared) -> Result<Vec<(i64, String, String)>, sqlx::Error> { |
| 126 |
let mut out = Vec::new(); |
| 127 |
for slug in &app.config.apt.repos { |
| 128 |
let Some((owner, name)) = slug.split_once('/') else { continue }; |
| 129 |
let Some(row) = db::repos::get(&app.db, owner, name).await? else { continue }; |
| 130 |
if row.visibility() != hub_api::Visibility::Public { |
| 131 |
tracing::warn!("apt: {slug} is not public, so its packages are not served"); |
| 132 |
continue; |
| 133 |
} |
| 134 |
let rows: Vec<(i64, String, String)> = sqlx::query_as( |
| 135 |
"SELECT a.id, a.name, a.hash FROM release_assets a JOIN releases r ON r.id = a.release_id \ |
| 5d90e0e0 Andy McCall |
136 |
WHERE r.repo_id = $1 AND a.name LIKE '%.deb' ORDER BY a.id", |
| ea85398a Andy McCall ↶ |
137 |
) |
| 138 |
.bind(row.id) |
| 139 |
.fetch_all(&app.db) |
| 140 |
.await?; |
| 141 |
out.extend(rows); |
| 142 |
} |
| 143 |
Ok(out) |
| 144 |
} |
| 145 |
|
| 146 |
|
| 147 |
fn pool_path(component: &str, e: &Entry) -> String { |
| 148 |
let first: String = e.package.chars().take(1).collect(); |
| 149 |
format!("pool/{component}/{first}/{}/{}", e.package, e.file) |
| 150 |
} |
| 151 |
|
| 152 |
|
| 153 |
fn rfc2822(secs: i64) -> String { |
| 154 |
let t = time::OffsetDateTime::from_unix_timestamp(secs).unwrap_or(time::OffsetDateTime::UNIX_EPOCH); |
| 155 |
let fmt = time::macros::format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] UTC"); |
| 156 |
t.format(&fmt).unwrap_or_default() |
| 157 |
} |
| 158 |
|
| 159 |
|
| 160 |
async fn gpg(app: &Shared, args: &[&str], input: &[u8]) -> Result<Vec<u8>, String> { |
| 161 |
use tokio::io::AsyncWriteExt; |
| 162 |
let home = app.config.data.join(&app.config.apt.gnupg_home); |
| 163 |
let mut child = tokio::process::Command::new("gpg") |
| 164 |
.arg("--homedir") |
| 165 |
.arg(&home) |
| 166 |
.args(["--batch", "--yes", "--quiet", "--no-tty", "--pinentry-mode", "loopback", "--passphrase", ""]) |
| 167 |
.args(args) |
| 168 |
.stdin(std::process::Stdio::piped()) |
| 169 |
.stdout(std::process::Stdio::piped()) |
| 170 |
.stderr(std::process::Stdio::piped()) |
| 171 |
.spawn() |
| 172 |
.map_err(|e| format!("gpg: {e}"))?; |
| 173 |
let mut stdin = child.stdin.take().expect("piped"); |
| 174 |
stdin.write_all(input).await.map_err(|e| e.to_string())?; |
| 175 |
drop(stdin); |
| 176 |
let out = child.wait_with_output().await.map_err(|e| e.to_string())?; |
| 177 |
if !out.status.success() { |
| 178 |
return Err(format!("gpg {}: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim())); |
| 179 |
} |
| 180 |
Ok(out.stdout) |
| 181 |
} |
| 182 |
|
| 183 |
|
| 184 |
async fn index(app: &Shared) -> Result<Arc<Index>, String> { |
| 185 |
let assets = debs(app).await.map_err(|e| e.to_string())?; |
| 186 |
let key: String = assets.iter().map(|(id, _, h)| format!("{id}:{h};")).collect(); |
| 187 |
if let Some(i) = built().lock().unwrap().as_ref().filter(|i| i.key == key) { |
| 188 |
return Ok(i.clone()); |
| 189 |
} |
| 190 |
let cfg = &app.config.apt; |
| 191 |
let mut entries: Vec<Entry> = Vec::new(); |
| 192 |
let mut pool = HashMap::new(); |
| 193 |
for (_, name, hash) in &assets { |
| 194 |
let known = parsed().lock().unwrap().get(hash).cloned(); |
| 195 |
let e = match known { |
| 196 |
Some(e) => Entry { file: name.clone(), ..e }, |
| 197 |
None => { |
| 198 |
let bytes = tokio::fs::read(crate::web::releases::blob_path(app, hash)).await.map_err(|e| e.to_string())?; |
| 199 |
match entry_of(name, &bytes) { |
| 200 |
Ok(e) => { |
| 201 |
parsed().lock().unwrap().insert(hash.clone(), e.clone()); |
| 202 |
e |
| 203 |
} |
| 204 |
Err(why) => { |
| 205 |
tracing::warn!("apt: {name} is left out: {why}"); |
| 206 |
continue; |
| 207 |
} |
| 208 |
} |
| 209 |
} |
| 210 |
}; |
| 211 |
if pool.contains_key(&e.file) { |
| 212 |
continue; |
| 213 |
} |
| 214 |
pool.insert(e.file.clone(), hash.clone()); |
| 215 |
entries.push(e); |
| 216 |
} |
| 217 |
let mut files = BTreeMap::new(); |
| 218 |
for arch in &cfg.architectures { |
| 219 |
let mut text = String::new(); |
| 220 |
for e in entries.iter().filter(|e| e.architecture == *arch || e.architecture == "all") { |
| 221 |
text.push_str(&format!( |
| 222 |
"{}\nFilename: {}\nSize: {}\nMD5sum: {}\nSHA256: {}\n\n", |
| 223 |
e.control, |
| 224 |
pool_path(&cfg.component, e), |
| 225 |
e.size, |
| 226 |
e.md5, |
| 227 |
e.sha256 |
| 228 |
)); |
| 229 |
} |
| 230 |
let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best()); |
| 231 |
std::io::Write::write_all(&mut gz, text.as_bytes()).map_err(|e| e.to_string())?; |
| 232 |
let gz = gz.finish().map_err(|e| e.to_string())?; |
| 233 |
files.insert(format!("{}/binary-{arch}/Packages", cfg.component), text.into_bytes()); |
| 234 |
files.insert(format!("{}/binary-{arch}/Packages.gz", cfg.component), gz); |
| 235 |
} |
| 236 |
let sums = |f: &dyn Fn(&[u8]) -> String| -> String { |
| 237 |
files.iter().map(|(path, bytes)| format!(" {} {:>8} {path}\n", f(bytes), bytes.len())).collect() |
| 238 |
}; |
| 239 |
let release = format!( |
| 240 |
"Origin: {site}\nLabel: {site}\nSuite: {suite}\nCodename: {suite}\nDate: {date}\nArchitectures: {arches}\nComponents: {component}\nDescription: {site} packages (oxy and oh)\nMD5Sum:\n{md5}SHA256:\n{sha256}", |
| 241 |
site = app.config.site.name, |
| 242 |
suite = cfg.suite, |
| 243 |
date = rfc2822(db::now_ms() / 1000), |
| 244 |
arches = cfg.architectures.join(" "), |
| 245 |
component = cfg.component, |
| 246 |
md5 = sums(&|b| hex::encode(md5::Md5::digest(b))), |
| 247 |
sha256 = sums(&|b| hex::encode(sha2::Sha256::digest(b))), |
| 248 |
); |
| 249 |
let in_release = gpg(app, &["--clearsign", "--digest-algo", "SHA512"], release.as_bytes()).await?; |
| 250 |
let release_gpg = gpg(app, &["--detach-sign", "--armor", "--digest-algo", "SHA512"], release.as_bytes()).await?; |
| 251 |
let i = Arc::new(Index { key, files, release, in_release, release_gpg, pool }); |
| 252 |
*built().lock().unwrap() = Some(i.clone()); |
| 253 |
Ok(i) |
| 254 |
} |
| 255 |
|
| 256 |
|
| 257 |
|
| 258 |
fn enabled(app: &Shared) -> bool { |
| 259 |
!app.config.apt.repos.is_empty() |
| 260 |
} |
| 261 |
|
| 262 |
fn reply(ct: &str, bytes: Vec<u8>) -> Response { |
| 263 |
([(header::CONTENT_TYPE, ct.to_string()), (header::CACHE_CONTROL, "no-cache".to_string())], bytes).into_response() |
| 264 |
} |
| 265 |
|
| 266 |
fn not_found() -> Response { |
| 267 |
(StatusCode::NOT_FOUND, "not found\n").into_response() |
| 268 |
} |
| 269 |
|
| 270 |
fn failed(why: String) -> Response { |
| 271 |
tracing::warn!("apt: {why}"); |
| 272 |
(StatusCode::SERVICE_UNAVAILABLE, "the package index is not available just now\n").into_response() |
| 273 |
} |
| 274 |
|
| 275 |
|
| 276 |
pub async fn home(State(app): State<Shared>) -> Response { |
| 277 |
if !enabled(&app) { |
| 278 |
return not_found(); |
| 279 |
} |
| 280 |
let base = app.config.base_url(); |
| 281 |
let cfg = &app.config.apt; |
| 282 |
let text = format!( |
| 283 |
"{site} apt repository: oxy and oh for Debian and Ubuntu ({arches}).\n\n\ |
| 284 |
Installing a .deb from a release adds this repository by itself. To add it by hand:\n\n\ |
| 72cde3da Andy McCall ↶ |
285 |
curl -fsSL {base}/apt/oxygenhub.gpg | sudo tee /usr/share/keyrings/oxygenhub-archive-keyring.gpg >/dev/null\n\ |
| ea85398a Andy McCall ↶ |
286 |
sudo tee /etc/apt/sources.list.d/oxygenhub.sources >/dev/null <<EOF\n\ |
| 287 |
Types: deb\nURIs: {base}/apt\nSuites: {suite}\nComponents: {component}\n\ |
| 288 |
Signed-By: /usr/share/keyrings/oxygenhub-archive-keyring.gpg\nEOF\n\ |
| 289 |
sudo apt update && sudo apt install oxy oh\n\n\ |
| 290 |
The signing key's fingerprint is in {base}/apt/oxygenhub.asc (gpg --show-keys).\n", |
| 291 |
site = app.config.site.name, |
| 292 |
arches = cfg.architectures.join(", "), |
| 293 |
suite = cfg.suite, |
| 294 |
component = cfg.component, |
| 295 |
); |
| 296 |
reply("text/plain; charset=utf-8", text.into_bytes()) |
| 297 |
} |
| 298 |
|
| 299 |
|
| 300 |
pub async fn key(State(app): State<Shared>, Path(file): Path<String>) -> Response { |
| 301 |
if !enabled(&app) { |
| 302 |
return not_found(); |
| 303 |
} |
| 304 |
let armor = match file.as_str() { |
| 305 |
"oxygenhub.gpg" => false, |
| 306 |
"oxygenhub.asc" => true, |
| 307 |
_ => return not_found(), |
| 308 |
}; |
| 309 |
let args: &[&str] = if armor { &["--armor", "--export"] } else { &["--export"] }; |
| 310 |
match gpg(&app, args, b"").await { |
| 311 |
Ok(k) if !k.is_empty() => reply(if armor { "text/plain; charset=utf-8" } else { "application/pgp-keys" }, k), |
| 312 |
Ok(_) => failed("the signing key is missing".into()), |
| 313 |
Err(e) => failed(e), |
| 314 |
} |
| 315 |
} |
| 316 |
|
| 317 |
|
| 318 |
pub async fn dists(State(app): State<Shared>, Path((suite, rest)): Path<(String, String)>) -> Response { |
| 319 |
if !enabled(&app) || suite != app.config.apt.suite { |
| 320 |
return not_found(); |
| 321 |
} |
| 322 |
let i = match index(&app).await { |
| 323 |
Ok(i) => i, |
| 324 |
Err(e) => return failed(e), |
| 325 |
}; |
| 326 |
match rest.as_str() { |
| 327 |
"InRelease" => reply("text/plain; charset=utf-8", i.in_release.clone()), |
| 328 |
"Release" => reply("text/plain; charset=utf-8", i.release.clone().into_bytes()), |
| 329 |
"Release.gpg" => reply("application/pgp-signature", i.release_gpg.clone()), |
| 330 |
path => match i.files.get(path) { |
| 331 |
Some(b) if path.ends_with(".gz") => reply("application/gzip", b.clone()), |
| 332 |
Some(b) => reply("text/plain; charset=utf-8", b.clone()), |
| 333 |
None => not_found(), |
| 334 |
}, |
| 335 |
} |
| 336 |
} |
| 337 |
|
| 338 |
|
| 339 |
pub async fn pool(State(app): State<Shared>, Path(path): Path<String>) -> Response { |
| 340 |
if !enabled(&app) { |
| 341 |
return not_found(); |
| 342 |
} |
| 343 |
let i = match index(&app).await { |
| 344 |
Ok(i) => i, |
| 345 |
Err(e) => return failed(e), |
| 346 |
}; |
| 347 |
let file = path.rsplit('/').next().unwrap_or(""); |
| 348 |
let Some(hash) = i.pool.get(file) else { return not_found() }; |
| 349 |
match tokio::fs::read(crate::web::releases::blob_path(&app, hash)).await { |
| 350 |
Ok(bytes) => ( |
| 351 |
[ |
| 352 |
(header::CONTENT_TYPE, "application/vnd.debian.binary-package".to_string()), |
| 353 |
(header::CACHE_CONTROL, "public, max-age=31536000, immutable".to_string()), |
| 354 |
], |
| 355 |
bytes, |
| 356 |
) |
| 357 |
.into_response(), |
| 358 |
Err(_) => not_found(), |
| 359 |
} |
| 360 |
} |
| 361 |
|
| 362 |
#[cfg(test)] |
| 363 |
mod tests { |
| 364 |
use super::*; |
| 365 |
|
| 366 |
|
| 367 |
|
| 368 |
pub fn deb(control: &str, member: &str) -> Vec<u8> { |
| 369 |
let mut tar_bytes = Vec::new(); |
| 370 |
{ |
| 371 |
let mut t = tar::Builder::new(&mut tar_bytes); |
| 372 |
let mut h = tar::Header::new_gnu(); |
| 373 |
h.set_size(control.len() as u64); |
| 374 |
h.set_mode(0o644); |
| 375 |
h.set_cksum(); |
| 376 |
t.append_data(&mut h, "./control", control.as_bytes()).unwrap(); |
| 377 |
t.finish().unwrap(); |
| 378 |
} |
| 379 |
let packed = match member { |
| 380 |
"control.tar.gz" => { |
| 381 |
let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); |
| 382 |
std::io::Write::write_all(&mut gz, &tar_bytes).unwrap(); |
| 383 |
gz.finish().unwrap() |
| 384 |
} |
| 385 |
"control.tar.xz" => { |
| 386 |
let mut out = Vec::new(); |
| 387 |
lzma_rs::xz_compress(&mut tar_bytes.as_slice(), &mut out).unwrap(); |
| 388 |
out |
| 389 |
} |
| 390 |
"control.tar.zst" => zstd::encode_all(tar_bytes.as_slice(), 3).unwrap(), |
| 391 |
_ => tar_bytes, |
| 392 |
}; |
| 393 |
let mut out = b"!<arch>\n".to_vec(); |
| 394 |
for (name, data) in [("debian-binary", b"2.0\n".to_vec()), (member, packed), ("data.tar.gz", vec![0u8; 3])] { |
| 395 |
out.extend_from_slice(format!("{:<16}{:<12}{:<6}{:<6}{:<8}{:<10}`\n", name, 0, 0, 0, 100644, data.len()).as_bytes()); |
| 396 |
out.extend_from_slice(&data); |
| 397 |
if data.len() % 2 == 1 { |
| 398 |
out.push(b'\n'); |
| 399 |
} |
| 400 |
} |
| 401 |
out |
| 402 |
} |
| 403 |
|
| 404 |
#[test] |
| 405 |
fn control_is_read_from_every_compression() { |
| 406 |
let control = "Package: oxy\nVersion: 0.3.0\nArchitecture: amd64\nDescription: test\n"; |
| 407 |
for m in ["control.tar", "control.tar.gz", "control.tar.xz", "control.tar.zst"] { |
| 408 |
let e = entry_of("oxy_0.3.0_amd64.deb", &deb(control, m)).unwrap(); |
| 409 |
assert_eq!((e.package.as_str(), e.architecture.as_str()), ("oxy", "amd64"), "{m}"); |
| 410 |
assert!(e.control.contains("Version: 0.3.0")); |
| 411 |
} |
| 412 |
assert!(deb_control(b"not a deb").is_err()); |
| 413 |
assert_eq!( |
| 414 |
pool_path("main", &entry_of("oxy_0.3.0_amd64.deb", &deb(control, "control.tar.gz")).unwrap()), |
| 415 |
"pool/main/o/oxy/oxy_0.3.0_amd64.deb" |
| 416 |
); |
| 417 |
assert_eq!(rfc2822(0), "Thu, 01 Jan 1970 00:00:00 UTC"); |
| 418 |
} |
| 419 |
} |