//! The apt repository at `/apt` (oxygenhub#85): the `.deb` assets of the //! configured repositories' releases, indexed and signed so that //! `apt install oxy oh` and `apt upgrade` work. Nothing is stored apart: //! the index is built from the releases when they change, and the pool //! serves the assets' own bytes. use crate::{db, Shared}; use axum::extract::{Path, State}; use axum::http::{header, StatusCode}; use axum::response::{IntoResponse, Response}; use sha2::Digest; use std::collections::{BTreeMap, HashMap}; use std::io::Read; use std::sync::{Arc, Mutex, OnceLock}; /// One package as the index lists it. #[derive(Clone, Debug)] struct Entry { /// The control paragraph, as the package has it. control: String, package: String, architecture: String, /// The asset's name, which is its pool file name. file: String, size: u64, md5: String, sha256: String, } /// The signed index, built from one set of assets. struct Index { /// What it was built from: each `.deb` asset's id and hash. key: String, /// `main/binary-ARCH/Packages` and `.gz`, by their path under the suite. files: BTreeMap>, release: String, in_release: Vec, release_gpg: Vec, /// Pool file name to the asset's blob hash. pool: HashMap, } /// Parsed `.deb`s by blob hash: a package is read once. fn parsed() -> &'static Mutex> { static P: OnceLock>> = OnceLock::new(); P.get_or_init(Default::default) } fn built() -> &'static Mutex>> { static B: OnceLock>>> = OnceLock::new(); B.get_or_init(Default::default) } // ------------------------------------------------------------- .deb files /// The members of an `ar` archive, by name. fn ar_members(bytes: &[u8]) -> Result, String> { let body = bytes.strip_prefix(b"!\n").ok_or("not an ar archive")?; let mut out = Vec::new(); let mut at = 0; while at + 60 <= body.len() { let head = &body[at..at + 60]; let name = String::from_utf8_lossy(&head[..16]).trim().trim_end_matches('/').to_string(); let size: usize = String::from_utf8_lossy(&head[48..58]).trim().parse().map_err(|_| "bad ar member size")?; let start = at + 60; let end = start.checked_add(size).filter(|e| *e <= body.len()).ok_or("truncated ar archive")?; out.push((name, &body[start..end])); at = end + (size & 1); } Ok(out) } /// The `control` file of a `.deb`. pub fn deb_control(bytes: &[u8]) -> Result { let members = ar_members(bytes)?; let (name, data) = members.into_iter().find(|(n, _)| n.starts_with("control.tar")).ok_or("no control.tar member")?; let mut tar_bytes = Vec::new(); match name.as_str() { "control.tar" => tar_bytes.extend_from_slice(data), "control.tar.gz" => { flate2::read::GzDecoder::new(data).read_to_end(&mut tar_bytes).map_err(|e| e.to_string())?; } "control.tar.xz" => lzma_rs::xz_decompress(&mut std::io::BufReader::new(data), &mut tar_bytes).map_err(|e| e.to_string())?, "control.tar.zst" => tar_bytes = zstd::decode_all(data).map_err(|e| e.to_string())?, other => return Err(format!("{other}: a compression this hub does not read")), } let mut archive = tar::Archive::new(tar_bytes.as_slice()); for entry in archive.entries().map_err(|e| e.to_string())? { let mut entry = entry.map_err(|e| e.to_string())?; let path = entry.path().map_err(|e| e.to_string())?.to_string_lossy().to_string(); if path.trim_start_matches("./") == "control" { let mut text = String::new(); entry.read_to_string(&mut text).map_err(|e| e.to_string())?; return Ok(text); } } Err("no control file".into()) } /// A control paragraph's field. fn field<'a>(control: &'a str, name: &str) -> Option<&'a str> { control.lines().find_map(|l| l.strip_prefix(name).and_then(|r| r.strip_prefix(':')).map(str::trim)) } fn entry_of(file: &str, bytes: &[u8]) -> Result { let control = deb_control(bytes)?; let package = field(&control, "Package").ok_or("no Package field")?.to_string(); let architecture = field(&control, "Architecture").ok_or("no Architecture field")?.to_string(); field(&control, "Version").ok_or("no Version field")?; Ok(Entry { control: control.trim_end().to_string(), package, architecture, file: file.to_string(), size: bytes.len() as u64, md5: hex::encode(md5::Md5::digest(bytes)), sha256: hex::encode(sha2::Sha256::digest(bytes)), }) } // ------------------------------------------------------------------ index /// The `.deb` assets of the configured public repositories: (asset id, /// name, blob hash). async fn debs(app: &Shared) -> Result, sqlx::Error> { let mut out = Vec::new(); for slug in &app.config.apt.repos { let Some((owner, name)) = slug.split_once('/') else { continue }; let Some(row) = db::repos::get(&app.db, owner, name).await? else { continue }; if row.visibility() != hub_api::Visibility::Public { tracing::warn!("apt: {slug} is not public, so its packages are not served"); continue; } let rows: Vec<(i64, String, String)> = sqlx::query_as( "SELECT a.id, a.name, a.hash FROM release_assets a JOIN releases r ON r.id = a.release_id \ WHERE r.repo_id = $1 AND a.name LIKE '%.deb' ORDER BY a.id", ) .bind(row.id) .fetch_all(&app.db) .await?; out.extend(rows); } Ok(out) } /// The pool path of a package: `pool/COMPONENT/p/PACKAGE/FILE`. fn pool_path(component: &str, e: &Entry) -> String { let first: String = e.package.chars().take(1).collect(); format!("pool/{component}/{first}/{}/{}", e.package, e.file) } /// RFC 2822 date in UTC, as Release files carry it. fn rfc2822(secs: i64) -> String { let t = time::OffsetDateTime::from_unix_timestamp(secs).unwrap_or(time::OffsetDateTime::UNIX_EPOCH); let fmt = time::macros::format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] UTC"); t.format(&fmt).unwrap_or_default() } /// Run gpg with the repository's key on `input`. async fn gpg(app: &Shared, args: &[&str], input: &[u8]) -> Result, String> { use tokio::io::AsyncWriteExt; let home = app.config.data.join(&app.config.apt.gnupg_home); let mut child = tokio::process::Command::new("gpg") .arg("--homedir") .arg(&home) .args(["--batch", "--yes", "--quiet", "--no-tty", "--pinentry-mode", "loopback", "--passphrase", ""]) .args(args) .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn() .map_err(|e| format!("gpg: {e}"))?; let mut stdin = child.stdin.take().expect("piped"); stdin.write_all(input).await.map_err(|e| e.to_string())?; drop(stdin); let out = child.wait_with_output().await.map_err(|e| e.to_string())?; if !out.status.success() { return Err(format!("gpg {}: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim())); } Ok(out.stdout) } /// The index for the assets there are now, built when they changed. async fn index(app: &Shared) -> Result, String> { let assets = debs(app).await.map_err(|e| e.to_string())?; let key: String = assets.iter().map(|(id, _, h)| format!("{id}:{h};")).collect(); if let Some(i) = built().lock().unwrap().as_ref().filter(|i| i.key == key) { return Ok(i.clone()); } let cfg = &app.config.apt; let mut entries: Vec = Vec::new(); let mut pool = HashMap::new(); for (_, name, hash) in &assets { let known = parsed().lock().unwrap().get(hash).cloned(); let e = match known { Some(e) => Entry { file: name.clone(), ..e }, None => { let bytes = tokio::fs::read(crate::web::releases::blob_path(app, hash)).await.map_err(|e| e.to_string())?; match entry_of(name, &bytes) { Ok(e) => { parsed().lock().unwrap().insert(hash.clone(), e.clone()); e } Err(why) => { tracing::warn!("apt: {name} is left out: {why}"); continue; } } } }; if pool.contains_key(&e.file) { continue; } pool.insert(e.file.clone(), hash.clone()); entries.push(e); } let mut files = BTreeMap::new(); for arch in &cfg.architectures { let mut text = String::new(); for e in entries.iter().filter(|e| e.architecture == *arch || e.architecture == "all") { text.push_str(&format!( "{}\nFilename: {}\nSize: {}\nMD5sum: {}\nSHA256: {}\n\n", e.control, pool_path(&cfg.component, e), e.size, e.md5, e.sha256 )); } let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best()); std::io::Write::write_all(&mut gz, text.as_bytes()).map_err(|e| e.to_string())?; let gz = gz.finish().map_err(|e| e.to_string())?; files.insert(format!("{}/binary-{arch}/Packages", cfg.component), text.into_bytes()); files.insert(format!("{}/binary-{arch}/Packages.gz", cfg.component), gz); } let sums = |f: &dyn Fn(&[u8]) -> String| -> String { files.iter().map(|(path, bytes)| format!(" {} {:>8} {path}\n", f(bytes), bytes.len())).collect() }; let release = format!( "Origin: {site}\nLabel: {site}\nSuite: {suite}\nCodename: {suite}\nDate: {date}\nArchitectures: {arches}\nComponents: {component}\nDescription: {site} packages (oxy and oh)\nMD5Sum:\n{md5}SHA256:\n{sha256}", site = app.config.site.name, suite = cfg.suite, date = rfc2822(db::now_ms() / 1000), arches = cfg.architectures.join(" "), component = cfg.component, md5 = sums(&|b| hex::encode(md5::Md5::digest(b))), sha256 = sums(&|b| hex::encode(sha2::Sha256::digest(b))), ); let in_release = gpg(app, &["--clearsign", "--digest-algo", "SHA512"], release.as_bytes()).await?; let release_gpg = gpg(app, &["--detach-sign", "--armor", "--digest-algo", "SHA512"], release.as_bytes()).await?; let i = Arc::new(Index { key, files, release, in_release, release_gpg, pool }); *built().lock().unwrap() = Some(i.clone()); Ok(i) } // ----------------------------------------------------------------- routes fn enabled(app: &Shared) -> bool { !app.config.apt.repos.is_empty() } fn reply(ct: &str, bytes: Vec) -> Response { ([(header::CONTENT_TYPE, ct.to_string()), (header::CACHE_CONTROL, "no-cache".to_string())], bytes).into_response() } fn not_found() -> Response { (StatusCode::NOT_FOUND, "not found\n").into_response() } fn failed(why: String) -> Response { tracing::warn!("apt: {why}"); (StatusCode::SERVICE_UNAVAILABLE, "the package index is not available just now\n").into_response() } /// `GET /apt`: how to use it. pub async fn home(State(app): State) -> Response { if !enabled(&app) { return not_found(); } let base = app.config.base_url(); let cfg = &app.config.apt; let text = format!( "{site} apt repository: oxy and oh for Debian and Ubuntu ({arches}).\n\n\ Installing a .deb from a release adds this repository by itself. To add it by hand:\n\n\ sudo curl -fsSLo /usr/share/keyrings/oxygenhub-archive-keyring.gpg {base}/apt/oxygenhub.gpg\n\ sudo tee /etc/apt/sources.list.d/oxygenhub.sources >/dev/null <, Path(file): Path) -> Response { if !enabled(&app) { return not_found(); } let armor = match file.as_str() { "oxygenhub.gpg" => false, "oxygenhub.asc" => true, _ => return not_found(), }; let args: &[&str] = if armor { &["--armor", "--export"] } else { &["--export"] }; match gpg(&app, args, b"").await { Ok(k) if !k.is_empty() => reply(if armor { "text/plain; charset=utf-8" } else { "application/pgp-keys" }, k), Ok(_) => failed("the signing key is missing".into()), Err(e) => failed(e), } } /// `GET /apt/dists/SUITE/…`: the Release files and the Packages indexes. pub async fn dists(State(app): State, Path((suite, rest)): Path<(String, String)>) -> Response { if !enabled(&app) || suite != app.config.apt.suite { return not_found(); } let i = match index(&app).await { Ok(i) => i, Err(e) => return failed(e), }; match rest.as_str() { "InRelease" => reply("text/plain; charset=utf-8", i.in_release.clone()), "Release" => reply("text/plain; charset=utf-8", i.release.clone().into_bytes()), "Release.gpg" => reply("application/pgp-signature", i.release_gpg.clone()), path => match i.files.get(path) { Some(b) if path.ends_with(".gz") => reply("application/gzip", b.clone()), Some(b) => reply("text/plain; charset=utf-8", b.clone()), None => not_found(), }, } } /// `GET /apt/pool/…/FILE`: a package's bytes. pub async fn pool(State(app): State, Path(path): Path) -> Response { if !enabled(&app) { return not_found(); } let i = match index(&app).await { Ok(i) => i, Err(e) => return failed(e), }; let file = path.rsplit('/').next().unwrap_or(""); let Some(hash) = i.pool.get(file) else { return not_found() }; match tokio::fs::read(crate::web::releases::blob_path(&app, hash)).await { Ok(bytes) => ( [ (header::CONTENT_TYPE, "application/vnd.debian.binary-package".to_string()), (header::CACHE_CONTROL, "public, max-age=31536000, immutable".to_string()), ], bytes, ) .into_response(), Err(_) => not_found(), } } #[cfg(test)] mod tests { use super::*; /// A minimal `.deb`: an ar archive with a control archive compressed /// as `name` says. pub fn deb(control: &str, member: &str) -> Vec { let mut tar_bytes = Vec::new(); { let mut t = tar::Builder::new(&mut tar_bytes); let mut h = tar::Header::new_gnu(); h.set_size(control.len() as u64); h.set_mode(0o644); h.set_cksum(); t.append_data(&mut h, "./control", control.as_bytes()).unwrap(); t.finish().unwrap(); } let packed = match member { "control.tar.gz" => { let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); std::io::Write::write_all(&mut gz, &tar_bytes).unwrap(); gz.finish().unwrap() } "control.tar.xz" => { let mut out = Vec::new(); lzma_rs::xz_compress(&mut tar_bytes.as_slice(), &mut out).unwrap(); out } "control.tar.zst" => zstd::encode_all(tar_bytes.as_slice(), 3).unwrap(), _ => tar_bytes, }; let mut out = b"!\n".to_vec(); for (name, data) in [("debian-binary", b"2.0\n".to_vec()), (member, packed), ("data.tar.gz", vec![0u8; 3])] { out.extend_from_slice(format!("{:<16}{:<12}{:<6}{:<6}{:<8}{:<10}`\n", name, 0, 0, 0, 100644, data.len()).as_bytes()); out.extend_from_slice(&data); if data.len() % 2 == 1 { out.push(b'\n'); } } out } #[test] fn control_is_read_from_every_compression() { let control = "Package: oxy\nVersion: 0.3.0\nArchitecture: amd64\nDescription: test\n"; for m in ["control.tar", "control.tar.gz", "control.tar.xz", "control.tar.zst"] { let e = entry_of("oxy_0.3.0_amd64.deb", &deb(control, m)).unwrap(); assert_eq!((e.package.as_str(), e.architecture.as_str()), ("oxy", "amd64"), "{m}"); assert!(e.control.contains("Version: 0.3.0")); } assert!(deb_control(b"not a deb").is_err()); assert_eq!( pool_path("main", &entry_of("oxy_0.3.0_amd64.deb", &deb(control, "control.tar.gz")).unwrap()), "pool/main/o/oxy/oxy_0.3.0_amd64.deb" ); assert_eq!(rfc2822(0), "Thu, 01 Jan 1970 00:00:00 UTC"); } }