apt.rs 38.6%

122 of 316 lines run, 18 of 104 functions · run #492 on proposal/340 · run missed

//! The apt repository at `/apt` (oxygenhub#85): the `.deb` assets of the
//! configured repositories' releases, indexed and signed so that
//! `apt install oxy oh` and `apt upgrade` work. Nothing is stored apart:
//! the index is built from the releases when they change, and the pool
//! serves the assets' own bytes.

use crate::{db, Shared};
use axum::extract::{Path, State};
use axum::http::{header, StatusCode};
use axum::response::{IntoResponse, Response};
use sha2::Digest;
use std::collections::{BTreeMap, HashMap};
use std::io::Read;
use std::sync::{Arc, Mutex, OnceLock};

/// One package as the index lists it.
#[derive(Clone, Debug)]
struct Entry {
    /// The control paragraph, as the package has it.
    control: String,
    package: String,
    architecture: String,
    /// The asset's name, which is its pool file name.
    file: String,
    size: u64,
    md5: String,
    sha256: String,
}

/// The signed index, built from one set of assets.
struct Index {
    /// What it was built from: each `.deb` asset's id and hash.
    key: String,
    /// `main/binary-ARCH/Packages` and `.gz`, by their path under the suite.
    files: BTreeMap<String, Vec<u8>>,
    release: String,
    in_release: Vec<u8>,
    release_gpg: Vec<u8>,
    /// Pool file name to the asset's blob hash.
    pool: HashMap<String, String>,
}

/// Parsed `.deb`s by blob hash: a package is read once.
fn parsed() -> &'static Mutex<HashMap<String, Entry>> {
    static P: OnceLock<Mutex<HashMap<String, Entry>>> = OnceLock::new();
    P.get_or_init(Default::default)
}

fn built() -> &'static Mutex<Option<Arc<Index>>> {
    static B: OnceLock<Mutex<Option<Arc<Index>>>> = OnceLock::new();
    B.get_or_init(Default::default)
}

// ------------------------------------------------------------- .deb files

/// The members of an `ar` archive, by name.
fn ar_members(bytes: &[u8]) -> Result<Vec<(String, &[u8])>, String> {
    let body = bytes.strip_prefix(b"!<arch>\n").ok_or("not an ar archive")?;
    let mut out = Vec::new();
    let mut at = 0;
    while at + 60 <= body.len() {
        let head = &body[at..at + 60];
        let name = String::from_utf8_lossy(&head[..16]).trim().trim_end_matches('/').to_string();
        let size: usize = String::from_utf8_lossy(&head[48..58]).trim().parse().map_err(|_| "bad ar member size")?;
        let start = at + 60;
        let end = start.checked_add(size).filter(|e| *e <= body.len()).ok_or("truncated ar archive")?;
        out.push((name, &body[start..end]));
        at = end + (size & 1);
    }
    Ok(out)
}

/// The `control` file of a `.deb`.
pub fn deb_control(bytes: &[u8]) -> Result<String, String> {
    let members = ar_members(bytes)?;
    let (name, data) = members.into_iter().find(|(n, _)| n.starts_with("control.tar")).ok_or("no control.tar member")?;
    let mut tar_bytes = Vec::new();
    match name.as_str() {
        "control.tar" => tar_bytes.extend_from_slice(data),
        "control.tar.gz" => {
            flate2::read::GzDecoder::new(data).read_to_end(&mut tar_bytes).map_err(|e| e.to_string())?;
        }
        "control.tar.xz" => lzma_rs::xz_decompress(&mut std::io::BufReader::new(data), &mut tar_bytes).map_err(|e| e.to_string())?,
        "control.tar.zst" => tar_bytes = zstd::decode_all(data).map_err(|e| e.to_string())?,
        other => return Err(format!("{other}: a compression this hub does not read")),
    }
    let mut archive = tar::Archive::new(tar_bytes.as_slice());
    for entry in archive.entries().map_err(|e| e.to_string())? {
        let mut entry = entry.map_err(|e| e.to_string())?;
        let path = entry.path().map_err(|e| e.to_string())?.to_string_lossy().to_string();
        if path.trim_start_matches("./") == "control" {
            let mut text = String::new();
            entry.read_to_string(&mut text).map_err(|e| e.to_string())?;
            return Ok(text);
        }
    }
    Err("no control file".into())
}

/// A control paragraph's field.
fn field<'a>(control: &'a str, name: &str) -> Option<&'a str> {
    control.lines().find_map(|l| l.strip_prefix(name).and_then(|r| r.strip_prefix(':')).map(str::trim))
}

fn entry_of(file: &str, bytes: &[u8]) -> Result<Entry, String> {
    let control = deb_control(bytes)?;
    let package = field(&control, "Package").ok_or("no Package field")?.to_string();
    let architecture = field(&control, "Architecture").ok_or("no Architecture field")?.to_string();
    field(&control, "Version").ok_or("no Version field")?;
    Ok(Entry {
        control: control.trim_end().to_string(),
        package,
        architecture,
        file: file.to_string(),
        size: bytes.len() as u64,
        md5: hex::encode(md5::Md5::digest(bytes)),
        sha256: hex::encode(sha2::Sha256::digest(bytes)),
    })
}

// ------------------------------------------------------------------ index

/// The `.deb` assets of the configured public repositories: (asset id,
/// name, blob hash).
async fn debs(app: &Shared) -> Result<Vec<(i64, String, String)>, sqlx::Error> {
    let mut out = Vec::new();
    for slug in &app.config.apt.repos {
        let Some((owner, name)) = slug.split_once('/') else { continue };
        let Some(row) = db::repos::get(&app.db, owner, name).await? else { continue };
        if row.visibility() != hub_api::Visibility::Public {
            tracing::warn!("apt: {slug} is not public, so its packages are not served");
            continue;
        }
        let rows: Vec<(i64, String, String)> = sqlx::query_as(
            "SELECT a.id, a.name, a.hash FROM release_assets a JOIN releases r ON r.id = a.release_id \
             WHERE r.repo_id = $1 AND a.name LIKE '%.deb' ORDER BY a.id",
        )
        .bind(row.id)
        .fetch_all(&app.db)
        .await?;
        out.extend(rows);
    }
    Ok(out)
}

/// The pool path of a package: `pool/COMPONENT/p/PACKAGE/FILE`.
fn pool_path(component: &str, e: &Entry) -> String {
    let first: String = e.package.chars().take(1).collect();
    format!("pool/{component}/{first}/{}/{}", e.package, e.file)
}

/// RFC 2822 date in UTC, as Release files carry it.
fn rfc2822(secs: i64) -> String {
    let t = time::OffsetDateTime::from_unix_timestamp(secs).unwrap_or(time::OffsetDateTime::UNIX_EPOCH);
    let fmt = time::macros::format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] UTC");
    t.format(&fmt).unwrap_or_default()
}

/// Run gpg with the repository's key on `input`.
async fn gpg(app: &Shared, args: &[&str], input: &[u8]) -> Result<Vec<u8>, String> {
    use tokio::io::AsyncWriteExt;
    let home = app.config.data.join(&app.config.apt.gnupg_home);
    let mut child = tokio::process::Command::new("gpg")
        .arg("--homedir")
        .arg(&home)
        .args(["--batch", "--yes", "--quiet", "--no-tty", "--pinentry-mode", "loopback", "--passphrase", ""])
        .args(args)
        .stdin(std::process::Stdio::piped())
        .stdout(std::process::Stdio::piped())
        .stderr(std::process::Stdio::piped())
        .spawn()
        .map_err(|e| format!("gpg: {e}"))?;
    let mut stdin = child.stdin.take().expect("piped");
    stdin.write_all(input).await.map_err(|e| e.to_string())?;
    drop(stdin);
    let out = child.wait_with_output().await.map_err(|e| e.to_string())?;
    if !out.status.success() {
        return Err(format!("gpg {}: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim()));
    }
    Ok(out.stdout)
}

/// The index for the assets there are now, built when they changed.
async fn index(app: &Shared) -> Result<Arc<Index>, String> {
    let assets = debs(app).await.map_err(|e| e.to_string())?;
    let key: String = assets.iter().map(|(id, _, h)| format!("{id}:{h};")).collect();
    if let Some(i) = built().lock().unwrap().as_ref().filter(|i| i.key == key) {
        return Ok(i.clone());
    }
    let cfg = &app.config.apt;
    let mut entries: Vec<Entry> = Vec::new();
    let mut pool = HashMap::new();
    for (_, name, hash) in &assets {
        let known = parsed().lock().unwrap().get(hash).cloned();
        let e = match known {
            Some(e) => Entry { file: name.clone(), ..e },
            None => {
                let bytes = tokio::fs::read(crate::web::releases::blob_path(app, hash)).await.map_err(|e| e.to_string())?;
                match entry_of(name, &bytes) {
                    Ok(e) => {
                        parsed().lock().unwrap().insert(hash.clone(), e.clone());
                        e
                    }
                    Err(why) => {
                        tracing::warn!("apt: {name} is left out: {why}");
                        continue;
                    }
                }
            }
        };
        if pool.contains_key(&e.file) {
            continue;
        }
        pool.insert(e.file.clone(), hash.clone());
        entries.push(e);
    }
    let mut files = BTreeMap::new();
    for arch in &cfg.architectures {
        let mut text = String::new();
        for e in entries.iter().filter(|e| e.architecture == *arch || e.architecture == "all") {
            text.push_str(&format!(
                "{}\nFilename: {}\nSize: {}\nMD5sum: {}\nSHA256: {}\n\n",
                e.control,
                pool_path(&cfg.component, e),
                e.size,
                e.md5,
                e.sha256
            ));
        }
        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best());
        std::io::Write::write_all(&mut gz, text.as_bytes()).map_err(|e| e.to_string())?;
        let gz = gz.finish().map_err(|e| e.to_string())?;
        files.insert(format!("{}/binary-{arch}/Packages", cfg.component), text.into_bytes());
        files.insert(format!("{}/binary-{arch}/Packages.gz", cfg.component), gz);
    }
    let sums = |f: &dyn Fn(&[u8]) -> String| -> String {
        files.iter().map(|(path, bytes)| format!(" {} {:>8} {path}\n", f(bytes), bytes.len())).collect()
    };
    let release = format!(
        "Origin: {site}\nLabel: {site}\nSuite: {suite}\nCodename: {suite}\nDate: {date}\nArchitectures: {arches}\nComponents: {component}\nDescription: {site} packages (oxy and oh)\nMD5Sum:\n{md5}SHA256:\n{sha256}",
        site = app.config.site.name,
        suite = cfg.suite,
        date = rfc2822(db::now_ms() / 1000),
        arches = cfg.architectures.join(" "),
        component = cfg.component,
        md5 = sums(&|b| hex::encode(md5::Md5::digest(b))),
        sha256 = sums(&|b| hex::encode(sha2::Sha256::digest(b))),
    );
    let in_release = gpg(app, &["--clearsign", "--digest-algo", "SHA512"], release.as_bytes()).await?;
    let release_gpg = gpg(app, &["--detach-sign", "--armor", "--digest-algo", "SHA512"], release.as_bytes()).await?;
    let i = Arc::new(Index { key, files, release, in_release, release_gpg, pool });
    *built().lock().unwrap() = Some(i.clone());
    Ok(i)
}

// ----------------------------------------------------------------- routes

fn enabled(app: &Shared) -> bool {
    !app.config.apt.repos.is_empty()
}

fn reply(ct: &str, bytes: Vec<u8>) -> Response {
    ([(header::CONTENT_TYPE, ct.to_string()), (header::CACHE_CONTROL, "no-cache".to_string())], bytes).into_response()
}

fn not_found() -> Response {
    (StatusCode::NOT_FOUND, "not found\n").into_response()
}

fn failed(why: String) -> Response {
    tracing::warn!("apt: {why}");
    (StatusCode::SERVICE_UNAVAILABLE, "the package index is not available just now\n").into_response()
}

/// `GET /apt`: how to use it.
pub async fn home(State(app): State<Shared>) -> Response {
    if !enabled(&app) {
        return not_found();
    }
    let base = app.config.base_url();
    let cfg = &app.config.apt;
    let text = format!(
        "{site} apt repository: oxy and oh for Debian and Ubuntu ({arches}).\n\n\
         Installing a .deb from a release adds this repository by itself. To add it by hand:\n\n\
         curl -fsSL {base}/apt/oxygenhub.gpg | sudo tee /usr/share/keyrings/oxygenhub-archive-keyring.gpg >/dev/null\n\
         sudo tee /etc/apt/sources.list.d/oxygenhub.sources >/dev/null <<EOF\n\
         Types: deb\nURIs: {base}/apt\nSuites: {suite}\nComponents: {component}\n\
         Signed-By: /usr/share/keyrings/oxygenhub-archive-keyring.gpg\nEOF\n\
         sudo apt update && sudo apt install oxy oh\n\n\
         The signing key's fingerprint is in {base}/apt/oxygenhub.asc (gpg --show-keys).\n",
        site = app.config.site.name,
        arches = cfg.architectures.join(", "),
        suite = cfg.suite,
        component = cfg.component,
    );
    reply("text/plain; charset=utf-8", text.into_bytes())
}

/// `GET /apt/oxygenhub.gpg` (binary keyring) and `/apt/oxygenhub.asc`.
pub async fn key(State(app): State<Shared>, Path(file): Path<String>) -> Response {
    if !enabled(&app) {
        return not_found();
    }
    let armor = match file.as_str() {
        "oxygenhub.gpg" => false,
        "oxygenhub.asc" => true,
        _ => return not_found(),
    };
    let args: &[&str] = if armor { &["--armor", "--export"] } else { &["--export"] };
    match gpg(&app, args, b"").await {
        Ok(k) if !k.is_empty() => reply(if armor { "text/plain; charset=utf-8" } else { "application/pgp-keys" }, k),
        Ok(_) => failed("the signing key is missing".into()),
        Err(e) => failed(e),
    }
}

/// `GET /apt/dists/SUITE/…`: the Release files and the Packages indexes.
pub async fn dists(State(app): State<Shared>, Path((suite, rest)): Path<(String, String)>) -> Response {
    if !enabled(&app) || suite != app.config.apt.suite {
        return not_found();
    }
    let i = match index(&app).await {
        Ok(i) => i,
        Err(e) => return failed(e),
    };
    match rest.as_str() {
        "InRelease" => reply("text/plain; charset=utf-8", i.in_release.clone()),
        "Release" => reply("text/plain; charset=utf-8", i.release.clone().into_bytes()),
        "Release.gpg" => reply("application/pgp-signature", i.release_gpg.clone()),
        path => match i.files.get(path) {
            Some(b) if path.ends_with(".gz") => reply("application/gzip", b.clone()),
            Some(b) => reply("text/plain; charset=utf-8", b.clone()),
            None => not_found(),
        },
    }
}

/// `GET /apt/pool/…/FILE`: a package's bytes.
pub async fn pool(State(app): State<Shared>, Path(path): Path<String>) -> Response {
    if !enabled(&app) {
        return not_found();
    }
    let i = match index(&app).await {
        Ok(i) => i,
        Err(e) => return failed(e),
    };
    let file = path.rsplit('/').next().unwrap_or("");
    let Some(hash) = i.pool.get(file) else { return not_found() };
    match tokio::fs::read(crate::web::releases::blob_path(&app, hash)).await {
        Ok(bytes) => (
            [
                (header::CONTENT_TYPE, "application/vnd.debian.binary-package".to_string()),
                (header::CACHE_CONTROL, "public, max-age=31536000, immutable".to_string()),
            ],
            bytes,
        )
            .into_response(),
        Err(_) => not_found(),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A minimal `.deb`: an ar archive with a control archive compressed
    /// as `name` says.
    pub fn deb(control: &str, member: &str) -> Vec<u8> {
        let mut tar_bytes = Vec::new();
        {
            let mut t = tar::Builder::new(&mut tar_bytes);
            let mut h = tar::Header::new_gnu();
            h.set_size(control.len() as u64);
            h.set_mode(0o644);
            h.set_cksum();
            t.append_data(&mut h, "./control", control.as_bytes()).unwrap();
            t.finish().unwrap();
        }
        let packed = match member {
            "control.tar.gz" => {
                let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
                std::io::Write::write_all(&mut gz, &tar_bytes).unwrap();
                gz.finish().unwrap()
            }
            "control.tar.xz" => {
                let mut out = Vec::new();
                lzma_rs::xz_compress(&mut tar_bytes.as_slice(), &mut out).unwrap();
                out
            }
            "control.tar.zst" => zstd::encode_all(tar_bytes.as_slice(), 3).unwrap(),
            _ => tar_bytes,
        };
        let mut out = b"!<arch>\n".to_vec();
        for (name, data) in [("debian-binary", b"2.0\n".to_vec()), (member, packed), ("data.tar.gz", vec![0u8; 3])] {
            out.extend_from_slice(format!("{:<16}{:<12}{:<6}{:<6}{:<8}{:<10}`\n", name, 0, 0, 0, 100644, data.len()).as_bytes());
            out.extend_from_slice(&data);
            if data.len() % 2 == 1 {
                out.push(b'\n');
            }
        }
        out
    }

    #[test]
    fn control_is_read_from_every_compression() {
        let control = "Package: oxy\nVersion: 0.3.0\nArchitecture: amd64\nDescription: test\n";
        for m in ["control.tar", "control.tar.gz", "control.tar.xz", "control.tar.zst"] {
            let e = entry_of("oxy_0.3.0_amd64.deb", &deb(control, m)).unwrap();
            assert_eq!((e.package.as_str(), e.architecture.as_str()), ("oxy", "amd64"), "{m}");
            assert!(e.control.contains("Version: 0.3.0"));
        }
        assert!(deb_control(b"not a deb").is_err());
        assert_eq!(
            pool_path("main", &entry_of("oxy_0.3.0_amd64.deb", &deb(control, "control.tar.gz")).unwrap()),
            "pool/main/o/oxy/oxy_0.3.0_amd64.deb"
        );
        assert_eq!(rfc2822(0), "Thu, 01 Jan 1970 00:00:00 UTC");
    }
}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1		//! The apt repository at `/apt` (oxygenhub#85): the `.deb` assets of the
2		//! configured repositories' releases, indexed and signed so that
3		//! `apt install oxy oh` and `apt upgrade` work. Nothing is stored apart:
4		//! the index is built from the releases when they change, and the pool
5		//! serves the assets' own bytes.
6
7		use crate::{db, Shared};
8		use axum::extract::{Path, State};
9		use axum::http::{header, StatusCode};
10		use axum::response::{IntoResponse, Response};
11		use sha2::Digest;
12		use std::collections::{BTreeMap, HashMap};
13		use std::io::Read;
14		use std::sync::{Arc, Mutex, OnceLock};
15
16		/// One package as the index lists it.
17		#[derive(Clone, Debug)]
18		struct Entry {
19		/// The control paragraph, as the package has it.
20		control: String,
21		package: String,
22		architecture: String,
23		/// The asset's name, which is its pool file name.
24		file: String,
25		size: u64,
26		md5: String,
27		sha256: String,
28		}
29
30		/// The signed index, built from one set of assets.
31		struct Index {
32		/// What it was built from: each `.deb` asset's id and hash.
33		key: String,
34		/// `main/binary-ARCH/Packages` and `.gz`, by their path under the suite.
35		files: BTreeMap<String, Vec<u8>>,
36		release: String,
37		in_release: Vec<u8>,
38		release_gpg: Vec<u8>,
39		/// Pool file name to the asset's blob hash.
40		pool: HashMap<String, String>,
41		}
42
43		/// Parsed `.deb`s by blob hash: a package is read once.
44	0×	fn parsed() -> &'static Mutex<HashMap<String, Entry>> {
45		static P: OnceLock<Mutex<HashMap<String, Entry>>> = OnceLock::new();
46	0×	P.get_or_init(Default::default)
47	0×	}
48
49	0×	fn built() -> &'static Mutex<Option<Arc<Index>>> {
50		static B: OnceLock<Mutex<Option<Arc<Index>>>> = OnceLock::new();
51	0×	B.get_or_init(Default::default)
52	0×	}
53
54		// ------------------------------------------------------------- .deb files
55
56		/// The members of an `ar` archive, by name.
57	6×	fn ar_members(bytes: &[u8]) -> Result<Vec<(String, &[u8])>, String> {
58	6×	let body = bytes.strip_prefix(b"!<arch>\n").ok_or("not an ar archive")?;
59	5×	let mut out = Vec::new();
60	5×	let mut at = 0;
61	20×	while at + 60 <= body.len() {
62	15×	let head = &body[at..at + 60];
63	15×	let name = String::from_utf8_lossy(&head[..16]).trim().trim_end_matches('/').to_string();
64	15×	let size: usize = String::from_utf8_lossy(&head[48..58]).trim().parse().map_err(\|_\| "bad ar member size")?;
65	15×	let start = at + 60;
66	15×	let end = start.checked_add(size).filter(\|e\| *e <= body.len()).ok_or("truncated ar archive")?;
67	15×	out.push((name, &body[start..end]));
68	15×	at = end + (size & 1);
69		}
70	5×	Ok(out)
71	6×	}
72
73		/// The `control` file of a `.deb`.
74	6×	pub fn deb_control(bytes: &[u8]) -> Result<String, String> {
75	6×	let members = ar_members(bytes)?;
76	10×	let (name, data) = members.into_iter().find(\|(n, _)\| n.starts_with("control.tar")).ok_or("no control.tar member")?;
77	5×	let mut tar_bytes = Vec::new();
78	5×	match name.as_str() {
79	5×	"control.tar" => tar_bytes.extend_from_slice(data),
80	4×	"control.tar.gz" => {
81	2×	flate2::read::GzDecoder::new(data).read_to_end(&mut tar_bytes).map_err(\|e\| e.to_string())?;
82		}
83	2×	"control.tar.xz" => lzma_rs::xz_decompress(&mut std::io::BufReader::new(data), &mut tar_bytes).map_err(\|e\| e.to_string())?,
84	1×	"control.tar.zst" => tar_bytes = zstd::decode_all(data).map_err(\|e\| e.to_string())?,
85	0×	other => return Err(format!("{other}: a compression this hub does not read")),
86		}
87	5×	let mut archive = tar::Archive::new(tar_bytes.as_slice());
88	5×	for entry in archive.entries().map_err(\|e\| e.to_string())? {
89	5×	let mut entry = entry.map_err(\|e\| e.to_string())?;
90	5×	let path = entry.path().map_err(\|e\| e.to_string())?.to_string_lossy().to_string();
91	5×	if path.trim_start_matches("./") == "control" {
92	5×	let mut text = String::new();
93	5×	entry.read_to_string(&mut text).map_err(\|e\| e.to_string())?;
94	5×	return Ok(text);
95	0×	}
96		}
97	0×	Err("no control file".into())
98	6×	}
99
100		/// A control paragraph's field.
101	15×	fn field<'a>(control: &'a str, name: &str) -> Option<&'a str> {
102	30×	control.lines().find_map(\|l\| l.strip_prefix(name).and_then(\|r\| r.strip_prefix(':')).map(str::trim))
103	15×	}
104
105	5×	fn entry_of(file: &str, bytes: &[u8]) -> Result<Entry, String> {
106	5×	let control = deb_control(bytes)?;
107	5×	let package = field(&control, "Package").ok_or("no Package field")?.to_string();
108	5×	let architecture = field(&control, "Architecture").ok_or("no Architecture field")?.to_string();
109	5×	field(&control, "Version").ok_or("no Version field")?;
110	5×	Ok(Entry {
111	5×	control: control.trim_end().to_string(),
112	5×	package,
113	5×	architecture,
114	5×	file: file.to_string(),
115	5×	size: bytes.len() as u64,
116	5×	md5: hex::encode(md5::Md5::digest(bytes)),
117	5×	sha256: hex::encode(sha2::Sha256::digest(bytes)),
118	5×	})
119	5×	}
120
121		// ------------------------------------------------------------------ index
122
123		/// The `.deb` assets of the configured public repositories: (asset id,
124		/// name, blob hash).
125	0×	async fn debs(app: &Shared) -> Result<Vec<(i64, String, String)>, sqlx::Error> {
126	0×	let mut out = Vec::new();
127	0×	for slug in &app.config.apt.repos {
128	0×	let Some((owner, name)) = slug.split_once('/') else { continue };
129	0×	let Some(row) = db::repos::get(&app.db, owner, name).await? else { continue };
130	0×	if row.visibility() != hub_api::Visibility::Public {
131	0×	tracing::warn!("apt: {slug} is not public, so its packages are not served");
132	0×	continue;
133	0×	}
134	0×	let rows: Vec<(i64, String, String)> = sqlx::query_as(
135	0×	"SELECT a.id, a.name, a.hash FROM release_assets a JOIN releases r ON r.id = a.release_id \
136	0×	WHERE r.repo_id = $1 AND a.name LIKE '%.deb' ORDER BY a.id",
137	0×	)
138	0×	.bind(row.id)
139	0×	.fetch_all(&app.db)
140	0×	.await?;
141	0×	out.extend(rows);
142		}
143	0×	Ok(out)
144	0×	}
145
146		/// The pool path of a package: `pool/COMPONENT/p/PACKAGE/FILE`.
147	1×	fn pool_path(component: &str, e: &Entry) -> String {
148	1×	let first: String = e.package.chars().take(1).collect();
149	1×	format!("pool/{component}/{first}/{}/{}", e.package, e.file)
150	1×	}
151
152		/// RFC 2822 date in UTC, as Release files carry it.
153	1×	fn rfc2822(secs: i64) -> String {
154	1×	let t = time::OffsetDateTime::from_unix_timestamp(secs).unwrap_or(time::OffsetDateTime::UNIX_EPOCH);
155	1×	let fmt = time::macros::format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] UTC");
156	1×	t.format(&fmt).unwrap_or_default()
157	1×	}
158
159		/// Run gpg with the repository's key on `input`.
160	0×	async fn gpg(app: &Shared, args: &[&str], input: &[u8]) -> Result<Vec<u8>, String> {
161		use tokio::io::AsyncWriteExt;
162	0×	let home = app.config.data.join(&app.config.apt.gnupg_home);
163	0×	let mut child = tokio::process::Command::new("gpg")
164	0×	.arg("--homedir")
165	0×	.arg(&home)
166	0×	.args(["--batch", "--yes", "--quiet", "--no-tty", "--pinentry-mode", "loopback", "--passphrase", ""])
167	0×	.args(args)
168	0×	.stdin(std::process::Stdio::piped())
169	0×	.stdout(std::process::Stdio::piped())
170	0×	.stderr(std::process::Stdio::piped())
171	0×	.spawn()
172	0×	.map_err(\|e\| format!("gpg: {e}"))?;
173	0×	let mut stdin = child.stdin.take().expect("piped");
174	0×	stdin.write_all(input).await.map_err(\|e\| e.to_string())?;
175	0×	drop(stdin);
176	0×	let out = child.wait_with_output().await.map_err(\|e\| e.to_string())?;
177	0×	if !out.status.success() {
178	0×	return Err(format!("gpg {}: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim()));
179	0×	}
180	0×	Ok(out.stdout)
181	0×	}
182
183		/// The index for the assets there are now, built when they changed.
184	0×	async fn index(app: &Shared) -> Result<Arc<Index>, String> {
185	0×	let assets = debs(app).await.map_err(\|e\| e.to_string())?;
186	0×	let key: String = assets.iter().map(\|(id, _, h)\| format!("{id}:{h};")).collect();
187	0×	if let Some(i) = built().lock().unwrap().as_ref().filter(\|i\| i.key == key) {
188	0×	return Ok(i.clone());
189	0×	}
190	0×	let cfg = &app.config.apt;
191	0×	let mut entries: Vec<Entry> = Vec::new();
192	0×	let mut pool = HashMap::new();
193	0×	for (_, name, hash) in &assets {
194	0×	let known = parsed().lock().unwrap().get(hash).cloned();
195	0×	let e = match known {
196	0×	Some(e) => Entry { file: name.clone(), ..e },
197		None => {
198	0×	let bytes = tokio::fs::read(crate::web::releases::blob_path(app, hash)).await.map_err(\|e\| e.to_string())?;
199	0×	match entry_of(name, &bytes) {
200	0×	Ok(e) => {
201	0×	parsed().lock().unwrap().insert(hash.clone(), e.clone());
202	0×	e
203		}
204	0×	Err(why) => {
205	0×	tracing::warn!("apt: {name} is left out: {why}");
206	0×	continue;
207		}
208		}
209		}
210		};
211	0×	if pool.contains_key(&e.file) {
212	0×	continue;
213	0×	}
214	0×	pool.insert(e.file.clone(), hash.clone());
215	0×	entries.push(e);
216		}
217	0×	let mut files = BTreeMap::new();
218	0×	for arch in &cfg.architectures {
219	0×	let mut text = String::new();
220	0×	for e in entries.iter().filter(\|e\| e.architecture == *arch \|\| e.architecture == "all") {
221	0×	text.push_str(&format!(
222	0×	"{}\nFilename: {}\nSize: {}\nMD5sum: {}\nSHA256: {}\n\n",
223	0×	e.control,
224	0×	pool_path(&cfg.component, e),
225	0×	e.size,
226	0×	e.md5,
227	0×	e.sha256
228	0×	));
229	0×	}
230	0×	let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best());
231	0×	std::io::Write::write_all(&mut gz, text.as_bytes()).map_err(\|e\| e.to_string())?;
232	0×	let gz = gz.finish().map_err(\|e\| e.to_string())?;
233	0×	files.insert(format!("{}/binary-{arch}/Packages", cfg.component), text.into_bytes());
234	0×	files.insert(format!("{}/binary-{arch}/Packages.gz", cfg.component), gz);
235		}
236	0×	let sums = \|f: &dyn Fn(&[u8]) -> String\| -> String {
237	0×	files.iter().map(\|(path, bytes)\| format!(" {} {:>8} {path}\n", f(bytes), bytes.len())).collect()
238	0×	};
239	0×	let release = format!(
240		"Origin: {site}\nLabel: {site}\nSuite: {suite}\nCodename: {suite}\nDate: {date}\nArchitectures: {arches}\nComponents: {component}\nDescription: {site} packages (oxy and oh)\nMD5Sum:\n{md5}SHA256:\n{sha256}",
241	0×	site = app.config.site.name,
242		suite = cfg.suite,
243	0×	date = rfc2822(db::now_ms() / 1000),
244	0×	arches = cfg.architectures.join(" "),
245		component = cfg.component,
246	0×	md5 = sums(&\|b\| hex::encode(md5::Md5::digest(b))),
247	0×	sha256 = sums(&\|b\| hex::encode(sha2::Sha256::digest(b))),
248		);
249	0×	let in_release = gpg(app, &["--clearsign", "--digest-algo", "SHA512"], release.as_bytes()).await?;
250	0×	let release_gpg = gpg(app, &["--detach-sign", "--armor", "--digest-algo", "SHA512"], release.as_bytes()).await?;
251	0×	let i = Arc::new(Index { key, files, release, in_release, release_gpg, pool });
252	0×	*built().lock().unwrap() = Some(i.clone());
253	0×	Ok(i)
254	0×	}
255
256		// ----------------------------------------------------------------- routes
257
258	2×	fn enabled(app: &Shared) -> bool {
259	2×	!app.config.apt.repos.is_empty()
260	2×	}
261
262	0×	fn reply(ct: &str, bytes: Vec<u8>) -> Response {
263	0×	([(header::CONTENT_TYPE, ct.to_string()), (header::CACHE_CONTROL, "no-cache".to_string())], bytes).into_response()
264	0×	}
265
266	2×	fn not_found() -> Response {
267	2×	(StatusCode::NOT_FOUND, "not found\n").into_response()
268	2×	}
269
270	0×	fn failed(why: String) -> Response {
271	0×	tracing::warn!("apt: {why}");
272	0×	(StatusCode::SERVICE_UNAVAILABLE, "the package index is not available just now\n").into_response()
273	0×	}
274
275		/// `GET /apt`: how to use it.
276	1×	pub async fn home(State(app): State<Shared>) -> Response {
277	1×	if !enabled(&app) {
278	1×	return not_found();
279	0×	}
280	0×	let base = app.config.base_url();
281	0×	let cfg = &app.config.apt;
282	0×	let text = format!(
283		"{site} apt repository: oxy and oh for Debian and Ubuntu ({arches}).\n\n\
284		Installing a .deb from a release adds this repository by itself. To add it by hand:\n\n\
285		curl -fsSL {base}/apt/oxygenhub.gpg \| sudo tee /usr/share/keyrings/oxygenhub-archive-keyring.gpg >/dev/null\n\
286		sudo tee /etc/apt/sources.list.d/oxygenhub.sources >/dev/null <<EOF\n\
287		Types: deb\nURIs: {base}/apt\nSuites: {suite}\nComponents: {component}\n\
288		Signed-By: /usr/share/keyrings/oxygenhub-archive-keyring.gpg\nEOF\n\
289		sudo apt update && sudo apt install oxy oh\n\n\
290		The signing key's fingerprint is in {base}/apt/oxygenhub.asc (gpg --show-keys).\n",
291	0×	site = app.config.site.name,
292	0×	arches = cfg.architectures.join(", "),
293		suite = cfg.suite,
294		component = cfg.component,
295		);
296	0×	reply("text/plain; charset=utf-8", text.into_bytes())
297	1×	}
298
299		/// `GET /apt/oxygenhub.gpg` (binary keyring) and `/apt/oxygenhub.asc`.
300	0×	pub async fn key(State(app): State<Shared>, Path(file): Path<String>) -> Response {
301	0×	if !enabled(&app) {
302	0×	return not_found();
303	0×	}
304	0×	let armor = match file.as_str() {
305	0×	"oxygenhub.gpg" => false,
306	0×	"oxygenhub.asc" => true,
307	0×	_ => return not_found(),
308		};
309	0×	let args: &[&str] = if armor { &["--armor", "--export"] } else { &["--export"] };
310	0×	match gpg(&app, args, b"").await {
311	0×	Ok(k) if !k.is_empty() => reply(if armor { "text/plain; charset=utf-8" } else { "application/pgp-keys" }, k),
312	0×	Ok(_) => failed("the signing key is missing".into()),
313	0×	Err(e) => failed(e),
314		}
315	0×	}
316
317		/// `GET /apt/dists/SUITE/…`: the Release files and the Packages indexes.
318	1×	pub async fn dists(State(app): State<Shared>, Path((suite, rest)): Path<(String, String)>) -> Response {
319	1×	if !enabled(&app) \|\| suite != app.config.apt.suite {
320	1×	return not_found();
321	0×	}
322	0×	let i = match index(&app).await {
323	0×	Ok(i) => i,
324	0×	Err(e) => return failed(e),
325		};
326	0×	match rest.as_str() {
327	0×	"InRelease" => reply("text/plain; charset=utf-8", i.in_release.clone()),
328	0×	"Release" => reply("text/plain; charset=utf-8", i.release.clone().into_bytes()),
329	0×	"Release.gpg" => reply("application/pgp-signature", i.release_gpg.clone()),
330	0×	path => match i.files.get(path) {
331	0×	Some(b) if path.ends_with(".gz") => reply("application/gzip", b.clone()),
332	0×	Some(b) => reply("text/plain; charset=utf-8", b.clone()),
333	0×	None => not_found(),
334		},
335		}
336	1×	}
337
338		/// `GET /apt/pool/…/FILE`: a package's bytes.
339	0×	pub async fn pool(State(app): State<Shared>, Path(path): Path<String>) -> Response {
340	0×	if !enabled(&app) {
341	0×	return not_found();
342	0×	}
343	0×	let i = match index(&app).await {
344	0×	Ok(i) => i,
345	0×	Err(e) => return failed(e),
346		};
347	0×	let file = path.rsplit('/').next().unwrap_or("");
348	0×	let Some(hash) = i.pool.get(file) else { return not_found() };
349	0×	match tokio::fs::read(crate::web::releases::blob_path(&app, hash)).await {
350	0×	Ok(bytes) => (
351	0×	[
352	0×	(header::CONTENT_TYPE, "application/vnd.debian.binary-package".to_string()),
353	0×	(header::CACHE_CONTROL, "public, max-age=31536000, immutable".to_string()),
354	0×	],
355	0×	bytes,
356	0×	)
357	0×	.into_response(),
358	0×	Err(_) => not_found(),
359		}
360	0×	}
361
362		#[cfg(test)]
363		mod tests {
364		use super::*;
365
366		/// A minimal `.deb`: an ar archive with a control archive compressed
367		/// as `name` says.
368	5×	pub fn deb(control: &str, member: &str) -> Vec<u8> {
369	5×	let mut tar_bytes = Vec::new();
370	5×	{
371	5×	let mut t = tar::Builder::new(&mut tar_bytes);
372	5×	let mut h = tar::Header::new_gnu();
373	5×	h.set_size(control.len() as u64);
374	5×	h.set_mode(0o644);
375	5×	h.set_cksum();
376	5×	t.append_data(&mut h, "./control", control.as_bytes()).unwrap();
377	5×	t.finish().unwrap();
378	5×	}
379	5×	let packed = match member {
380	5×	"control.tar.gz" => {
381	2×	let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
382	2×	std::io::Write::write_all(&mut gz, &tar_bytes).unwrap();
383	2×	gz.finish().unwrap()
384		}
385	3×	"control.tar.xz" => {
386	1×	let mut out = Vec::new();
387	1×	lzma_rs::xz_compress(&mut tar_bytes.as_slice(), &mut out).unwrap();
388	1×	out
389		}
390	2×	"control.tar.zst" => zstd::encode_all(tar_bytes.as_slice(), 3).unwrap(),
391	1×	_ => tar_bytes,
392		};
393	5×	let mut out = b"!<arch>\n".to_vec();
394	15×	for (name, data) in [("debian-binary", b"2.0\n".to_vec()), (member, packed), ("data.tar.gz", vec![0u8; 3])] {
395	15×	out.extend_from_slice(format!("{:<16}{:<12}{:<6}{:<6}{:<8}{:<10}`\n", name, 0, 0, 0, 100644, data.len()).as_bytes());
396	15×	out.extend_from_slice(&data);
397	15×	if data.len() % 2 == 1 {
398	7×	out.push(b'\n');
399	8×	}
400		}
401	5×	out
402	5×	}
403
404		#[test]
405	1×	fn control_is_read_from_every_compression() {
406	1×	let control = "Package: oxy\nVersion: 0.3.0\nArchitecture: amd64\nDescription: test\n";
407	4×	for m in ["control.tar", "control.tar.gz", "control.tar.xz", "control.tar.zst"] {
408	4×	let e = entry_of("oxy_0.3.0_amd64.deb", &deb(control, m)).unwrap();
409	4×	assert_eq!((e.package.as_str(), e.architecture.as_str()), ("oxy", "amd64"), "{m}");
410	4×	assert!(e.control.contains("Version: 0.3.0"));
411		}
412	1×	assert!(deb_control(b"not a deb").is_err());
413	1×	assert_eq!(
414	1×	pool_path("main", &entry_of("oxy_0.3.0_amd64.deb", &deb(control, "control.tar.gz")).unwrap()),
415		"pool/main/o/oxy/oxy_0.3.0_amd64.deb"
416		);
417	1×	assert_eq!(rfc2822(0), "Thu, 01 Jan 1970 00:00:00 UTC");
418	1×	}
419		}

Coverage / crates/hub/src/apt.rs 38.6%

Keyboard