flow deps check: is every crate of a Cargo.lock in the image (closes #101) f3b5b86e23ef

//! `flow deps check` (#101): whether every crate a `Cargo.lock` needs is

//! vendored in the build image at that version. Jobs run with no network,

//! so a crate the image lacks fails the run with "no matching package";

//! this says so before the push, naming the versions the image does have.

use std::collections::{BTreeMap, BTreeSet};

use std::path::Path;

/// A crate at a version.

pub type Crate = (String, String);

/// The packages of a `Cargo.lock` that come from somewhere (a registry or

/// git): the ones `cargo vendor` copies. A workspace's own members have no

/// `source` and are left out.

pub fn from_lock(text: &str) -> Result<BTreeSet<Crate>, String> {

    let lock: toml::Value = text.parse().map_err(|e| format!("not a Cargo.lock: {e}"))?;

    let mut out = BTreeSet::new();

    for p in lock.get("package").and_then(|p| p.as_array()).into_iter().flatten() {

        let (Some(name), Some(version)) = (p.get("name").and_then(|v| v.as_str()), p.get("version").and_then(|v| v.as_str())) else {

            continue;

        };

        if p.get("source").is_some() {

            out.insert((name.to_string(), version.to_string()));

        }

    }

    Ok(out)

}

/// A crate list as `flow deps list` and the image's `crates.txt` write it:

/// `NAME VERSION` a line; blank lines and `#` comments are skipped.

pub fn from_list(text: &str) -> BTreeSet<Crate> {

    text.lines()

        .map(str::trim)

        .filter(|l| !l.is_empty() && !l.starts_with('#'))

        .filter_map(|l| {

            let mut w = l.split_whitespace();

            Some((w.next()?.to_string(), w.next()?.to_string()))

        })

        .collect()

}

/// The crates of a `cargo vendor` directory (the image's `/opt/vendor`):

/// each subdirectory's `Cargo.toml` names its package and version.

pub fn from_vendor(dir: &Path) -> Result<BTreeSet<Crate>, String> {

    let entries = std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?;

    let mut out = BTreeSet::new();

    for e in entries.flatten() {

        let manifest = e.path().join("Cargo.toml");

        let Ok(text) = std::fs::read_to_string(&manifest) else { continue };

        let Ok(v) = text.parse::<toml::Value>() else { continue };

        let pkg = v.get("package");

        let field = |k: &str| pkg.and_then(|p| p.get(k)).and_then(|x| x.as_str()).map(str::to_string);

        if let (Some(n), Some(ver)) = (field("name"), field("version")) {

            out.insert((n, ver));

        }

    }

    Ok(out)

}

/// What `need` lacks from `have`, each with the versions of that crate

/// `have` does hold (empty when it holds none).

pub fn missing(need: &BTreeSet<Crate>, have: &BTreeSet<Crate>) -> Vec<(Crate, Vec<String>)> {

    let mut versions: BTreeMap<&str, Vec<String>> = BTreeMap::new();

    for (n, v) in have {

        versions.entry(n.as_str()).or_default().push(v.clone());

    }

    need.iter().filter(|c| !have.contains(*c)).map(|c| (c.clone(), versions.get(c.0.as_str()).cloned().unwrap_or_default())).collect()

}

/// A crate list in the form [`from_list`] reads.

pub fn render_list(crates: &BTreeSet<Crate>) -> String {

    crates.iter().map(|(n, v)| format!("{n} {v}\n")).collect()

}

#[cfg(test)]

mod tests {

    use super::*;

    const LOCK: &str = r#"

version = 4

[[package]]

name = "mine"

version = "0.1.0"

dependencies = ["rustls"]

[[package]]

name = "rustls"

version = "0.23.45"

source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]

name = "tokio-rustls"

version = "0.26.5"

source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]

name = "deranged"

version = "0.5.8"

source = "registry+https://github.com/rust-lang/crates.io-index"

"#;

    #[test]

    fn names_what_the_image_lacks_and_what_it_has_instead() {

        let need = from_lock(LOCK).unwrap();

        assert_eq!(need.len(), 3, "the workspace's own package is not vendored");

        let have = from_list("# the image\nrustls 0.23.45\ntokio-rustls 0.26.4\n\n");

        let m = missing(&need, &have);

        assert_eq!(

            m,

            vec![(("deranged".into(), "0.5.8".into()), vec![]), (("tokio-rustls".into(), "0.26.5".into()), vec!["0.26.4".into()]),]

        );

        assert!(missing(&need, &need).is_empty());

        assert_eq!(from_list(&render_list(&need)), need);

        assert!(from_lock("not = [toml").is_err());

    }

    #[test]

    fn reads_a_vendor_directory() {

        let dir = tempfile::tempdir().unwrap();

        for (d, n, v) in [("rustls-0.23.45", "rustls", "0.23.45"), ("pre-1.0.0-rc.1", "pre", "1.0.0-rc.1")] {

            std::fs::create_dir(dir.path().join(d)).unwrap();

            std::fs::write(dir.path().join(d).join("Cargo.toml"), format!("[package]\nname = \"{n}\"\nversion = \"{v}\"\n")).unwrap();

        }

        std::fs::create_dir(dir.path().join("junk")).unwrap();

        let got = from_vendor(dir.path()).unwrap();

        assert_eq!(got, from_list("rustls 0.23.45\npre 1.0.0-rc.1\n"));

    }

}

- **Added: `flow deps check` (#101).** Jobs have no network, so every crate a `Cargo.lock` needs must be vendored in the build image at that version. `flow deps check [Cargo.lock]` says whether it is, before the push: each missing crate with the versions the image does have and the `cargo update --precise` that pins one, exit 1. It reads the image's crate list (`--list`), a vendor directory (`--vendor`), or the lock files the image is built from (`--image-lock`, repeatable); inside a job, the image's own. `flow deps list` prints the set.

- **Changed: the build image publishes its crate list.** `images/rust/build.sh` writes `crates.txt` (`NAME VERSION` a line) beside the tarball and at `/opt/crates.txt` in the image.

pub mod deps;

enum DepsCmd {

    /// Whether every crate a Cargo.lock needs is in the image at that

    /// version; exit 1 naming each missing one and the versions it has

    Check {

        /// The lock file (default: Cargo.lock)

        #[arg(default_value = "Cargo.lock")]

        lock: PathBuf,

        #[command(flatten)]

        image: DepsSource,

    },

    /// Print the image's crates as `NAME VERSION` lines (its crates.txt)

    List {

        #[command(flatten)]

        image: DepsSource,

    },

}

/// Where the image's crate set comes from: its crate list (the crates.txt

/// published with the image), a vendor directory, or the lock files it is

/// built from; inside a job, /opt/vendor.

#[derive(clap::Args)]

#[group(multiple = false)]

struct DepsSource {

    /// The image's crate list (crates.txt, written by images/rust/build.sh)

    #[arg(long, value_name = "FILE")]

    list: Option<PathBuf>,

    /// A cargo vendor directory (default inside a job: /opt/crates.txt, else /opt/vendor)

    #[arg(long, value_name = "DIR")]

    vendor: Option<PathBuf>,

    /// The lock files the image vendors (oxygen's, oxygenhub's,

    /// oxygenflow's), as images/rust/build.sh does (repeatable)

    #[arg(long = "image-lock", value_name = "LOCK")]

    locks: Vec<PathBuf>,

}

impl DepsSource {

    fn crates(&self) -> Result<BTreeSet<flow::deps::Crate>> {

        let read = |p: &Path| std::fs::read_to_string(p).with_context(|| p.display().to_string());

        if let Some(f) = &self.list {

            return Ok(flow::deps::from_list(&read(f)?));

        }

        if !self.locks.is_empty() {

            let mut all = BTreeSet::new();

            for l in &self.locks {

                all.extend(flow::deps::from_lock(&read(l)?).map_err(|e| anyhow::anyhow!("{}: {e}", l.display()))?);

            }

            return Ok(all);

        }

        // Inside a job: the image's own list, else its vendor directory.

        if self.vendor.is_none() && Path::new("/opt/crates.txt").is_file() {

            return Ok(flow::deps::from_list(&read(Path::new("/opt/crates.txt"))?));

        }

        let dir = self.vendor.clone().unwrap_or_else(|| PathBuf::from("/opt/vendor"));

        if self.vendor.is_none() && !dir.is_dir() {

            bail!("say what the image holds: --list crates.txt (published with the image), --vendor DIR, or --image-lock for each lock file it is built from; inside a job, /opt/vendor is read");

        }

        flow::deps::from_vendor(&dir).map_err(|e| anyhow::anyhow!(e))

    }

}

#[derive(Subcommand)]

    /// The crates the build image vendors, for jobs with no network (#101)

    Deps {

        #[command(subcommand)]

        cmd: DepsCmd,

    },

        Cmd::Deps { cmd: DepsCmd::List { image } } => {

            print!("{}", flow::deps::render_list(&image.crates()?));

            Ok(())

        }

        Cmd::Deps { cmd: DepsCmd::Check { lock, image } } => {

            let text = std::fs::read_to_string(&lock).with_context(|| lock.display().to_string())?;

            let need = flow::deps::from_lock(&text).map_err(|e| anyhow::anyhow!("{}: {e}", lock.display()))?;

            let missing = flow::deps::missing(&need, &image.crates()?);

            if missing.is_empty() {

                println!("deps: all {} crates of {} are in the image", need.len(), lock.display());

                return Ok(());

            }

            eprintln!("deps: {} of {} crates of {} are not in the image:", missing.len(), need.len(), lock.display());

            for ((name, version), has) in &missing {

                let instead = match has.as_slice() {

                    [] => "the image has no version of it".to_string(),

                    vs => format!("the image has {}: cargo update -p {name}@{version} --precise {}", vs.join(", "), vs.last().unwrap()),

                };

                eprintln!("  {name} {version}  ({instead})");

            }

            eprintln!("pin a version the image has, or rebuild the image with this workspace's lock (images/rust/build.sh)");

            std::process::exit(1);

        }

# What it vendors, `NAME VERSION` a line: `flow deps check --list` (#101).

COPY crates.txt /opt/crates.txt

# The image's crates as `NAME VERSION` lines (#101): at /opt/crates.txt in

# the image and beside the tarball, for `flow deps check --list` to read.

for m in "$here"/vendor/*/Cargo.toml; do

    awk -F'"' '/^\[package\]/{p=1;next} /^\[/{p=0} p&&/^name = /{n=$2} p&&/^version = /{v=$2} END{print n" "v}' "$m"

done | sort > "$here/crates.txt"

chmod -R a+rX "$here/vendor" "$here/npm-cache" "$here/warm" "$here/crates.txt"

cp "$here/crates.txt" "${out%.tar}.crates.txt"

rm -rf "${here:?}/vendor" "${here:?}/npm-cache" "${here:?}/warm" "${here:?}/crates.txt"