oxy remote add takes a store's settings; oxy remote check 9aa399044d9d

//! Remotes kept on a store, through the command line: `oxy remote add`

//! with its store settings, `oxy remote check`, and push, clone and pull

//! through a `dir://` store (docs/stores.md).

use std::path::Path;

use std::process::Command;

fn oxy(dir: &Path, home: &Path, args: &[&str]) -> (i32, String, String) {

    let out = Command::new(env!("CARGO_BIN_EXE_oxy"))

        .arg("--plain")

        .args(args)

        .current_dir(dir)

        .env("OXY_HOME", home)

        .env("NO_COLOR", "1")

        .output()

        .unwrap();

    (

        out.status.code().unwrap_or(-1),

        String::from_utf8_lossy(&out.stdout).to_string(),

        String::from_utf8_lossy(&out.stderr).to_string(),

    )

}

fn ok(dir: &Path, home: &Path, args: &[&str]) -> String {

    let (code, out, err) = oxy(dir, home, args);

    assert_eq!(code, 0, "oxy {args:?}: {out}{err}");

    out

}

#[test]

fn store_remotes_through_the_cli() {

    let tmp = tempfile::tempdir().unwrap();

    let home = tmp.path().join("home");

    std::fs::create_dir_all(&home).unwrap();

    let a = tmp.path().join("a");

    ok(tmp.path(), &home, &["init", a.to_str().unwrap()]);

    std::fs::write(a.join("f.txt"), "hello\n").unwrap();

    ok(&a, &home, &["commit", "-m", "first"]);

    let bucket = tmp.path().join("bucket");

    let url = format!("dir://{}", bucket.display());

    // Store settings belong to bucket URLs only, and are kept on the URL.

    let (code, _, err) = oxy(

        &a,

        &home,

        &["remote", "add", "usb", &url, "--profile", "work"],

    );

    assert_ne!(code, 0);

    assert!(err.contains("--profile") && err.contains("s3://"), "{err}");

    ok(

        &a,

        &home,

        &[

            "remote",

            "add",

            "cloud",

            "s3://my-bucket/repos/a",

            "--profile",

            "work",

            "--region",

            "eu-west-2",

        ],

    );

    let listed = ok(&a, &home, &["remote", "list"]);

    assert!(

        listed.contains("s3://my-bucket/repos/a?profile=work&region=eu-west-2"),

        "{listed}"

    );

    ok(&a, &home, &["remote", "remove", "cloud"]);

    // A directory store: check before anything is there, push, check again.

    ok(&a, &home, &["remote", "add", "usb", &url]);

    let out = ok(&a, &home, &["remote", "check", "usb"]);

    assert!(

        out.contains("Reachable")

            && out.contains("holds no repository yet")

            && out.contains("Behaves"),

        "{out}"

    );

    assert_eq!(

        std::fs::read_dir(&bucket).unwrap().count(),

        0,

        "the check left nothing behind"

    );

    ok(&a, &home, &["push", "usb"]);

    let out = ok(&a, &home, &["remote", "check", "usb"]);

    assert!(

        out.contains("holds this repository")

            && out.contains("1 bookmark")

            && out.contains("1 bundle")

            && out.contains("Behaves"),

        "{out}"

    );

    // Clone from the store, push back, pull.

    let b = tmp.path().join("b");

    ok(tmp.path(), &home, &["clone", &url, b.to_str().unwrap()]);

    assert_eq!(std::fs::read_to_string(b.join("f.txt")).unwrap(), "hello\n");

    std::fs::write(b.join("f.txt"), "hello\nagain\n").unwrap();

    ok(&b, &home, &["commit", "-m", "second"]);

    ok(&b, &home, &["push"]);

    ok(&a, &home, &["pull", "usb"]);

    assert_eq!(

        std::fs::read_to_string(a.join("f.txt")).unwrap(),

        "hello\nagain\n"

    );

    // Another repository is told the store is taken.

    let c = tmp.path().join("c");

    ok(tmp.path(), &home, &["init", c.to_str().unwrap()]);

    std::fs::write(c.join("g.txt"), "other\n").unwrap();

    ok(&c, &home, &["commit", "-m", "mine"]);

    ok(&c, &home, &["remote", "add", "usb", &url]);

    let out = ok(&c, &home, &["remote", "check", "usb"]);

    assert!(

        out.contains("holds another repository") && out.contains("would be refused"),

        "{out}"

    );

    let (code, _, err) = oxy(&c, &home, &["push", "usb"]);

    assert_ne!(code, 0);

    assert!(err.contains("different repository"), "{err}");

}

            Ok(()) => {}

            Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),

            Err(e) => return Err(e.into()),

        }

        // A directory left empty goes too, up to the root, so the store

        // holds nothing but its objects.

        let mut dir = path.parent();

        while let Some(d) = dir {

            if d == self.root || fs::remove_dir(d).is_err() {

                break;

            }

            dir = d.parent();

        Ok(())

        assert_eq!(

            fs::read_dir(dir.path().join("bucket")).unwrap().count(),

            0,

            "no empty directories left"

        );

        /// A local path or the http(s) URL of a hub repository; or a store the repository is kept on: s3://BUCKET/PREFIX, gs://BUCKET/PREFIX, az://ACCOUNT/CONTAINER/PREFIX, dir://PATH

        /// For an s3:// store: the AWS profile to use (default: AWS_PROFILE, else "default")

        #[arg(long)]

        profile: Option<String>,

        /// For an s3:// store: its region (default: the environment, the profile, else asked of S3)

        #[arg(long)]

        region: Option<String>,

        /// For an s3://, gs:// or az:// store: another endpoint, such as an S3-compatible service

        #[arg(long)]

        endpoint: Option<String>,

        /// For a gs:// store: a service account key file, or application default credentials

        #[arg(long)]

        credentials: Option<String>,

    /// Check a remote: that it is reachable with the credentials in hand, what it holds, and (for a store) that it behaves

    Check {

        /// The remote to check

        name: String,

    },

            RemoteCmd::Add {

                name,

                url,

                profile,

                region,

                endpoint,

                credentials,

            } => sync::remote_add(

                &name,

                &url,

                &[

                    ("profile", profile),

                    ("region", region),

                    ("endpoint", endpoint),

                    ("credentials", credentials),

                ],

            ),

            RemoteCmd::Check { name } => sync::remote_check(&name),

/// Whether a URL names a store (a bucket or a directory in the store

/// layout) rather than a repository.

fn is_store_url(url: &str) -> bool {

    ["s3://", "gs://", "az://", "dir://"]

        .iter()

        .any(|p| url.starts_with(p))

}

/// Percent-encode what would break a query parameter.

fn encode_param(v: &str) -> String {

    let mut out = String::with_capacity(v.len());

    for b in v.bytes() {

        match b {

            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' | b'/' | b':' => {

                out.push(b as char)

            }

            _ => out.push_str(&format!("%{b:02X}")),

        }

    }

    out

}

/// A store URL with settings added as query parameters, which is where a

/// store remote keeps them (`s3://b/p?profile=work&region=eu-west-2`).

fn url_with_settings(url: &str, settings: &[(&str, Option<String>)]) -> Result<String> {

    let given: Vec<(&str, &str)> = settings

        .iter()

        .filter_map(|(k, v)| v.as_deref().map(|v| (*k, v)))

        .collect();

    if given.is_empty() {

        return Ok(url.to_string());

    }

    if !is_store_url(url) || url.starts_with("dir://") {

        let names: Vec<String> = given.iter().map(|(k, _)| format!("--{k}")).collect();

        anyhow::bail!(

            "{} apply to a store remote (s3://, gs://, az://), not to {url}",

            names.join(", ")

        );

    }

    let mut out = url.to_string();

    for (k, v) in given {

        out.push(if out.contains('?') { '&' } else { '?' });

        out.push_str(k);

        out.push('=');

        out.push_str(&encode_param(v));

    }

    Ok(out)

}

pub fn remote_add(name: &str, url: &str, settings: &[(&str, Option<String>)]) -> Result<i32> {

    let url = url_with_settings(url, settings)?;

    let url = url.as_str();

}

/// `oxy remote check NAME`: reach the remote with the credentials in

/// hand, say what it holds, and, for a store, put it through the

/// conformance check under a probe prefix (left empty afterwards).

pub fn remote_check(name: &str) -> Result<i32> {

    let repo = open()?;

    let Some(url) = repo.config.remotes.get(name).cloned() else {

        anyhow::bail!("no remote {name:?}; oxy remote list shows the remotes there are");

    };

    let options = repo.options_for_url(&url);

    let mine = repo.id()?;

    let started = std::time::Instant::now();

    let sp = ui::spinner(&format!("Checking {name}…"));

    let Some(store) = oxy_core::protocol::open_store(&url, &options) else {

        // A hub or another repository: reach it and read its references.

        let (_, remote) = repo.open_remote_at(&url)?;

        let id = remote.repo_id()?;

        let refs = remote.refs()?;

        sp.done(&format!(

            "{} {} {}",

            ui::done("Reachable"),

            ui::bookmark(name),

            ui::dim(format!("({url}, {} ms)", started.elapsed().as_millis()))

        ));

        describe_holding(id, mine, refs.refs.len(), refs.tags.len(), None);

        return Ok(0);

    };

    let store = store.with_context(|| format!("opening the store of {name}"))?;

    // What it holds, in one listing and two reads.

    let marker = match store.get("oxygen.toml") {

        Ok(b) => Some(b),

        Err(oxy_core::strata::Error::NotFound(_)) => None,

        Err(e) => {

            sp.clear();

            return Err(anyhow::anyhow!("{e}"))

                .with_context(|| format!("reaching the store of {name}"));

        }

    };

    let reached = started.elapsed();

    let bundles = store.list("bundles/")?.len();

    let record = store.read_ref("refs")?;

    sp.done(&format!(

        "{} {} {}",

        ui::done("Reachable"),

        ui::bookmark(name),

        ui::dim(format!("({url}, {} ms)", reached.as_millis()))

    ));

    match marker {

        None => println!(

            "  {} {}",

            ui::note(),

            ui::dim("holds no repository yet: oxy push NAME binds it to this one")

        ),

        Some(_) => {

            let (id, bookmarks, tags) = match record {

                Some((bytes, _)) => {

                    let (id, resp) = oxy_core::protocol::wire::decode_refs(&bytes)?;

                    (id, resp.refs.len(), resp.tags.len())

                }

                None => (mine, 0, 0),

            };

            describe_holding(id, mine, bookmarks, tags, Some(bundles));

        }

    }

    // Then the behaviour a repository depends on, under a prefix of its own.

    let sp = ui::spinner("Checking that the store behaves…");

    let probe = format!("oxy-check-{}", std::process::id());

    let began = std::time::Instant::now();

    match oxy_core::strata::conformance(store.as_ref(), &probe) {

        Ok(passed) => {

            sp.clear();

            for p in &passed {

                println!("  {} {}", ui::ok(ui::glyph("✔", "*")), p);

            }

            println!(

                "{} {}",

                ui::done("Behaves"),

                ui::dim(format!(

                    "({} checks in {} ms; nothing left behind)",

                    passed.len(),

                    began.elapsed().as_millis()

                ))

            );

            Ok(0)

        }

        Err(e) => {

            sp.clear();

            // Leave nothing behind, even after a failed check.

            if let Ok(left) = store.list(&probe) {

                for entry in left {

                    let _ = store.delete(&entry.key);

                }

            }

            println!("{} {}", ui::err(ui::glyph("✘", "x")), e);

            println!(

                "  {} {}",

                ui::note(),

                ui::dim("a store that fails this cannot keep a repository safely; check the service and the credentials' permissions")

            );

            Ok(1)

        }

    }

}

fn describe_holding(

    id: [u8; 16],

    mine: [u8; 16],

    bookmarks: usize,

    tags: usize,

    bundles: Option<usize>,

) {

    let short: String = id[..6].iter().map(|b| format!("{b:02x}")).collect();

    let mut what = format!(

        "repository {short}: {} and {}",

        plural(bookmarks, "bookmark"),

        plural(tags, "tag")

    );

    if let Some(b) = bundles {

        what.push_str(&format!(" in {}", plural(b, "bundle")));

    }

    if id == mine {

        println!("  {} {}", ui::note(), ui::dim(format!("holds this {what}")));

    } else if id == oxy_core::protocol::UNBOUND_ID {

        println!("  {} {}", ui::note(), ui::dim("holds no repository yet"));

    } else {

        println!(

            "  {} {}",

            ui::caution(),

            ui::warn(format!("holds another {what}; a push would be refused"))

        );

    }