Cloud stores take credentials handed over 7026bd7dc5b8

- **Cloud stores take credentials handed over (#68).** `oxy_strata::Credentials` (an S3 access key, a Google service account key, an Azure service principal) can be given when a store is opened, and a program can register a lookup for the stores it opens by URL. With them, nothing from the environment or the providers' files is used, which lets a hub keep each repository's credentials itself. Their debug form never shows a secret.

/// With `keys` given, only they are used: nothing from the environment

/// or a profile, and the region and endpoint come from the URL alone.

pub fn open(url: &StoreUrl, keys: Option<(&str, &str, Option<&str>)>) -> Result<AmazonS3> {

    if let Some((id, secret, token)) = keys {

        let mut b = AmazonS3Builder::new()

            .with_bucket_name(&url.host)

            .with_http_connector(crate::http::UreqConnector)

            .with_conditional_put(S3ConditionalPut::ETagMatch)

            .with_access_key_id(id)

            .with_secret_access_key(secret);

        if let Some(t) = token {

            b = b.with_token(t);

        }

        let endpoint = url.param("endpoint").map(str::to_string);

        return finish(b, url, endpoint, url.param("region").map(str::to_string));

    }

    let region = url

        .param("region")

        .map(str::to_string)

        .or_else(|| env("AWS_REGION"))

        .or_else(|| env("AWS_DEFAULT_REGION"))

        .or_else(|| section.and_then(|s| s.get("region").cloned()));

    finish(b, url, endpoint, region)

}

/// The endpoint and region, then the store.

fn finish(

    mut b: AmazonS3Builder,

    url: &StoreUrl,

    endpoint: Option<String>,

    region: Option<String>,

) -> Result<AmazonS3> {

    let bucket = url.host.clone();

/// With `principal` (tenant, client id, client secret) given, only it

/// is used.

pub fn open(

    url: &StoreUrl,

    principal: Option<(&str, &str, &str)>,

) -> Result<(MicrosoftAzure, String)> {

    if let Some((tenant, id, secret)) = principal {

        let mut b = MicrosoftAzureBuilder::new()

            .with_account(&url.host)

            .with_container_name(&container)

            .with_http_connector(crate::http::UreqConnector)

            .with_client_secret_authorization(id, secret, tenant);

        if let Some(endpoint) = url.param("endpoint") {

            b = b

                .with_endpoint(endpoint.to_string())

                .with_allow_http(endpoint.starts_with("http://"));

        }

        let store = b

            .build()

            .map_err(|e| Error::Backend(format!("{}: {e}", url.plain())))?;

        return Ok((store, prefix));

    }

/// With `key` (a service account key's JSON) given, only it is used.

pub fn open(url: &StoreUrl, key: Option<&str>) -> Result<GoogleCloudStorage> {

    if let Some(key) = key {

        let mut b = GoogleCloudStorageBuilder::new()

            .with_bucket_name(&url.host)

            .with_http_connector(crate::http::UreqConnector)

            .with_service_account_key(key);

        if let Some(endpoint) = url.param("endpoint") {

            b = b.with_base_url(endpoint);

        }

        return b

            .build()

            .map_err(|e| Error::Backend(format!("{}: {e}", url.plain())));

    }

    let lookup = *CREDENTIALS.lock().unwrap_or_else(|p| p.into_inner());

    let given = lookup.and_then(|f| f(url));

    Some(CloudStore::open_with(url, given.as_ref()).map(|s| Box::new(s) as Box<dyn Store>))

/// Credentials a program keeps itself and hands over, instead of the

/// provider's usual places: a hub keeps each repository's sealed. When a

/// store is opened with them, nothing from the environment or the

/// provider's files is used.

#[derive(Clone, PartialEq, Eq)]

pub enum Credentials {

    /// An S3 access key.

    Aws {

        access_key_id: String,

        secret_access_key: String,

        session_token: Option<String>,

    },

    /// A Google Cloud service account key, as the JSON Google issues.

    Gcp { service_account_key: String },

    /// An Azure service principal.

    Azure {

        tenant_id: String,

        client_id: String,

        client_secret: String,

    },

}

impl std::fmt::Debug for Credentials {

    /// Names the kind and the public part only: never the secret.

    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

        match self {

            Credentials::Aws { access_key_id, .. } => {

                write!(f, "Aws({access_key_id}, secret hidden)")

            }

            Credentials::Gcp { .. } => write!(f, "Gcp(service account key hidden)"),

            Credentials::Azure { client_id, .. } => write!(f, "Azure({client_id}, secret hidden)"),

        }

    }

}

impl Credentials {

    /// The scheme of the stores these are for.

    pub fn scheme(&self) -> &'static str {

        match self {

            Credentials::Aws { .. } => "s3",

            Credentials::Gcp { .. } => "gs",

            Credentials::Azure { .. } => "az",

        }

    }

    /// As JSON, for keeping (sealed) and reading back with [`Credentials::from_json`].

    pub fn to_json(&self) -> String {

        let v = match self {

            Credentials::Aws {

                access_key_id,

                secret_access_key,

                session_token,

            } => serde_json::json!({

                "kind": "aws", "access_key_id": access_key_id, "secret_access_key": secret_access_key,

                "session_token": session_token,

            }),

            Credentials::Gcp {

                service_account_key,

            } => {

                serde_json::json!({ "kind": "gcp", "service_account_key": service_account_key })

            }

            Credentials::Azure {

                tenant_id,

                client_id,

                client_secret,

            } => serde_json::json!({

                "kind": "azure", "tenant_id": tenant_id, "client_id": client_id, "client_secret": client_secret,

            }),

        };

        v.to_string()

    }

    pub fn from_json(text: &str) -> Result<Credentials> {

        let v: serde_json::Value = serde_json::from_str(text)

            .map_err(|e| Error::Backend(format!("credentials are not JSON: {e}")))?;

        let field = |k: &str| -> Result<String> {

            v.get(k)

                .and_then(|x| x.as_str())

                .filter(|x| !x.trim().is_empty())

                .map(|x| x.trim().to_string())

                .ok_or_else(|| Error::Backend(format!("credentials need {k}")))

        };

        match v.get("kind").and_then(|k| k.as_str()) {

            Some("aws") => Ok(Credentials::Aws {

                access_key_id: field("access_key_id")?,

                secret_access_key: field("secret_access_key")?,

                session_token: field("session_token").ok(),

            }),

            Some("gcp") => {

                let key = field("service_account_key")?;

                let parsed: serde_json::Value = serde_json::from_str(&key).map_err(|_| {

                    Error::Backend("the service account key is not the JSON Google issues".into())

                })?;

                if parsed.get("type").and_then(|t| t.as_str()) != Some("service_account") {

                    return Err(Error::Backend(

                        "the key is not a service account key (its type is not service_account)"

                            .into(),

                    ));

                }

                Ok(Credentials::Gcp {

                    service_account_key: key,

                })

            }

            Some("azure") => Ok(Credentials::Azure {

                tenant_id: field("tenant_id")?,

                client_id: field("client_id")?,

                client_secret: field("client_secret")?,

            }),

            _ => Err(Error::Backend(

                "credentials need a kind: aws, gcp or azure".into(),

            )),

        }

    }

}

/// Finds the credentials a program keeps for a store URL.

pub type CredentialsLookup = fn(&str) -> Option<Credentials>;

static CREDENTIALS: Mutex<Option<CredentialsLookup>> = Mutex::new(None);

/// Have stores opened by URL (as a repository's `storage.url` is) use the

/// credentials `lookup` finds for it, where it finds any.

pub fn set_credentials_lookup(lookup: CredentialsLookup) {

    *CREDENTIALS.lock().unwrap_or_else(|p| p.into_inner()) = Some(lookup);

}

        Self::open_with(url, None)

    }

    /// [`CloudStore::open`] with credentials given, which must be for

    /// the URL's kind of store; then only they are used.

    pub fn open_with(url: &str, given: Option<&Credentials>) -> Result<CloudStore> {

        if let Some(c) = given.filter(|c| c.scheme() != parsed.scheme) {

            return Err(Error::Backend(format!(

                "{}: the credentials given are for {}:// stores",

                parsed.plain(),

                c.scheme()

            )));

        }

        let aws_keys = match given {

            Some(Credentials::Aws {

                access_key_id,

                secret_access_key,

                session_token,

            }) => Some((

                access_key_id.as_str(),

                secret_access_key.as_str(),

                session_token.as_deref(),

            )),

            _ => None,

        };

        let gcp_key = match given {

            Some(Credentials::Gcp {

                service_account_key,

            }) => Some(service_account_key.as_str()),

            _ => None,

        };

        let principal = match given {

            Some(Credentials::Azure {

                tenant_id,

                client_id,

                client_secret,

            }) => Some((

                tenant_id.as_str(),

                client_id.as_str(),

                client_secret.as_str(),

            )),

            _ => None,

        };

                    Arc::new(aws::open(&parsed, aws_keys)?),

                    Arc::new(gcp::open(&parsed, gcp_key)?),

                    let (store, prefix) = azure::open(&parsed, principal)?;

    #[test]

    fn credentials_round_trip_and_never_show_their_secret() {

        let aws = Credentials::Aws {

            access_key_id: "AKIDEXAMPLE".into(),

            secret_access_key: "wJalrXUtnFEMI".into(),

            session_token: None,

        };

        assert_eq!(Credentials::from_json(&aws.to_json()).unwrap(), aws);

        assert_eq!(format!("{aws:?}"), "Aws(AKIDEXAMPLE, secret hidden)");

        let az = Credentials::Azure {

            tenant_id: "t".into(),

            client_id: "c".into(),

            client_secret: "s3cr3t".into(),

        };

        assert_eq!(Credentials::from_json(&az.to_json()).unwrap(), az);

        assert!(!format!("{az:?}").contains("s3cr3t"));

        let key = r#"{"type":"service_account","client_email":"x@y","private_key":"k"}"#;

        let gcp = Credentials::Gcp {

            service_account_key: key.into(),

        };

        assert_eq!(Credentials::from_json(&gcp.to_json()).unwrap(), gcp);

        assert!(!format!("{gcp:?}").contains("private_key"));

        for (bad, why) in [

            (r#"{"kind":"aws","access_key_id":"a"}"#, "secret_access_key"),

            (

                r#"{"kind":"gcp","service_account_key":"{\"type\":\"authorized_user\"}"}"#,

                "not a service account key",

            ),

            (r#"{"kind":"ftp"}"#, "aws, gcp or azure"),

            ("nope", "not JSON"),

        ] {

            let e = Credentials::from_json(bad).unwrap_err().to_string();

            assert!(e.contains(why), "{bad}: {e}");

        }

        let e = CloudStore::open_with("gs://bucket/p", Some(&aws))

            .err()

            .unwrap()

            .to_string();

        assert!(e.contains("are for s3:// stores"), "{e}");

    }

/// Each store opened with credentials handed over (`CloudStore::open_with`,

/// as a hub does with the ones it keeps sealed), the profile and key file

/// parameters left off the URL, passes the conformance check.

#[test]

fn every_configured_store_conforms_with_credentials_handed_over() {

    use oxy_strata::Credentials;

    let env = |k: &str| std::env::var(k).ok().filter(|v| !v.is_empty());

    let mut tried = 0;

    if let (Some(url), Some(profile)) = (env("OXY_TEST_S3"), env("OXY_TEST_S3_PROFILE")) {

        let home = std::path::PathBuf::from(env("HOME").unwrap());

        let text = std::fs::read_to_string(home.join(".aws/credentials")).unwrap_or_default();

        let mut section = String::new();

        let mut keys = std::collections::BTreeMap::new();

        for line in text.lines().map(str::trim) {

            if let Some(name) = line.strip_prefix('[').and_then(|l| l.strip_suffix(']')) {

                section = name.to_string();

            } else if let Some((k, v)) = line.split_once('=') {

                if section == profile {

                    keys.insert(k.trim().to_string(), v.trim().to_string());

                }

            }

        }

        if let (Some(id), Some(secret)) = (

            keys.get("aws_access_key_id"),

            keys.get("aws_secret_access_key"),

        ) {

            let c = Credentials::Aws {

                access_key_id: id.clone(),

                secret_access_key: secret.clone(),

                session_token: keys.get("aws_session_token").cloned(),

            };

            check_handed_over(&url, &c);

            tried += 1;

        }

    }

    if let (Some(url), Some(file)) = (env("OXY_TEST_GCS"), env("OXY_TEST_GCS_CREDENTIALS")) {

        let c = Credentials::Gcp {

            service_account_key: std::fs::read_to_string(file).unwrap(),

        };

        check_handed_over(&url, &c);

        tried += 1;

    }

    if let (Some(url), Some(file)) = (env("OXY_TEST_AZURE"), env("OXY_TEST_AZURE_CREDENTIALS")) {

        let mut vars = std::collections::BTreeMap::new();

        for line in std::fs::read_to_string(file).unwrap().lines() {

            if let Some((k, v)) = line.split_once('=') {

                vars.insert(k.trim().to_string(), v.trim().to_string());

            }

        }

        let c = Credentials::Azure {

            tenant_id: vars["AZURE_TENANT_ID"].clone(),

            client_id: vars["AZURE_CLIENT_ID"].clone(),

            client_secret: vars["AZURE_CLIENT_SECRET"].clone(),

        };

        check_handed_over(&url, &c);

        tried += 1;

    }

    eprintln!("{tried} stores checked with credentials handed over");

}

fn check_handed_over(url: &str, c: &oxy_strata::Credentials) {

    let store =

        oxy_strata::CloudStore::open_with(url, Some(c)).unwrap_or_else(|e| panic!("{url}: {e}"));

    let prefix = format!("probe-{}", std::process::id());

    let passed = conformance(&store, &prefix).unwrap_or_else(|e| {

        remove_all(&store, &prefix);

        panic!("{url} with {c:?}: {e}")

    });

    eprintln!("{url} with {c:?}: {} checks passed", passed.len());

    assert_eq!(passed.len(), 8);

}