aws.rs 63.4%

118 of 186 lines run, 16 of 61 functions · run #427 on main · run missed

//! Amazon S3: `s3://BUCKET/PREFIX`.
//!
//! Credentials, in the order the AWS tools use them:
//! 1. `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (and
//!    `AWS_SESSION_TOKEN`) in the environment, unless the URL names a
//!    profile;
//! 2. the profile (`?profile=NAME`, else `AWS_PROFILE`, else `default`):
//!    its keys from `~/.aws/credentials`;
//! 3. what the AWS CLI can produce for that profile
//!    (`aws configure export-credentials`): SSO, login sessions, assumed
//!    roles and credential processes, so anything that works for `aws`
//!    works for `oxy`;
//! 4. the instance's or container's role, through the provider's own
//!    lookup.
//!
//! The region comes from `?region=`, the environment, the profile, or is
//! asked of S3 itself; the endpoint (`?endpoint=`, for MinIO and other
//! S3-alikes) from the URL, the environment or the profile.

use crate::ini;
use crate::StoreUrl;
use object_store::aws::{AmazonS3, AmazonS3Builder, S3ConditionalPut};
use oxy_core::strata::{Error, Result};
use std::path::PathBuf;

pub const HINT: &str = "set AWS_PROFILE (or add ?profile=NAME to the remote URL) to a profile in ~/.aws/credentials, run `aws sso login` or `aws login` for it, or set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY";

struct Found {
    key_id: String,
    secret: String,
    token: Option<String>,
}

fn env(name: &str) -> Option<String> {
    std::env::var(name).ok().filter(|v| !v.is_empty())
}

fn home() -> Option<PathBuf> {
    env("HOME")
        .or_else(|| env("USERPROFILE"))
        .map(PathBuf::from)
}

fn read_ini(env_override: &str, default_rel: &str) -> ini::Sections {
    let path = env(env_override)
        .map(PathBuf::from)
        .or_else(|| home().map(|h| h.join(default_rel)));
    path.and_then(|p| std::fs::read_to_string(p).ok())
        .map(|t| ini::parse(&t))
        .unwrap_or_default()
}

/// A profile's section in `~/.aws/config`: `[profile NAME]`, or
/// `[default]`; `~/.aws/credentials` uses `[NAME]` for both.
fn config_section<'a>(
    config: &'a ini::Sections,
    profile: &str,
) -> Option<&'a std::collections::BTreeMap<String, String>> {
    if profile == "default" {
        config.get("default")
    } else {
        config
            .get(&format!("profile {profile}"))
            .or_else(|| config.get(profile))
    }
}

/// Ask the AWS CLI for the profile's credentials, if it is installed and
/// knows the profile.
fn from_cli(profile: &str) -> Option<Found> {
    let out = std::process::Command::new("aws")
        .args([
            "configure",
            "export-credentials",
            "--profile",
            profile,
            "--format",
            "process",
        ])
        .stdin(std::process::Stdio::null())
        .stderr(std::process::Stdio::null())
        .output()
        .ok()?;
    if !out.status.success() {
        return None;
    }
    let v: serde_json::Value = serde_json::from_slice(&out.stdout).ok()?;
    Some(Found {
        key_id: v.get("AccessKeyId")?.as_str()?.to_string(),
        secret: v.get("SecretAccessKey")?.as_str()?.to_string(),
        token: v
            .get("SessionToken")
            .and_then(|t| t.as_str())
            .map(str::to_string),
    })
}

/// With `keys` given, only they are used: nothing from the environment
/// or a profile, and the region and endpoint come from the URL alone.
pub fn open(url: &StoreUrl, keys: Option<(&str, &str, Option<&str>)>) -> Result<AmazonS3> {
    if let Some((id, secret, token)) = keys {
        let mut b = AmazonS3Builder::new()
            .with_bucket_name(&url.host)
            .with_http_connector(crate::http::UreqConnector)
            .with_conditional_put(S3ConditionalPut::ETagMatch)
            .with_access_key_id(id)
            .with_secret_access_key(secret);
        if let Some(t) = token {
            b = b.with_token(t);
        }
        let endpoint = url.param("endpoint").map(str::to_string);
        return finish(b, url, endpoint, url.param("region").map(str::to_string));
    }
    let bucket = url.host.clone();
    let explicit_profile = url.param("profile").map(str::to_string);
    let profile = explicit_profile
        .clone()
        .or_else(|| env("AWS_PROFILE"))
        .unwrap_or_else(|| "default".into());
    let config = read_ini("AWS_CONFIG_FILE", ".aws/config");
    let creds = read_ini("AWS_SHARED_CREDENTIALS_FILE", ".aws/credentials");
    let section = config_section(&config, &profile);
    let known_profile = section.is_some() || creds.contains_key(&profile);

    // from_env picks up the environment's keys, region, endpoint, and the
    // container and instance roles; what follows fills in or overrides.
    let mut b = AmazonS3Builder::from_env()
        .with_bucket_name(&bucket)
        .with_http_connector(crate::http::UreqConnector)
        .with_conditional_put(S3ConditionalPut::ETagMatch);

    let env_keys = match (env("AWS_ACCESS_KEY_ID"), env("AWS_SECRET_ACCESS_KEY")) {
        (Some(k), Some(s)) if explicit_profile.is_none() => Some(Found {
            key_id: k,
            secret: s,
            token: env("AWS_SESSION_TOKEN"),
        }),
        _ => None,
    };
    let found = env_keys
        .or_else(|| {
            let c = creds.get(&profile)?;
            Some(Found {
                key_id: c.get("aws_access_key_id")?.clone(),
                secret: c.get("aws_secret_access_key")?.clone(),
                token: c.get("aws_session_token").cloned(),
            })
        })
        .or_else(|| {
            if known_profile {
                from_cli(&profile)
            } else {
                None
            }
        });
    match found {
        Some(f) => {
            b = b
                .with_access_key_id(f.key_id)
                .with_secret_access_key(f.secret);
            if let Some(t) = f.token {
                b = b.with_token(t);
            }
        }
        None if explicit_profile.is_some() && !known_profile => {
            return Err(Error::Backend(format!(
                "{}: profile {profile:?} is not in ~/.aws/config or ~/.aws/credentials: {HINT}",
                url.plain()
            )));
        }
        None => {}
    }

    let endpoint = url
        .param("endpoint")
        .map(str::to_string)
        .or_else(|| env("AWS_ENDPOINT_URL_S3"))
        .or_else(|| env("AWS_ENDPOINT_URL"))
        .or_else(|| env("AWS_ENDPOINT"))
        .or_else(|| section.and_then(|s| s.get("endpoint_url").cloned()));
    let region = url
        .param("region")
        .map(str::to_string)
        .or_else(|| env("AWS_REGION"))
        .or_else(|| env("AWS_DEFAULT_REGION"))
        .or_else(|| section.and_then(|s| s.get("region").cloned()));
    finish(b, url, endpoint, region)
}

/// The endpoint and region, then the store.
fn finish(
    mut b: AmazonS3Builder,
    url: &StoreUrl,
    endpoint: Option<String>,
    region: Option<String>,
) -> Result<AmazonS3> {
    let bucket = url.host.clone();
    if let Some(e) = &endpoint {
        b = b
            .with_endpoint(e.clone())
            .with_allow_http(e.starts_with("http://"));
        // S3-alikes serve buckets by path, not by host name.
        b = b.with_virtual_hosted_style_request(false);
    }

    let region = match region {
        Some(r) => r,
        None if endpoint.is_some() => "us-east-1".into(),
        None => crate::http::bucket_region(&bucket).ok_or_else(|| {
            Error::Backend(format!(
                "{}: no region was given and S3 did not say which region holds the bucket: add ?region=NAME to the remote URL, set AWS_REGION, or put region in the profile",
                url.plain()
            ))
        })?,
    };
    b = b.with_region(region);

    b.build()
        .map_err(|e| Error::Backend(format!("{}: {e}: {HINT}", url.plain())))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn profile_sections_are_found_under_either_heading() {
        let config =
            ini::parse("[default]\nregion = a\n[profile work]\nregion = b\n[plain]\nregion = c\n");
        assert_eq!(config_section(&config, "default").unwrap()["region"], "a");
        assert_eq!(config_section(&config, "work").unwrap()["region"], "b");
        assert_eq!(config_section(&config, "plain").unwrap()["region"], "c");
        assert!(config_section(&config, "none").is_none());
    }
}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1		//! Amazon S3: `s3://BUCKET/PREFIX`.
2		//!
3		//! Credentials, in the order the AWS tools use them:
4		//! 1. `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (and
5		//! `AWS_SESSION_TOKEN`) in the environment, unless the URL names a
6		//! profile;
7		//! 2. the profile (`?profile=NAME`, else `AWS_PROFILE`, else `default`):
8		//! its keys from `~/.aws/credentials`;
9		//! 3. what the AWS CLI can produce for that profile
10		//! (`aws configure export-credentials`): SSO, login sessions, assumed
11		//! roles and credential processes, so anything that works for `aws`
12		//! works for `oxy`;
13		//! 4. the instance's or container's role, through the provider's own
14		//! lookup.
15		//!
16		//! The region comes from `?region=`, the environment, the profile, or is
17		//! asked of S3 itself; the endpoint (`?endpoint=`, for MinIO and other
18		//! S3-alikes) from the URL, the environment or the profile.
19
20		use crate::ini;
21		use crate::StoreUrl;
22		use object_store::aws::{AmazonS3, AmazonS3Builder, S3ConditionalPut};
23		use oxy_core::strata::{Error, Result};
24		use std::path::PathBuf;
25
26		pub const HINT: &str = "set AWS_PROFILE (or add ?profile=NAME to the remote URL) to a profile in ~/.aws/credentials, run `aws sso login` or `aws login` for it, or set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY";
27
28		struct Found {
29		key_id: String,
30		secret: String,
31		token: Option<String>,
32		}
33
34	7×	fn env(name: &str) -> Option<String> {
35	7×	std::env::var(name).ok().filter(\|v\| !v.is_empty())
36	7×	}
37
38	2×	fn home() -> Option<PathBuf> {
39	2×	env("HOME")
40	2×	.or_else(\|\| env("USERPROFILE"))
41	2×	.map(PathBuf::from)
42	2×	}
43
44	2×	fn read_ini(env_override: &str, default_rel: &str) -> ini::Sections {
45	2×	let path = env(env_override)
46	2×	.map(PathBuf::from)
47	2×	.or_else(\|\| home().map(\|h\| h.join(default_rel)));
48	2×	path.and_then(\|p\| std::fs::read_to_string(p).ok())
49	2×	.map(\|t\| ini::parse(&t))
50	2×	.unwrap_or_default()
51	2×	}
52
53		/// A profile's section in `~/.aws/config`: `[profile NAME]`, or
54		/// `[default]`; `~/.aws/credentials` uses `[NAME]` for both.
55	5×	fn config_section<'a>(
56	5×	config: &'a ini::Sections,
57	5×	profile: &str,
58	5×	) -> Option<&'a std::collections::BTreeMap<String, String>> {
59	5×	if profile == "default" {
60	2×	config.get("default")
61		} else {
62	3×	config
63	3×	.get(&format!("profile {profile}"))
64	3×	.or_else(\|\| config.get(profile))
65		}
66	5×	}
67
68		/// Ask the AWS CLI for the profile's credentials, if it is installed and
69		/// knows the profile.
70	0×	fn from_cli(profile: &str) -> Option<Found> {
71	0×	let out = std::process::Command::new("aws")
72	0×	.args([
73	0×	"configure",
74	0×	"export-credentials",
75	0×	"--profile",
76	0×	profile,
77	0×	"--format",
78	0×	"process",
79	0×	])
80	0×	.stdin(std::process::Stdio::null())
81	0×	.stderr(std::process::Stdio::null())
82	0×	.output()
83	0×	.ok()?;
84	0×	if !out.status.success() {
85	0×	return None;
86	0×	}
87	0×	let v: serde_json::Value = serde_json::from_slice(&out.stdout).ok()?;
88		Some(Found {
89	0×	key_id: v.get("AccessKeyId")?.as_str()?.to_string(),
90	0×	secret: v.get("SecretAccessKey")?.as_str()?.to_string(),
91	0×	token: v
92	0×	.get("SessionToken")
93	0×	.and_then(\|t\| t.as_str())
94	0×	.map(str::to_string),
95		})
96	0×	}
97
98		/// With `keys` given, only they are used: nothing from the environment
99		/// or a profile, and the region and endpoint come from the URL alone.
100	3×	pub fn open(url: &StoreUrl, keys: Option<(&str, &str, Option<&str>)>) -> Result<AmazonS3> {
101	3×	if let Some((id, secret, token)) = keys {
102	2×	let mut b = AmazonS3Builder::new()
103	2×	.with_bucket_name(&url.host)
104	2×	.with_http_connector(crate::http::UreqConnector)
105	2×	.with_conditional_put(S3ConditionalPut::ETagMatch)
106	2×	.with_access_key_id(id)
107	2×	.with_secret_access_key(secret);
108	2×	if let Some(t) = token {
109	0×	b = b.with_token(t);
110	2×	}
111	2×	let endpoint = url.param("endpoint").map(str::to_string);
112	2×	return finish(b, url, endpoint, url.param("region").map(str::to_string));
113	1×	}
114	1×	let bucket = url.host.clone();
115	1×	let explicit_profile = url.param("profile").map(str::to_string);
116	1×	let profile = explicit_profile
117	1×	.clone()
118	1×	.or_else(\|\| env("AWS_PROFILE"))
119	1×	.unwrap_or_else(\|\| "default".into());
120	1×	let config = read_ini("AWS_CONFIG_FILE", ".aws/config");
121	1×	let creds = read_ini("AWS_SHARED_CREDENTIALS_FILE", ".aws/credentials");
122	1×	let section = config_section(&config, &profile);
123	1×	let known_profile = section.is_some() \|\| creds.contains_key(&profile);
124
125		// from_env picks up the environment's keys, region, endpoint, and the
126		// container and instance roles; what follows fills in or overrides.
127	1×	let mut b = AmazonS3Builder::from_env()
128	1×	.with_bucket_name(&bucket)
129	1×	.with_http_connector(crate::http::UreqConnector)
130	1×	.with_conditional_put(S3ConditionalPut::ETagMatch);
131
132	1×	let env_keys = match (env("AWS_ACCESS_KEY_ID"), env("AWS_SECRET_ACCESS_KEY")) {
133	0×	(Some(k), Some(s)) if explicit_profile.is_none() => Some(Found {
134	0×	key_id: k,
135	0×	secret: s,
136	0×	token: env("AWS_SESSION_TOKEN"),
137	0×	}),
138	1×	_ => None,
139		};
140	1×	let found = env_keys
141	1×	.or_else(\|\| {
142	1×	let c = creds.get(&profile)?;
143		Some(Found {
144	0×	key_id: c.get("aws_access_key_id")?.clone(),
145	0×	secret: c.get("aws_secret_access_key")?.clone(),
146	0×	token: c.get("aws_session_token").cloned(),
147		})
148	1×	})
149	1×	.or_else(\|\| {
150	1×	if known_profile {
151	0×	from_cli(&profile)
152		} else {
153	1×	None
154		}
155	1×	});
156	1×	match found {
157	0×	Some(f) => {
158	0×	b = b
159	0×	.with_access_key_id(f.key_id)
160	0×	.with_secret_access_key(f.secret);
161	0×	if let Some(t) = f.token {
162	0×	b = b.with_token(t);
163	0×	}
164		}
165	1×	None if explicit_profile.is_some() && !known_profile => {
166	0×	return Err(Error::Backend(format!(
167	0×	"{}: profile {profile:?} is not in ~/.aws/config or ~/.aws/credentials: {HINT}",
168	0×	url.plain()
169	0×	)));
170		}
171	1×	None => {}
172		}
173
174	1×	let endpoint = url
175	1×	.param("endpoint")
176	1×	.map(str::to_string)
177	1×	.or_else(\|\| env("AWS_ENDPOINT_URL_S3"))
178	1×	.or_else(\|\| env("AWS_ENDPOINT_URL"))
179	1×	.or_else(\|\| env("AWS_ENDPOINT"))
180	1×	.or_else(\|\| section.and_then(\|s\| s.get("endpoint_url").cloned()));
181	1×	let region = url
182	1×	.param("region")
183	1×	.map(str::to_string)
184	1×	.or_else(\|\| env("AWS_REGION"))
185	1×	.or_else(\|\| env("AWS_DEFAULT_REGION"))
186	1×	.or_else(\|\| section.and_then(\|s\| s.get("region").cloned()));
187	1×	finish(b, url, endpoint, region)
188	3×	}
189
190		/// The endpoint and region, then the store.
191	3×	fn finish(
192	3×	mut b: AmazonS3Builder,
193	3×	url: &StoreUrl,
194	3×	endpoint: Option<String>,
195	3×	region: Option<String>,
196	3×	) -> Result<AmazonS3> {
197	3×	let bucket = url.host.clone();
198	3×	if let Some(e) = &endpoint {
199	1×	b = b
200	1×	.with_endpoint(e.clone())
201	1×	.with_allow_http(e.starts_with("http://"));
202	1×	// S3-alikes serve buckets by path, not by host name.
203	1×	b = b.with_virtual_hosted_style_request(false);
204	2×	}
205
206	3×	let region = match region {
207	3×	Some(r) => r,
208	0×	None if endpoint.is_some() => "us-east-1".into(),
209	0×	None => crate::http::bucket_region(&bucket).ok_or_else(\|\| {
210	0×	Error::Backend(format!(
211	0×	"{}: no region was given and S3 did not say which region holds the bucket: add ?region=NAME to the remote URL, set AWS_REGION, or put region in the profile",
212	0×	url.plain()
213	0×	))
214	0×	})?,
215		};
216	3×	b = b.with_region(region);
217
218	3×	b.build()
219	3×	.map_err(\|e\| Error::Backend(format!("{}: {e}: {HINT}", url.plain())))
220	3×	}
221
222		#[cfg(test)]
223		mod tests {
224		use super::*;
225
226		#[test]
227	1×	fn profile_sections_are_found_under_either_heading() {
228	1×	let config =
229	1×	ini::parse("[default]\nregion = a\n[profile work]\nregion = b\n[plain]\nregion = c\n");
230	1×	assert_eq!(config_section(&config, "default").unwrap()["region"], "a");
231	1×	assert_eq!(config_section(&config, "work").unwrap()["region"], "b");
232	1×	assert_eq!(config_section(&config, "plain").unwrap()["region"], "c");
233	1×	assert!(config_section(&config, "none").is_none());
234	1×	}
235		}

Coverage / crates/oxy-strata/src/aws.rs 63.4%

Keyboard