OXYGENHUB
# f2f91ceb0e9f oxygen / crates / oxy-strata / src / aws.rs
8.2 KiB · Rust · last changed Cloud stores take credentials handed over 1 month ago source blame history raw
1
//! Amazon S3: `s3://BUCKET/PREFIX`.
2
//!
3
//! Credentials, in the order the AWS tools use them:
4
//! 1. `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (and
5
//!    `AWS_SESSION_TOKEN`) in the environment, unless the URL names a
6
//!    profile;
7
//! 2. the profile (`?profile=NAME`, else `AWS_PROFILE`, else `default`):
8
//!    its keys from `~/.aws/credentials`;
9
//! 3. what the AWS CLI can produce for that profile
10
//!    (`aws configure export-credentials`): SSO, login sessions, assumed
11
//!    roles and credential processes, so anything that works for `aws`
12
//!    works for `oxy`;
13
//! 4. the instance's or container's role, through the provider's own
14
//!    lookup.
15
//!
16
//! The region comes from `?region=`, the environment, the profile, or is
17
//! asked of S3 itself; the endpoint (`?endpoint=`, for MinIO and other
18
//! S3-alikes) from the URL, the environment or the profile.
19
20
use crate::ini;
21
use crate::StoreUrl;
22
use object_store::aws::{AmazonS3, AmazonS3Builder, S3ConditionalPut};
23
use oxy_core::strata::{Error, Result};
24
use std::path::PathBuf;
25
26
pub const HINT: &str = "set AWS_PROFILE (or add ?profile=NAME to the remote URL) to a profile in ~/.aws/credentials, run `aws sso login` or `aws login` for it, or set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY";
27
28
struct Found {
29
    key_id: String,
30
    secret: String,
31
    token: Option<String>,
32
}
33
34
fn env(name: &str) -> Option<String> {
35
    std::env::var(name).ok().filter(|v| !v.is_empty())
36
}
37
38
fn home() -> Option<PathBuf> {
39
    env("HOME")
40
        .or_else(|| env("USERPROFILE"))
41
        .map(PathBuf::from)
42
}
43
44
fn read_ini(env_override: &str, default_rel: &str) -> ini::Sections {
45
    let path = env(env_override)
46
        .map(PathBuf::from)
47
        .or_else(|| home().map(|h| h.join(default_rel)));
48
    path.and_then(|p| std::fs::read_to_string(p).ok())
49
        .map(|t| ini::parse(&t))
50
        .unwrap_or_default()
51
}
52
53
/// A profile's section in `~/.aws/config`: `[profile NAME]`, or
54
/// `[default]`; `~/.aws/credentials` uses `[NAME]` for both.
55
fn config_section<'a>(
56
    config: &'a ini::Sections,
57
    profile: &str,
58
) -> Option<&'a std::collections::BTreeMap<String, String>> {
59
    if profile == "default" {
60
        config.get("default")
61
    } else {
62
        config
63
            .get(&format!("profile {profile}"))
64
            .or_else(|| config.get(profile))
65
    }
66
}
67
68
/// Ask the AWS CLI for the profile's credentials, if it is installed and
69
/// knows the profile.
70
fn from_cli(profile: &str) -> Option<Found> {
71
    let out = std::process::Command::new("aws")
72
        .args([
73
            "configure",
74
            "export-credentials",
75
            "--profile",
76
            profile,
77
            "--format",
78
            "process",
79
        ])
80
        .stdin(std::process::Stdio::null())
81
        .stderr(std::process::Stdio::null())
82
        .output()
83
        .ok()?;
84
    if !out.status.success() {
85
        return None;
86
    }
87
    let v: serde_json::Value = serde_json::from_slice(&out.stdout).ok()?;
88
    Some(Found {
89
        key_id: v.get("AccessKeyId")?.as_str()?.to_string(),
90
        secret: v.get("SecretAccessKey")?.as_str()?.to_string(),
91
        token: v
92
            .get("SessionToken")
93
            .and_then(|t| t.as_str())
94
            .map(str::to_string),
95
    })
96
}
97
98
/// With `keys` given, only they are used: nothing from the environment
99
/// or a profile, and the region and endpoint come from the URL alone.
100
pub fn open(url: &StoreUrl, keys: Option<(&str, &str, Option<&str>)>) -> Result<AmazonS3> {
101
    if let Some((id, secret, token)) = keys {
102
        let mut b = AmazonS3Builder::new()
103
            .with_bucket_name(&url.host)
104
            .with_http_connector(crate::http::UreqConnector)
105
            .with_conditional_put(S3ConditionalPut::ETagMatch)
106
            .with_access_key_id(id)
107
            .with_secret_access_key(secret);
108
        if let Some(t) = token {
109
            b = b.with_token(t);
110
        }
111
        let endpoint = url.param("endpoint").map(str::to_string);
112
        return finish(b, url, endpoint, url.param("region").map(str::to_string));
113
    }
114
    let bucket = url.host.clone();
115
    let explicit_profile = url.param("profile").map(str::to_string);
116
    let profile = explicit_profile
117
        .clone()
118
        .or_else(|| env("AWS_PROFILE"))
119
        .unwrap_or_else(|| "default".into());
120
    let config = read_ini("AWS_CONFIG_FILE", ".aws/config");
121
    let creds = read_ini("AWS_SHARED_CREDENTIALS_FILE", ".aws/credentials");
122
    let section = config_section(&config, &profile);
123
    let known_profile = section.is_some() || creds.contains_key(&profile);
124
125
    // from_env picks up the environment's keys, region, endpoint, and the
126
    // container and instance roles; what follows fills in or overrides.
127
    let mut b = AmazonS3Builder::from_env()
128
        .with_bucket_name(&bucket)
129
        .with_http_connector(crate::http::UreqConnector)
130
        .with_conditional_put(S3ConditionalPut::ETagMatch);
131
132
    let env_keys = match (env("AWS_ACCESS_KEY_ID"), env("AWS_SECRET_ACCESS_KEY")) {
133
        (Some(k), Some(s)) if explicit_profile.is_none() => Some(Found {
134
            key_id: k,
135
            secret: s,
136
            token: env("AWS_SESSION_TOKEN"),
137
        }),
138
        _ => None,
139
    };
140
    let found = env_keys
141
        .or_else(|| {
142
            let c = creds.get(&profile)?;
143
            Some(Found {
144
                key_id: c.get("aws_access_key_id")?.clone(),
145
                secret: c.get("aws_secret_access_key")?.clone(),
146
                token: c.get("aws_session_token").cloned(),
147
            })
148
        })
149
        .or_else(|| {
150
            if known_profile {
151
                from_cli(&profile)
152
            } else {
153
                None
154
            }
155
        });
156
    match found {
157
        Some(f) => {
158
            b = b
159
                .with_access_key_id(f.key_id)
160
                .with_secret_access_key(f.secret);
161
            if let Some(t) = f.token {
162
                b = b.with_token(t);
163
            }
164
        }
165
        None if explicit_profile.is_some() && !known_profile => {
166
            return Err(Error::Backend(format!(
167
                "{}: profile {profile:?} is not in ~/.aws/config or ~/.aws/credentials: {HINT}",
168
                url.plain()
169
            )));
170
        }
171
        None => {}
172
    }
173
174
    let endpoint = url
175
        .param("endpoint")
176
        .map(str::to_string)
177
        .or_else(|| env("AWS_ENDPOINT_URL_S3"))
178
        .or_else(|| env("AWS_ENDPOINT_URL"))
179
        .or_else(|| env("AWS_ENDPOINT"))
180
        .or_else(|| section.and_then(|s| s.get("endpoint_url").cloned()));
181
    let region = url
182
        .param("region")
183
        .map(str::to_string)
184
        .or_else(|| env("AWS_REGION"))
185
        .or_else(|| env("AWS_DEFAULT_REGION"))
186
        .or_else(|| section.and_then(|s| s.get("region").cloned()));
187
    finish(b, url, endpoint, region)
188
}
189
190
/// The endpoint and region, then the store.
191
fn finish(
192
    mut b: AmazonS3Builder,
193
    url: &StoreUrl,
194
    endpoint: Option<String>,
195
    region: Option<String>,
196
) -> Result<AmazonS3> {
197
    let bucket = url.host.clone();
198
    if let Some(e) = &endpoint {
199
        b = b
200
            .with_endpoint(e.clone())
201
            .with_allow_http(e.starts_with("http://"));
202
        // S3-alikes serve buckets by path, not by host name.
203
        b = b.with_virtual_hosted_style_request(false);
204
    }
205
206
    let region = match region {
207
        Some(r) => r,
208
        None if endpoint.is_some() => "us-east-1".into(),
209
        None => crate::http::bucket_region(&bucket).ok_or_else(|| {
210
            Error::Backend(format!(
211
                "{}: no region was given and S3 did not say which region holds the bucket: add ?region=NAME to the remote URL, set AWS_REGION, or put region in the profile",
212
                url.plain()
213
            ))
214
        })?,
215
    };
216
    b = b.with_region(region);
217
218
    b.build()
219
        .map_err(|e| Error::Backend(format!("{}: {e}: {HINT}", url.plain())))
220
}
221
222
#[cfg(test)]
223
mod tests {
224
    use super::*;
225
226
    #[test]
227
    fn profile_sections_are_found_under_either_heading() {
228
        let config =
229
            ini::parse("[default]\nregion = a\n[profile work]\nregion = b\n[plain]\nregion = c\n");
230
        assert_eq!(config_section(&config, "default").unwrap()["region"], "a");
231
        assert_eq!(config_section(&config, "work").unwrap()["region"], "b");
232
        assert_eq!(config_section(&config, "plain").unwrap()["region"], "c");
233
        assert!(config_section(&config, "none").is_none());
234
    }
235
}