OXYGENHUB
# ce118287d9da oxygen / crates / oxy-strata / src / aws.rs
7.3 KiB · Rust · last changed Strata: S3, Google Cloud Storage and Azure Blob Storage stores 1 month ago source blame history raw
1
//! Amazon S3: `s3://BUCKET/PREFIX`.
2
//!
3
//! Credentials, in the order the AWS tools use them:
4
//! 1. `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (and
5
//!    `AWS_SESSION_TOKEN`) in the environment, unless the URL names a
6
//!    profile;
7
//! 2. the profile (`?profile=NAME`, else `AWS_PROFILE`, else `default`):
8
//!    its keys from `~/.aws/credentials`;
9
//! 3. what the AWS CLI can produce for that profile
10
//!    (`aws configure export-credentials`): SSO, login sessions, assumed
11
//!    roles and credential processes, so anything that works for `aws`
12
//!    works for `oxy`;
13
//! 4. the instance's or container's role, through the provider's own
14
//!    lookup.
15
//!
16
//! The region comes from `?region=`, the environment, the profile, or is
17
//! asked of S3 itself; the endpoint (`?endpoint=`, for MinIO and other
18
//! S3-alikes) from the URL, the environment or the profile.
19
20
use crate::ini;
21
use crate::StoreUrl;
22
use object_store::aws::{AmazonS3, AmazonS3Builder, S3ConditionalPut};
23
use object_store::ClientOptions;
24
use oxy_core::strata::{Error, Result};
25
use std::path::PathBuf;
26
27
pub const HINT: &str = "set AWS_PROFILE (or add ?profile=NAME to the remote URL) to a profile in ~/.aws/credentials, run `aws sso login` or `aws login` for it, or set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY";
28
29
struct Found {
30
    key_id: String,
31
    secret: String,
32
    token: Option<String>,
33
}
34
35
fn env(name: &str) -> Option<String> {
36
    std::env::var(name).ok().filter(|v| !v.is_empty())
37
}
38
39
fn home() -> Option<PathBuf> {
40
    env("HOME")
41
        .or_else(|| env("USERPROFILE"))
42
        .map(PathBuf::from)
43
}
44
45
fn read_ini(env_override: &str, default_rel: &str) -> ini::Sections {
46
    let path = env(env_override)
47
        .map(PathBuf::from)
48
        .or_else(|| home().map(|h| h.join(default_rel)));
49
    path.and_then(|p| std::fs::read_to_string(p).ok())
50
        .map(|t| ini::parse(&t))
51
        .unwrap_or_default()
52
}
53
54
/// A profile's section in `~/.aws/config`: `[profile NAME]`, or
55
/// `[default]`; `~/.aws/credentials` uses `[NAME]` for both.
56
fn config_section<'a>(
57
    config: &'a ini::Sections,
58
    profile: &str,
59
) -> Option<&'a std::collections::BTreeMap<String, String>> {
60
    if profile == "default" {
61
        config.get("default")
62
    } else {
63
        config
64
            .get(&format!("profile {profile}"))
65
            .or_else(|| config.get(profile))
66
    }
67
}
68
69
/// Ask the AWS CLI for the profile's credentials, if it is installed and
70
/// knows the profile.
71
fn from_cli(profile: &str) -> Option<Found> {
72
    let out = std::process::Command::new("aws")
73
        .args([
74
            "configure",
75
            "export-credentials",
76
            "--profile",
77
            profile,
78
            "--format",
79
            "process",
80
        ])
81
        .stdin(std::process::Stdio::null())
82
        .stderr(std::process::Stdio::null())
83
        .output()
84
        .ok()?;
85
    if !out.status.success() {
86
        return None;
87
    }
88
    let v: serde_json::Value = serde_json::from_slice(&out.stdout).ok()?;
89
    Some(Found {
90
        key_id: v.get("AccessKeyId")?.as_str()?.to_string(),
91
        secret: v.get("SecretAccessKey")?.as_str()?.to_string(),
92
        token: v
93
            .get("SessionToken")
94
            .and_then(|t| t.as_str())
95
            .map(str::to_string),
96
    })
97
}
98
99
pub fn open(url: &StoreUrl, runtime: &tokio::runtime::Runtime) -> Result<AmazonS3> {
100
    let bucket = url.host.clone();
101
    let explicit_profile = url.param("profile").map(str::to_string);
102
    let profile = explicit_profile
103
        .clone()
104
        .or_else(|| env("AWS_PROFILE"))
105
        .unwrap_or_else(|| "default".into());
106
    let config = read_ini("AWS_CONFIG_FILE", ".aws/config");
107
    let creds = read_ini("AWS_SHARED_CREDENTIALS_FILE", ".aws/credentials");
108
    let section = config_section(&config, &profile);
109
    let known_profile = section.is_some() || creds.contains_key(&profile);
110
111
    // from_env picks up the environment's keys, region, endpoint, and the
112
    // container and instance roles; what follows fills in or overrides.
113
    let mut b = AmazonS3Builder::from_env()
114
        .with_bucket_name(&bucket)
115
        .with_conditional_put(S3ConditionalPut::ETagMatch);
116
117
    let env_keys = match (env("AWS_ACCESS_KEY_ID"), env("AWS_SECRET_ACCESS_KEY")) {
118
        (Some(k), Some(s)) if explicit_profile.is_none() => Some(Found {
119
            key_id: k,
120
            secret: s,
121
            token: env("AWS_SESSION_TOKEN"),
122
        }),
123
        _ => None,
124
    };
125
    let found = env_keys
126
        .or_else(|| {
127
            let c = creds.get(&profile)?;
128
            Some(Found {
129
                key_id: c.get("aws_access_key_id")?.clone(),
130
                secret: c.get("aws_secret_access_key")?.clone(),
131
                token: c.get("aws_session_token").cloned(),
132
            })
133
        })
134
        .or_else(|| {
135
            if known_profile {
136
                from_cli(&profile)
137
            } else {
138
                None
139
            }
140
        });
141
    match found {
142
        Some(f) => {
143
            b = b
144
                .with_access_key_id(f.key_id)
145
                .with_secret_access_key(f.secret);
146
            if let Some(t) = f.token {
147
                b = b.with_token(t);
148
            }
149
        }
150
        None if explicit_profile.is_some() && !known_profile => {
151
            return Err(Error::Backend(format!(
152
                "{}: profile {profile:?} is not in ~/.aws/config or ~/.aws/credentials: {HINT}",
153
                url.plain()
154
            )));
155
        }
156
        None => {}
157
    }
158
159
    let endpoint = url
160
        .param("endpoint")
161
        .map(str::to_string)
162
        .or_else(|| env("AWS_ENDPOINT_URL_S3"))
163
        .or_else(|| env("AWS_ENDPOINT_URL"))
164
        .or_else(|| env("AWS_ENDPOINT"))
165
        .or_else(|| section.and_then(|s| s.get("endpoint_url").cloned()));
166
    if let Some(e) = &endpoint {
167
        b = b
168
            .with_endpoint(e.clone())
169
            .with_allow_http(e.starts_with("http://"));
170
        // S3-alikes serve buckets by path, not by host name.
171
        b = b.with_virtual_hosted_style_request(false);
172
    }
173
174
    let region = url
175
        .param("region")
176
        .map(str::to_string)
177
        .or_else(|| env("AWS_REGION"))
178
        .or_else(|| env("AWS_DEFAULT_REGION"))
179
        .or_else(|| section.and_then(|s| s.get("region").cloned()));
180
    let region = match region {
181
        Some(r) => r,
182
        None if endpoint.is_some() => "us-east-1".into(),
183
        None => runtime
184
            .block_on(object_store::aws::resolve_bucket_region(&bucket, &ClientOptions::new()))
185
            .map_err(|e| {
186
                Error::Backend(format!(
187
                    "{}: no region was given and S3 did not say which region holds the bucket ({e}): add ?region=NAME to the remote URL, set AWS_REGION, or put region in the profile",
188
                    url.plain()
189
                ))
190
            })?,
191
    };
192
    b = b.with_region(region);
193
194
    b.build()
195
        .map_err(|e| Error::Backend(format!("{}: {e}: {HINT}", url.plain())))
196
}
197
198
#[cfg(test)]
199
mod tests {
200
    use super::*;
201
202
    #[test]
203
    fn profile_sections_are_found_under_either_heading() {
204
        let config =
205
            ini::parse("[default]\nregion = a\n[profile work]\nregion = b\n[plain]\nregion = c\n");
206
        assert_eq!(config_section(&config, "default").unwrap()["region"], "a");
207
        assert_eq!(config_section(&config, "work").unwrap()["region"], "b");
208
        assert_eq!(config_section(&config, "plain").unwrap()["region"], "c");
209
        assert!(config_section(&config, "none").is_none());
210
    }
211
}