OXYGENHUB
◆ v0.4.0 oxygen / crates / oxy-strata / src / aws.rs
7.2 KiB · Rust · last changed Strata: the cloud stores send through ureq, so every release target links 1 month ago source blame history raw
1
//! Amazon S3: `s3://BUCKET/PREFIX`.
2
//!
3
//! Credentials, in the order the AWS tools use them:
4
//! 1. `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (and
5
//!    `AWS_SESSION_TOKEN`) in the environment, unless the URL names a
6
//!    profile;
7
//! 2. the profile (`?profile=NAME`, else `AWS_PROFILE`, else `default`):
8
//!    its keys from `~/.aws/credentials`;
9
//! 3. what the AWS CLI can produce for that profile
10
//!    (`aws configure export-credentials`): SSO, login sessions, assumed
11
//!    roles and credential processes, so anything that works for `aws`
12
//!    works for `oxy`;
13
//! 4. the instance's or container's role, through the provider's own
14
//!    lookup.
15
//!
16
//! The region comes from `?region=`, the environment, the profile, or is
17
//! asked of S3 itself; the endpoint (`?endpoint=`, for MinIO and other
18
//! S3-alikes) from the URL, the environment or the profile.
19
20
use crate::ini;
21
use crate::StoreUrl;
22
use object_store::aws::{AmazonS3, AmazonS3Builder, S3ConditionalPut};
23
use oxy_core::strata::{Error, Result};
24
use std::path::PathBuf;
25
26
pub const HINT: &str = "set AWS_PROFILE (or add ?profile=NAME to the remote URL) to a profile in ~/.aws/credentials, run `aws sso login` or `aws login` for it, or set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY";
27
28
struct Found {
29
    key_id: String,
30
    secret: String,
31
    token: Option<String>,
32
}
33
34
fn env(name: &str) -> Option<String> {
35
    std::env::var(name).ok().filter(|v| !v.is_empty())
36
}
37
38
fn home() -> Option<PathBuf> {
39
    env("HOME")
40
        .or_else(|| env("USERPROFILE"))
41
        .map(PathBuf::from)
42
}
43
44
fn read_ini(env_override: &str, default_rel: &str) -> ini::Sections {
45
    let path = env(env_override)
46
        .map(PathBuf::from)
47
        .or_else(|| home().map(|h| h.join(default_rel)));
48
    path.and_then(|p| std::fs::read_to_string(p).ok())
49
        .map(|t| ini::parse(&t))
50
        .unwrap_or_default()
51
}
52
53
/// A profile's section in `~/.aws/config`: `[profile NAME]`, or
54
/// `[default]`; `~/.aws/credentials` uses `[NAME]` for both.
55
fn config_section<'a>(
56
    config: &'a ini::Sections,
57
    profile: &str,
58
) -> Option<&'a std::collections::BTreeMap<String, String>> {
59
    if profile == "default" {
60
        config.get("default")
61
    } else {
62
        config
63
            .get(&format!("profile {profile}"))
64
            .or_else(|| config.get(profile))
65
    }
66
}
67
68
/// Ask the AWS CLI for the profile's credentials, if it is installed and
69
/// knows the profile.
70
fn from_cli(profile: &str) -> Option<Found> {
71
    let out = std::process::Command::new("aws")
72
        .args([
73
            "configure",
74
            "export-credentials",
75
            "--profile",
76
            profile,
77
            "--format",
78
            "process",
79
        ])
80
        .stdin(std::process::Stdio::null())
81
        .stderr(std::process::Stdio::null())
82
        .output()
83
        .ok()?;
84
    if !out.status.success() {
85
        return None;
86
    }
87
    let v: serde_json::Value = serde_json::from_slice(&out.stdout).ok()?;
88
    Some(Found {
89
        key_id: v.get("AccessKeyId")?.as_str()?.to_string(),
90
        secret: v.get("SecretAccessKey")?.as_str()?.to_string(),
91
        token: v
92
            .get("SessionToken")
93
            .and_then(|t| t.as_str())
94
            .map(str::to_string),
95
    })
96
}
97
98
pub fn open(url: &StoreUrl) -> Result<AmazonS3> {
99
    let bucket = url.host.clone();
100
    let explicit_profile = url.param("profile").map(str::to_string);
101
    let profile = explicit_profile
102
        .clone()
103
        .or_else(|| env("AWS_PROFILE"))
104
        .unwrap_or_else(|| "default".into());
105
    let config = read_ini("AWS_CONFIG_FILE", ".aws/config");
106
    let creds = read_ini("AWS_SHARED_CREDENTIALS_FILE", ".aws/credentials");
107
    let section = config_section(&config, &profile);
108
    let known_profile = section.is_some() || creds.contains_key(&profile);
109
110
    // from_env picks up the environment's keys, region, endpoint, and the
111
    // container and instance roles; what follows fills in or overrides.
112
    let mut b = AmazonS3Builder::from_env()
113
        .with_bucket_name(&bucket)
114
        .with_http_connector(crate::http::UreqConnector)
115
        .with_conditional_put(S3ConditionalPut::ETagMatch);
116
117
    let env_keys = match (env("AWS_ACCESS_KEY_ID"), env("AWS_SECRET_ACCESS_KEY")) {
118
        (Some(k), Some(s)) if explicit_profile.is_none() => Some(Found {
119
            key_id: k,
120
            secret: s,
121
            token: env("AWS_SESSION_TOKEN"),
122
        }),
123
        _ => None,
124
    };
125
    let found = env_keys
126
        .or_else(|| {
127
            let c = creds.get(&profile)?;
128
            Some(Found {
129
                key_id: c.get("aws_access_key_id")?.clone(),
130
                secret: c.get("aws_secret_access_key")?.clone(),
131
                token: c.get("aws_session_token").cloned(),
132
            })
133
        })
134
        .or_else(|| {
135
            if known_profile {
136
                from_cli(&profile)
137
            } else {
138
                None
139
            }
140
        });
141
    match found {
142
        Some(f) => {
143
            b = b
144
                .with_access_key_id(f.key_id)
145
                .with_secret_access_key(f.secret);
146
            if let Some(t) = f.token {
147
                b = b.with_token(t);
148
            }
149
        }
150
        None if explicit_profile.is_some() && !known_profile => {
151
            return Err(Error::Backend(format!(
152
                "{}: profile {profile:?} is not in ~/.aws/config or ~/.aws/credentials: {HINT}",
153
                url.plain()
154
            )));
155
        }
156
        None => {}
157
    }
158
159
    let endpoint = url
160
        .param("endpoint")
161
        .map(str::to_string)
162
        .or_else(|| env("AWS_ENDPOINT_URL_S3"))
163
        .or_else(|| env("AWS_ENDPOINT_URL"))
164
        .or_else(|| env("AWS_ENDPOINT"))
165
        .or_else(|| section.and_then(|s| s.get("endpoint_url").cloned()));
166
    if let Some(e) = &endpoint {
167
        b = b
168
            .with_endpoint(e.clone())
169
            .with_allow_http(e.starts_with("http://"));
170
        // S3-alikes serve buckets by path, not by host name.
171
        b = b.with_virtual_hosted_style_request(false);
172
    }
173
174
    let region = url
175
        .param("region")
176
        .map(str::to_string)
177
        .or_else(|| env("AWS_REGION"))
178
        .or_else(|| env("AWS_DEFAULT_REGION"))
179
        .or_else(|| section.and_then(|s| s.get("region").cloned()));
180
    let region = match region {
181
        Some(r) => r,
182
        None if endpoint.is_some() => "us-east-1".into(),
183
        None => crate::http::bucket_region(&bucket).ok_or_else(|| {
184
            Error::Backend(format!(
185
                "{}: no region was given and S3 did not say which region holds the bucket: add ?region=NAME to the remote URL, set AWS_REGION, or put region in the profile",
186
                url.plain()
187
            ))
188
        })?,
189
    };
190
    b = b.with_region(region);
191
192
    b.build()
193
        .map_err(|e| Error::Backend(format!("{}: {e}: {HINT}", url.plain())))
194
}
195
196
#[cfg(test)]
197
mod tests {
198
    use super::*;
199
200
    #[test]
201
    fn profile_sections_are_found_under_either_heading() {
202
        let config =
203
            ini::parse("[default]\nregion = a\n[profile work]\nregion = b\n[plain]\nregion = c\n");
204
        assert_eq!(config_section(&config, "default").unwrap()["region"], "a");
205
        assert_eq!(config_section(&config, "work").unwrap()["region"], "b");
206
        assert_eq!(config_section(&config, "plain").unwrap()["region"], "c");
207
        assert!(config_section(&config, "none").is_none());
208
    }
209
}