OXYGENHUB
◆ v0.9.0 oxygen / crates / oxy-strata / src / http.rs
4.8 KiB · Rust · last changed Strata: the cloud stores send through ureq, so every release target links 1 month ago source blame history raw
2cd40ea1 Andy McCall ↶ 1
//! The HTTP client the cloud stores send their requests through: `ureq`
2
//! on rustls with the web roots, the client oxy-core itself uses, so the
3
//! stores cross-compile for every target oxy is released for. Each
4
//! request runs on the runtime's blocking pool; bodies are held whole,
5
//! which is how the stores use them (a bundle, a record, a listing).
6
7
use async_trait::async_trait;
8
use bytes::Bytes;
9
use http_body_util::{BodyExt, Full};
10
use object_store::client::{
11
    ClientOptions, HttpClient, HttpConnector, HttpError, HttpErrorKind, HttpRequest, HttpResponse,
12
    HttpResponseBody, HttpService,
13
};
14
use std::io::Read;
15
use std::time::Duration;
16
17
/// How long a request may take in all, and to connect.
18
const TIMEOUT: Duration = Duration::from_secs(600);
19
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
20
21
/// A rustls configuration trusting the web roots.
22
fn tls() -> Result<std::sync::Arc<rustls::ClientConfig>, String> {
23
    oxy_core::protocol::tls_client_config(None).map_err(|e| e.to_string())
24
}
25
26
/// Make the agent every request of a store goes through.
27
pub fn agent() -> Result<ureq::Agent, String> {
28
    Ok(ureq::AgentBuilder::new()
29
        .tls_config(tls()?)
30
        .timeout_connect(CONNECT_TIMEOUT)
31
        .timeout(TIMEOUT)
32
        .build())
33
}
34
35
#[derive(Debug, Clone, Copy, Default)]
36
pub struct UreqConnector;
37
38
impl HttpConnector for UreqConnector {
39
    fn connect(&self, _options: &ClientOptions) -> object_store::Result<HttpClient> {
40
        let agent = agent().map_err(|e| object_store::Error::Generic {
41
            store: "ureq",
42
            source: e.into(),
43
        })?;
44
        Ok(HttpClient::new(UreqService { agent }))
45
    }
46
}
47
48
#[derive(Debug, Clone)]
49
struct UreqService {
50
    agent: ureq::Agent,
51
}
52
53
#[async_trait]
54
impl HttpService for UreqService {
55
    async fn call(&self, req: HttpRequest) -> Result<HttpResponse, HttpError> {
56
        let (parts, body) = req.into_parts();
57
        let bytes = body.collect().await?.to_bytes();
58
        let agent = self.agent.clone();
59
        tokio::task::spawn_blocking(move || send(&agent, parts, &bytes))
60
            .await
61
            .map_err(|e| HttpError::new(HttpErrorKind::Interrupted, e))?
62
    }
63
}
64
65
fn send(
66
    agent: &ureq::Agent,
67
    parts: http::request::Parts,
68
    bytes: &[u8],
69
) -> Result<HttpResponse, HttpError> {
70
    let url = parts.uri.to_string();
71
    let mut r = agent.request(parts.method.as_str(), &url);
72
    for (name, value) in &parts.headers {
73
        if let Ok(v) = value.to_str() {
74
            r = r.set(name.as_str(), v);
75
        }
76
    }
77
    let sent = if bytes.is_empty()
78
        && matches!(
79
            parts.method,
80
            http::Method::GET | http::Method::HEAD | http::Method::DELETE
81
        ) {
82
        r.call()
83
    } else {
84
        r.send_bytes(bytes)
85
    };
86
    let resp = match sent {
87
        Ok(resp) => resp,
88
        // A refusal is a response the store interprets (a 404, a 412).
89
        Err(ureq::Error::Status(_, resp)) => resp,
90
        Err(ureq::Error::Transport(t)) => {
91
            let kind = match t.kind() {
92
                ureq::ErrorKind::Dns
93
                | ureq::ErrorKind::ConnectionFailed
94
                | ureq::ErrorKind::ProxyConnect => HttpErrorKind::Connect,
95
                ureq::ErrorKind::Io => HttpErrorKind::Timeout,
96
                _ => HttpErrorKind::Request,
97
            };
98
            return Err(HttpError::new(kind, t));
99
        }
100
    };
101
    let status = http::StatusCode::from_u16(resp.status())
102
        .map_err(|e| HttpError::new(HttpErrorKind::Decode, e))?;
103
    let mut builder = http::Response::builder().status(status);
104
    for name in resp.headers_names() {
105
        for value in resp.all(&name) {
106
            builder = builder.header(name.as_str(), value);
107
        }
108
    }
109
    let mut body = Vec::new();
110
    resp.into_reader()
111
        .read_to_end(&mut body)
112
        .map_err(|e| HttpError::new(HttpErrorKind::Decode, e))?;
113
    let body = HttpResponseBody::new(
114
        Full::new(Bytes::from(body)).map_err(|never: std::convert::Infallible| match never {}),
115
    );
116
    builder
117
        .body(body)
118
        .map_err(|e| HttpError::new(HttpErrorKind::Decode, e))
119
}
120
121
/// The region that holds an S3 bucket, as S3 itself reports it in
122
/// `x-amz-bucket-region` on any request, even a refused one.
123
pub fn bucket_region(bucket: &str) -> Option<String> {
124
    let agent = ureq::AgentBuilder::new()
125
        .tls_config(tls().ok()?)
126
        .timeout(Duration::from_secs(30))
127
        .redirects(0)
128
        .build();
129
    // S3's wildcard certificate covers one label: dotted names go by path.
130
    let url = if bucket.contains('.') {
131
        format!("https://s3.amazonaws.com/{bucket}")
132
    } else {
133
        format!("https://{bucket}.s3.amazonaws.com")
134
    };
135
    let resp = match agent.head(&url).call() {
136
        Ok(r) | Err(ureq::Error::Status(_, r)) => r,
137
        Err(_) => return None,
138
    };
139
    resp.header("x-amz-bucket-region").map(str::to_string)
140
}