OXYGENHUB
◆ carry-identical oxygen / crates / oxy-strata / src / lib.rs
25.0 KiB · Rust · last changed A cloud store can be used from inside an async runtime 1 month ago source blame history raw
1 
//! Oxygen Strata: repositories kept on cloud object storage (docs/stores.md).
2 
//!
3 
//! This crate gives `oxy` the stores that need an HTTP client and the
4 
//! providers' signing: Amazon S3 (`s3://BUCKET/PREFIX`), Google Cloud
5 
//! Storage (`gs://BUCKET/PREFIX`) and Azure Blob Storage
6 
//! (`az://ACCOUNT/CONTAINER/PREFIX`), each as a [`Store`] over the
7 
//! `object_store` crate. Credentials come from each provider's standard
8 
//! places, with nothing oxy-specific to set up; a URL may carry settings
9 
//! as query parameters (`?profile=work&region=eu-west-2`), which is where
10 
//! `oxy remote add --profile` keeps them.
11 
//!
12 
//! [`register`] hands these stores to oxy-core's remote opener; a program
13 
//! that links this crate calls it once at start.
14
15 
mod aws;
16 
mod azure;
17 
mod gcp;
18 
mod http;
19 
mod ini;
20
21 
use futures::TryStreamExt;
22 
use object_store::path::Path as ObjectPath;
23 
use object_store::{
24 
    GetOptions, ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload, UpdateVersion,
25 
};
26 
use oxy_core::strata::{check_key, Entry, Error, Result, Store, Version};
27 
use std::collections::BTreeMap;
28 
use std::future::Future;
29 
use std::ops::Range;
30 
use std::sync::{Arc, Mutex};
31
32 
/// Make the cloud stores available to `oxy_core::protocol::open_remote`.
33 
/// Safe to call more than once.
34 
pub fn register() {
35 
    // object_store's reqwest is built with rustls and no bundled crypto
36 
    // provider; oxy links ring already, so that is the one to install.
37 
    let _ = rustls::crypto::ring::default_provider().install_default();
38 
    oxy_core::protocol::register_store_opener(open_url);
39 
}
40
41 
fn open_url(
42 
    url: &str,
43 
    _options: &oxy_core::protocol::RemoteOptions,
44 
) -> Option<Result<Box<dyn Store>>> {
45 
    let scheme = url.split_once("://")?.0;
46 
    if !matches!(scheme, "s3" | "gs" | "az") {
47 
        return None;
48 
    }
49 
    let lookup = *CREDENTIALS.lock().unwrap_or_else(|p| p.into_inner());
50 
    let given = lookup.and_then(|f| f(url));
51 
    Some(CloudStore::open_with(url, given.as_ref()).map(|s| Box::new(s) as Box<dyn Store>))
52 
}
53
54 
/// Credentials a program keeps itself and hands over, instead of the
55 
/// provider's usual places: a hub keeps each repository's sealed. When a
56 
/// store is opened with them, nothing from the environment or the
57 
/// provider's files is used.
58 
#[derive(Clone, PartialEq, Eq)]
59 
pub enum Credentials {
60 
    /// An S3 access key.
61 
    Aws {
62 
        access_key_id: String,
63 
        secret_access_key: String,
64 
        session_token: Option<String>,
65 
    },
66 
    /// A Google Cloud service account key, as the JSON Google issues.
67 
    Gcp { service_account_key: String },
68 
    /// An Azure service principal.
69 
    Azure {
70 
        tenant_id: String,
71 
        client_id: String,
72 
        client_secret: String,
73 
    },
74 
}
75
76 
impl std::fmt::Debug for Credentials {
77 
    /// Names the kind and the public part only: never the secret.
78 
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
79 
        match self {
80 
            Credentials::Aws { access_key_id, .. } => {
81 
                write!(f, "Aws({access_key_id}, secret hidden)")
82 
            }
83 
            Credentials::Gcp { .. } => write!(f, "Gcp(service account key hidden)"),
84 
            Credentials::Azure { client_id, .. } => write!(f, "Azure({client_id}, secret hidden)"),
85 
        }
86 
    }
87 
}
88
89 
impl Credentials {
90 
    /// The scheme of the stores these are for.
91 
    pub fn scheme(&self) -> &'static str {
92 
        match self {
93 
            Credentials::Aws { .. } => "s3",
94 
            Credentials::Gcp { .. } => "gs",
95 
            Credentials::Azure { .. } => "az",
96 
        }
97 
    }
98
99 
    /// As JSON, for keeping (sealed) and reading back with [`Credentials::from_json`].
100 
    pub fn to_json(&self) -> String {
101 
        let v = match self {
102 
            Credentials::Aws {
103 
                access_key_id,
104 
                secret_access_key,
105 
                session_token,
106 
            } => serde_json::json!({
107 
                "kind": "aws", "access_key_id": access_key_id, "secret_access_key": secret_access_key,
108 
                "session_token": session_token,
109 
            }),
110 
            Credentials::Gcp {
111 
                service_account_key,
112 
            } => {
113 
                serde_json::json!({ "kind": "gcp", "service_account_key": service_account_key })
114 
            }
115 
            Credentials::Azure {
116 
                tenant_id,
117 
                client_id,
118 
                client_secret,
119 
            } => serde_json::json!({
120 
                "kind": "azure", "tenant_id": tenant_id, "client_id": client_id, "client_secret": client_secret,
121 
            }),
122 
        };
123 
        v.to_string()
124 
    }
125
126 
    pub fn from_json(text: &str) -> Result<Credentials> {
127 
        let v: serde_json::Value = serde_json::from_str(text)
128 
            .map_err(|e| Error::Backend(format!("credentials are not JSON: {e}")))?;
129 
        let field = |k: &str| -> Result<String> {
130 
            v.get(k)
131 
                .and_then(|x| x.as_str())
132 
                .filter(|x| !x.trim().is_empty())
133 
                .map(|x| x.trim().to_string())
134 
                .ok_or_else(|| Error::Backend(format!("credentials need {k}")))
135 
        };
136 
        match v.get("kind").and_then(|k| k.as_str()) {
137 
            Some("aws") => Ok(Credentials::Aws {
138 
                access_key_id: field("access_key_id")?,
139 
                secret_access_key: field("secret_access_key")?,
140 
                session_token: field("session_token").ok(),
141 
            }),
142 
            Some("gcp") => {
143 
                let key = field("service_account_key")?;
144 
                let parsed: serde_json::Value = serde_json::from_str(&key).map_err(|_| {
145 
                    Error::Backend("the service account key is not the JSON Google issues".into())
146 
                })?;
147 
                if parsed.get("type").and_then(|t| t.as_str()) != Some("service_account") {
148 
                    return Err(Error::Backend(
149 
                        "the key is not a service account key (its type is not service_account)"
150 
                            .into(),
151 
                    ));
152 
                }
153 
                Ok(Credentials::Gcp {
154 
                    service_account_key: key,
155 
                })
156 
            }
157 
            Some("azure") => Ok(Credentials::Azure {
158 
                tenant_id: field("tenant_id")?,
159 
                client_id: field("client_id")?,
160 
                client_secret: field("client_secret")?,
161 
            }),
162 
            _ => Err(Error::Backend(
163 
                "credentials need a kind: aws, gcp or azure".into(),
164 
            )),
165 
        }
166 
    }
167 
}
168
169 
/// Finds the credentials a program keeps for a store URL.
170 
pub type CredentialsLookup = fn(&str) -> Option<Credentials>;
171
172 
static CREDENTIALS: Mutex<Option<CredentialsLookup>> = Mutex::new(None);
173
174 
/// Have stores opened by URL (as a repository's `storage.url` is) use the
175 
/// credentials `lookup` finds for it, where it finds any.
176 
pub fn set_credentials_lookup(lookup: CredentialsLookup) {
177 
    *CREDENTIALS.lock().unwrap_or_else(|p| p.into_inner()) = Some(lookup);
178 
}
179
180 
/// A parsed store URL: `SCHEME://HOST/PATH?PARAMS`.
181 
#[derive(Clone, Debug, PartialEq, Eq)]
182 
pub struct StoreUrl {
183 
    pub scheme: String,
184 
    /// The bucket (S3, GCS) or the storage account (Azure).
185 
    pub host: String,
186 
    /// The path under the host, without leading or trailing slashes; for
187 
    /// Azure the container is its first segment.
188 
    pub path: String,
189 
    pub params: BTreeMap<String, String>,
190 
}
191
192 
impl StoreUrl {
193 
    pub fn parse(url: &str) -> Result<StoreUrl> {
194 
        let bad = |why: &str| Error::Backend(format!("{url}: {why}"));
195 
        let (scheme, rest) = url.split_once("://").ok_or_else(|| bad("not a URL"))?;
196 
        let (rest, query) = match rest.split_once('?') {
197 
            Some((r, q)) => (r, Some(q)),
198 
            None => (rest, None),
199 
        };
200 
        let (host, path) = match rest.split_once('/') {
201 
            Some((h, p)) => (h, p.trim_matches('/')),
202 
            None => (rest, ""),
203 
        };
204 
        if host.is_empty() {
205 
            return Err(bad("names no bucket"));
206 
        }
207 
        let mut params = BTreeMap::new();
208 
        for pair in query.unwrap_or("").split('&').filter(|p| !p.is_empty()) {
209 
            let (k, v) = pair.split_once('=').unwrap_or((pair, ""));
210 
            params.insert(k.to_string(), percent_decode(v));
211 
        }
212 
        Ok(StoreUrl {
213 
            scheme: scheme.to_string(),
214 
            host: host.to_string(),
215 
            path: path.to_string(),
216 
            params,
217 
        })
218 
    }
219
220 
    /// The URL without its parameters, for messages.
221 
    pub fn plain(&self) -> String {
222 
        if self.path.is_empty() {
223 
            format!("{}://{}", self.scheme, self.host)
224 
        } else {
225 
            format!("{}://{}/{}", self.scheme, self.host, self.path)
226 
        }
227 
    }
228
229 
    pub fn param(&self, name: &str) -> Option<&str> {
230 
        self.params.get(name).map(String::as_str)
231 
    }
232 
}
233
234 
fn percent_decode(s: &str) -> String {
235 
    let bytes = s.as_bytes();
236 
    let mut out = Vec::with_capacity(bytes.len());
237 
    let mut i = 0;
238 
    while i < bytes.len() {
239 
        if bytes[i] == b'%' && i + 2 < bytes.len() {
240 
            if let Ok(v) = u8::from_str_radix(&s[i + 1..i + 3], 16) {
241 
                out.push(v);
242 
                i += 3;
243 
                continue;
244 
            }
245 
        }
246 
        out.push(bytes[i]);
247 
        i += 1;
248 
    }
249 
    String::from_utf8_lossy(&out).into_owned()
250 
}
251
252 
/// A store on a cloud provider: the provider's client, the prefix every
253 
/// key goes under, and a runtime to run the client on.
254 
pub struct CloudStore {
255 
    inner: Arc<dyn ObjectStore>,
256 
    prefix: String,
257 
    name: String,
258 
    hint: &'static str,
259 
    /// Always there until the store is dropped.
260 
    runtime: Option<tokio::runtime::Runtime>,
261 
    /// One call at a time: the runtime runs on the calling thread.
262 
    turn: Mutex<()>,
263 
}
264
265 
impl Drop for CloudStore {
266 
    /// Without waiting, so a store can be dropped inside a runtime too.
267 
    fn drop(&mut self) {
268 
        if let Some(rt) = self.runtime.take() {
269 
            rt.shutdown_background();
270 
        }
271 
    }
272 
}
273
274 
impl CloudStore {
275 
    /// Open the store a URL names. Nothing is sent yet: credentials are
276 
    /// resolved here, but the bucket is first spoken to on the first call.
277 
    pub fn open(url: &str) -> Result<CloudStore> {
278 
        Self::open_with(url, None)
279 
    }
280
281 
    /// [`CloudStore::open`] with credentials given, which must be for
282 
    /// the URL's kind of store; then only they are used.
283 
    pub fn open_with(url: &str, given: Option<&Credentials>) -> Result<CloudStore> {
284 
        let parsed = StoreUrl::parse(url)?;
285 
        if let Some(c) = given.filter(|c| c.scheme() != parsed.scheme) {
286 
            return Err(Error::Backend(format!(
287 
                "{}: the credentials given are for {}:// stores",
288 
                parsed.plain(),
289 
                c.scheme()
290 
            )));
291 
        }
292 
        let aws_keys = match given {
293 
            Some(Credentials::Aws {
294 
                access_key_id,
295 
                secret_access_key,
296 
                session_token,
297 
            }) => Some((
298 
                access_key_id.as_str(),
299 
                secret_access_key.as_str(),
300 
                session_token.as_deref(),
301 
            )),
302 
            _ => None,
303 
        };
304 
        let gcp_key = match given {
305 
            Some(Credentials::Gcp {
306 
                service_account_key,
307 
            }) => Some(service_account_key.as_str()),
308 
            _ => None,
309 
        };
310 
        let principal = match given {
311 
            Some(Credentials::Azure {
312 
                tenant_id,
313 
                client_id,
314 
                client_secret,
315 
            }) => Some((
316 
                tenant_id.as_str(),
317 
                client_id.as_str(),
318 
                client_secret.as_str(),
319 
            )),
320 
            _ => None,
321 
        };
322 
        let runtime = tokio::runtime::Builder::new_current_thread()
323 
            .enable_all()
324 
            .build()
325 
            .map_err(|e| Error::Backend(format!("starting the HTTP runtime: {e}")))?;
326 
        let (inner, prefix, hint): (Arc<dyn ObjectStore>, String, &'static str) =
327 
            match parsed.scheme.as_str() {
328 
                "s3" => (
329 
                    Arc::new(aws::open(&parsed, aws_keys)?),
330 
                    parsed.path.clone(),
331 
                    aws::HINT,
332 
                ),
333 
                "gs" => (
334 
                    Arc::new(gcp::open(&parsed, gcp_key)?),
335 
                    parsed.path.clone(),
336 
                    gcp::HINT,
337 
                ),
338 
                "az" => {
339 
                    let (store, prefix) = azure::open(&parsed, principal)?;
340 
                    (Arc::new(store), prefix, azure::HINT)
341 
                }
342 
                other => {
343 
                    return Err(Error::Backend(format!(
344 
                        "{url}: {other}:// is not a store oxy knows (s3://, gs://, az://, dir://)"
345 
                    )))
346 
                }
347 
            };
348 
        Ok(CloudStore {
349 
            inner,
350 
            prefix,
351 
            name: parsed.plain(),
352 
            hint,
353 
            runtime: Some(runtime),
354 
            turn: Mutex::new(()),
355 
        })
356 
    }
357
358 
    /// Run a request to completion. Called from a thread that is itself
359 
    /// driving async tasks (a server's handler), where blocking on another
360 
    /// runtime is not allowed, the request runs on a thread of its own.
361 
    fn block<F>(&self, f: F) -> F::Output
362 
    where
363 
        F: Future + Send,
364 
        F::Output: Send,
365 
    {
366 
        let _turn = self.turn.lock().unwrap_or_else(|p| p.into_inner());
367 
        let runtime = self
368 
            .runtime
369 
            .as_ref()
370 
            .expect("the runtime lives as long as the store");
371 
        if tokio::runtime::Handle::try_current().is_ok() {
372 
            return std::thread::scope(|s| {
373 
                s.spawn(|| runtime.block_on(f))
374 
                    .join()
375 
                    .unwrap_or_else(|p| std::panic::resume_unwind(p))
376 
            });
377 
        }
378 
        runtime.block_on(f)
379 
    }
380
381 
    fn path(&self, key: &str) -> Result<ObjectPath> {
382 
        check_key(key)?;
383 
        let full = if self.prefix.is_empty() {
384 
            key.to_string()
385 
        } else {
386 
            format!("{}/{key}", self.prefix)
387 
        };
388 
        ObjectPath::parse(&full).map_err(|e| Error::Backend(format!("{full}: {e}")))
389 
    }
390
391 
    /// Our error for the provider's, with what to do next where it helps.
392 
    fn map(&self, e: object_store::Error, key: &str) -> Error {
393 
        use object_store::Error as E;
394 
        match e {
395 
            E::NotFound { .. } => Error::NotFound(key.to_string()),
396 
            E::AlreadyExists { .. } => Error::Exists(key.to_string()),
397 
            E::Precondition { .. } | E::NotModified { .. } => Error::Conflict(key.to_string()),
398 
            other => {
399 
                let text = other.to_string();
400 
                let lower = text.to_ascii_lowercase();
401 
                let advice = if lower.contains("403")
402 
                    || lower.contains("accessdenied")
403 
                    || lower.contains("forbidden")
404 
                    || lower.contains("authorizationpermissionmismatch")
405 
                {
406 
                    format!(
407 
                        ": access denied; the credentials in use may not do this on {}: {}",
408 
                        self.name, self.hint
409 
                    )
410 
                } else if lower.contains("credential") || lower.contains("no valid credentials") {
411 
                    format!(": no usable credentials: {}", self.hint)
412 
                } else if lower.contains("nosuchbucket")
413 
                    || lower.contains("bucketnotfound")
414 
                    || lower.contains("containernotfound")
415 
                {
416 
                    format!(
417 
                        ": {} does not exist, or the credentials cannot see it",
418 
                        self.name
419 
                    )
420 
                } else if lower.contains("dns")
421 
                    || lower.contains("connect")
422 
                    || lower.contains("timed out")
423 
                {
424 
                    ": cannot reach the service; check the network, the endpoint and the region"
425 
                        .to_string()
426 
                } else {
427 
                    String::new()
428 
                };
429 
                Error::Backend(format!("{}: {text}{advice}", self.name))
430 
            }
431 
        }
432 
    }
433
434 
    fn version_of(e_tag: Option<String>, version: Option<String>) -> Version {
435 
        Version(format!(
436 
            "{}\t{}",
437 
            e_tag.unwrap_or_default(),
438 
            version.unwrap_or_default()
439 
        ))
440 
    }
441
442 
    fn update_version(v: &Version) -> UpdateVersion {
443 
        let (e_tag, version) = v.0.split_once('\t').unwrap_or((v.0.as_str(), ""));
444 
        UpdateVersion {
445 
            e_tag: (!e_tag.is_empty()).then(|| e_tag.to_string()),
446 
            version: (!version.is_empty()).then(|| version.to_string()),
447 
        }
448 
    }
449 
}
450
451 
impl Store for CloudStore {
452 
    fn describe(&self) -> String {
453 
        self.name.clone()
454 
    }
455
456 
    fn has(&self, key: &str) -> Result<bool> {
457 
        let path = self.path(key)?;
458 
        match self.block(self.inner.head(&path)) {
459 
            Ok(_) => Ok(true),
460 
            Err(object_store::Error::NotFound { .. }) => Ok(false),
461 
            Err(e) => Err(self.map(e, key)),
462 
        }
463 
    }
464
465 
    fn get(&self, key: &str) -> Result<Vec<u8>> {
466 
        let path = self.path(key)?;
467 
        self.block(async {
468 
            let r = self.inner.get(&path).await?;
469 
            r.bytes().await
470 
        })
471 
        .map(|b| b.to_vec())
472 
        .map_err(|e| self.map(e, key))
473 
    }
474
475 
    fn get_range(&self, key: &str, range: Range<u64>) -> Result<Vec<u8>> {
476 
        let path = self.path(key)?;
477 
        if range.start > range.end {
478 
            return Err(Error::Backend(format!(
479 
                "{key}: bytes {}..{} is not a range",
480 
                range.start, range.end
481 
            )));
482 
        }
483 
        if range.is_empty() {
484 
            return Ok(Vec::new());
485 
        }
486 
        let want = (range.end - range.start) as usize;
487 
        let bytes = self
488 
            .block(self.inner.get_range(&path, range.clone()))
489 
            .map_err(|e| self.map(e, key))?;
490 
        if bytes.len() != want {
491 
            return Err(Error::Backend(format!(
492 
                "{key}: bytes {}..{} asked, {} returned: the object is shorter",
493 
                range.start,
494 
                range.end,
495 
                bytes.len()
496 
            )));
497 
        }
498 
        Ok(bytes.to_vec())
499 
    }
500
501 
    fn put_new(&self, key: &str, bytes: &[u8]) -> Result<()> {
502 
        let path = self.path(key)?;
503 
        let opts = PutOptions {
504 
            mode: PutMode::Create,
505 
            ..Default::default()
506 
        };
507 
        self.block(
508 
            self.inner
509 
                .put_opts(&path, PutPayload::from(bytes.to_vec()), opts),
510 
        )
511 
        .map(|_| ())
512 
        .map_err(|e| self.map(e, key))
513 
    }
514
515 
    fn list(&self, prefix: &str) -> Result<Vec<Entry>> {
516 
        // Providers list by whole path segments; the asked prefix may end
517 
        // mid-segment, so list its directory and filter here.
518 
        let dir = prefix.rsplit_once('/').map(|(d, _)| d).unwrap_or("");
519 
        let full_dir = match (self.prefix.as_str(), dir) {
520 
            ("", "") => None,
521 
            ("", d) => Some(d.to_string()),
522 
            (p, "") => Some(p.to_string()),
523 
            (p, d) => Some(format!("{p}/{d}")),
524 
        };
525 
        let full_dir = full_dir
526 
            .map(|d| ObjectPath::parse(&d).map_err(|e| Error::Backend(format!("{d}: {e}"))))
527 
            .transpose()?;
528 
        let metas: Vec<object_store::ObjectMeta> = self
529 
            .block(self.inner.list(full_dir.as_ref()).try_collect())
530 
            .map_err(|e| self.map(e, prefix))?;
531 
        let strip = if self.prefix.is_empty() {
532 
            String::new()
533 
        } else {
534 
            format!("{}/", self.prefix)
535 
        };
536 
        let mut out: Vec<Entry> = metas
537 
            .into_iter()
538 
            .filter_map(|m| {
539 
                let loc = m.location.to_string();
540 
                let key = loc.strip_prefix(&strip)?;
541 
                key.starts_with(prefix).then(|| Entry {
542 
                    key: key.to_string(),
543 
                    size: m.size,
544 
                    modified_ms: u64::try_from(m.last_modified.timestamp_millis()).ok(),
545 
                })
546 
            })
547 
            .collect();
548 
        out.sort_by(|a, b| a.key.cmp(&b.key));
549 
        Ok(out)
550 
    }
551
552 
    fn delete(&self, key: &str) -> Result<()> {
553 
        let path = self.path(key)?;
554 
        match self.block(self.inner.delete(&path)) {
555 
            Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()),
556 
            Err(e) => Err(self.map(e, key)),
557 
        }
558 
    }
559
560 
    fn read_ref(&self, key: &str) -> Result<Option<(Vec<u8>, Version)>> {
561 
        let path = self.path(key)?;
562 
        let got = self.block(async {
563 
            let r = self.inner.get_opts(&path, GetOptions::default()).await?;
564 
            let meta = r.meta.clone();
565 
            let bytes = r.bytes().await?;
566 
            Ok::<_, object_store::Error>((bytes, meta))
567 
        });
568 
        match got {
569 
            Ok((bytes, meta)) => Ok(Some((
570 
                bytes.to_vec(),
571 
                Self::version_of(meta.e_tag, meta.version),
572 
            ))),
573 
            Err(object_store::Error::NotFound { .. }) => Ok(None),
574 
            Err(e) => Err(self.map(e, key)),
575 
        }
576 
    }
577
578 
    fn write_ref(&self, key: &str, bytes: &[u8], expected: Option<&Version>) -> Result<Version> {
579 
        let path = self.path(key)?;
580 
        let mode = match expected {
581 
            None => PutMode::Create,
582 
            Some(v) => PutMode::Update(Self::update_version(v)),
583 
        };
584 
        let opts = PutOptions {
585 
            mode,
586 
            ..Default::default()
587 
        };
588 
        let put = self
589 
            .block(
590 
                self.inner
591 
                    .put_opts(&path, PutPayload::from(bytes.to_vec()), opts),
592 
            )
593 
            .map_err(|e| match self.map(e, key) {
594 
                // A record that exists when none was expected is a lost
595 
                // race too, not a misuse of put_new.
596 
                Error::Exists(k) => Error::Conflict(k),
597 
                other => other,
598 
            })?;
599 
        Ok(Self::version_of(put.e_tag, put.version))
600 
    }
601 
}
602
603 
#[cfg(test)]
604 
mod tests {
605 
    #[tokio::test]
606 
    async fn a_store_can_be_used_from_inside_a_runtime() {
607 
        // As a server's handler would: blocking on the store's own runtime
608 
        // from here must neither panic nor deadlock, and nor must dropping it.
609 
        let store = CloudStore::open(
610 
            "s3://oxy-no-such-bucket/p?region=eu-west-2&endpoint=http://127.0.0.1:9",
611 
        )
612 
        .unwrap();
613 
        assert_eq!(store.block(async { 7 }), 7);
614 
        drop(store);
615 
    }
616
617 
    #[test]
618 
    fn credentials_round_trip_and_never_show_their_secret() {
619 
        let aws = Credentials::Aws {
620 
            access_key_id: "AKIDEXAMPLE".into(),
621 
            secret_access_key: "wJalrXUtnFEMI".into(),
622 
            session_token: None,
623 
        };
624 
        assert_eq!(Credentials::from_json(&aws.to_json()).unwrap(), aws);
625 
        assert_eq!(format!("{aws:?}"), "Aws(AKIDEXAMPLE, secret hidden)");
626 
        let az = Credentials::Azure {
627 
            tenant_id: "t".into(),
628 
            client_id: "c".into(),
629 
            client_secret: "s3cr3t".into(),
630 
        };
631 
        assert_eq!(Credentials::from_json(&az.to_json()).unwrap(), az);
632 
        assert!(!format!("{az:?}").contains("s3cr3t"));
633 
        let key = r#"{"type":"service_account","client_email":"x@y","private_key":"k"}"#;
634 
        let gcp = Credentials::Gcp {
635 
            service_account_key: key.into(),
636 
        };
637 
        assert_eq!(Credentials::from_json(&gcp.to_json()).unwrap(), gcp);
638 
        assert!(!format!("{gcp:?}").contains("private_key"));
639 
        for (bad, why) in [
640 
            (r#"{"kind":"aws","access_key_id":"a"}"#, "secret_access_key"),
641 
            (
642 
                r#"{"kind":"gcp","service_account_key":"{\"type\":\"authorized_user\"}"}"#,
643 
                "not a service account key",
644 
            ),
645 
            (r#"{"kind":"ftp"}"#, "aws, gcp or azure"),
646 
            ("nope", "not JSON"),
647 
        ] {
648 
            let e = Credentials::from_json(bad).unwrap_err().to_string();
649 
            assert!(e.contains(why), "{bad}: {e}");
650 
        }
651 
        let e = CloudStore::open_with("gs://bucket/p", Some(&aws))
652 
            .err()
653 
            .unwrap()
654 
            .to_string();
655 
        assert!(e.contains("are for s3:// stores"), "{e}");
656 
    }
657
658 
    use super::*;
659
660 
    #[test]
661 
    fn urls_parse_with_their_parameters() {
662 
        let u =
663 
            StoreUrl::parse("s3://my-bucket/repos/oxygen?profile=work&region=eu-west-2").unwrap();
664 
        assert_eq!(u.scheme, "s3");
665 
        assert_eq!(u.host, "my-bucket");
666 
        assert_eq!(u.path, "repos/oxygen");
667 
        assert_eq!(u.param("profile"), Some("work"));
668 
        assert_eq!(u.param("region"), Some("eu-west-2"));
669 
        assert_eq!(u.plain(), "s3://my-bucket/repos/oxygen");
670 
        let u = StoreUrl::parse("gs://bucket").unwrap();
671 
        assert_eq!((u.host.as_str(), u.path.as_str()), ("bucket", ""));
672 
        assert_eq!(u.plain(), "gs://bucket");
673 
        let u = StoreUrl::parse("az://acct/container/a/b/?endpoint=http%3A%2F%2F127.0.0.1%3A10000")
674 
            .unwrap();
675 
        assert_eq!(u.path, "container/a/b");
676 
        assert_eq!(u.param("endpoint"), Some("http://127.0.0.1:10000"));
677 
        assert!(StoreUrl::parse("s3://").is_err());
678 
        assert!(StoreUrl::parse("nonsense").is_err());
679 
    }
680
681 
    #[test]
682 
    fn versions_carry_both_an_etag_and_a_generation() {
683 
        let v = CloudStore::version_of(Some("\"abc\"".into()), Some("17".into()));
684 
        let u = CloudStore::update_version(&v);
685 
        assert_eq!(u.e_tag.as_deref(), Some("\"abc\""));
686 
        assert_eq!(u.version.as_deref(), Some("17"));
687 
        let u = CloudStore::update_version(&CloudStore::version_of(None, Some("3".into())));
688 
        assert_eq!((u.e_tag, u.version.as_deref()), (None, Some("3")));
689 
    }
690 
}