OXYGENHUB
# c71a5777509a oxygen / crates / oxy-strata / tests / cloud.rs
5.6 KiB · Rust · last changed Strata: S3, Google Cloud Storage and Azure Blob Storage stores 1 month ago source blame history raw
1
//! The cloud stores against real services or emulators, when the
2
//! environment names them; otherwise each test says so and passes.
3
//!
4
//! - `OXY_TEST_S3=s3://BUCKET/PREFIX[?params]`, with `OXY_TEST_S3_PROFILE`
5
//!   added as `?profile=`;
6
//! - `OXY_TEST_GCS=gs://BUCKET/PREFIX[?params]`, with
7
//!   `OXY_TEST_GCS_CREDENTIALS` added as `?credentials=`;
8
//! - `OXY_TEST_AZURE=az://ACCOUNT/CONTAINER/PREFIX[?params]`, with the
9
//!   `AZURE_*` variables read from the file `OXY_TEST_AZURE_CREDENTIALS`.
10
//!
11
//! Every test works under its own prefix below the one given, and deletes
12
//! what it wrote.
13
14
use oxy_core::repo::Repo;
15
use oxy_core::strata::{conformance, Store};
16
use oxy_strata::CloudStore;
17
use std::path::Path;
18
19
fn configured(scheme: &str) -> Option<String> {
20
    let (var, extra, key) = match scheme {
21
        "s3" => ("OXY_TEST_S3", "OXY_TEST_S3_PROFILE", "profile"),
22
        "gs" => ("OXY_TEST_GCS", "OXY_TEST_GCS_CREDENTIALS", "credentials"),
23
        "az" => ("OXY_TEST_AZURE", "OXY_TEST_AZURE_CREDENTIALS", ""),
24
        _ => unreachable!(),
25
    };
26
    let url = std::env::var(var).ok().filter(|u| !u.is_empty())?;
27
    let extra_value = std::env::var(extra).ok().filter(|v| !v.is_empty());
28
    if scheme == "az" {
29
        if let Some(file) = &extra_value {
30
            for line in std::fs::read_to_string(file).unwrap_or_default().lines() {
31
                if let Some((k, v)) = line.split_once('=') {
32
                    std::env::set_var(k.trim(), v.trim());
33
                }
34
            }
35
        }
36
        return Some(url);
37
    }
38
    Some(match extra_value {
39
        Some(v) if !url.contains(&format!("{key}=")) => {
40
            let sep = if url.contains('?') { '&' } else { '?' };
41
            format!("{url}{sep}{key}={v}")
42
        }
43
        _ => url,
44
    })
45
}
46
47
fn each_store() -> Vec<(String, CloudStore)> {
48
    oxy_strata::register();
49
    ["s3", "gs", "az"]
50
        .into_iter()
51
        .filter_map(|s| {
52
            let Some(url) = configured(s) else {
53
                eprintln!("{s}: not configured, skipped");
54
                return None;
55
            };
56
            let store = CloudStore::open(&url).unwrap_or_else(|e| panic!("{url}: {e}"));
57
            Some((url, store))
58
        })
59
        .collect()
60
}
61
62
fn remove_all(store: &dyn Store, prefix: &str) {
63
    for e in store.list(prefix).unwrap() {
64
        store.delete(&e.key).unwrap();
65
    }
66
}
67
68
/// What earlier runs that failed may have left under the test prefix:
69
/// other processes' `probe-*` and `repo-*` keys.
70
fn remove_stale(store: &dyn Store) {
71
    let mine = format!("-{}", std::process::id());
72
    for e in store.list("").unwrap() {
73
        let first = e.key.split('/').next().unwrap_or("");
74
        if (first.starts_with("probe-") || first.starts_with("repo-")) && !first.ends_with(&mine) {
75
            store.delete(&e.key).unwrap();
76
        }
77
    }
78
}
79
80
#[test]
81
fn every_configured_store_conforms() {
82
    for (url, store) in each_store() {
83
        remove_stale(&store);
84
        let prefix = format!("probe-{}", std::process::id());
85
        let passed = conformance(&store, &prefix).unwrap_or_else(|e| {
86
            remove_all(&store, &prefix);
87
            panic!("{url}: {e}")
88
        });
89
        eprintln!("{url}: {} checks passed", passed.len());
90
        assert_eq!(passed.len(), 8);
91
    }
92
}
93
94
/// A small repository pushed to the store, cloned back, and pulled, by
95
/// URL: what a user does. Then every key the test wrote is removed.
96
#[test]
97
fn a_repository_round_trips_through_every_configured_store() {
98
    for (url, base_store) in each_store() {
99
        remove_stale(&base_store);
100
        let dir = tempfile::tempdir().unwrap();
101
        let root = dir.path();
102
        std::fs::create_dir_all(root.join("home")).unwrap();
103
        std::env::set_var("OXY_HOME", root.join("home"));
104
        let (base, query) = match url.split_once('?') {
105
            Some((b, q)) => (b.to_string(), format!("?{q}")),
106
            None => (url.clone(), String::new()),
107
        };
108
        let remote_url = format!("{base}/repo-{}{query}", std::process::id());
109
        let plain = format!("{base}/repo-{}", std::process::id());
110
        let store = CloudStore::open(&remote_url).unwrap();
111
112
        let mut a = Repo::init(&root.join("a")).unwrap();
113
        a.config.set("remote.cloud.url", &remote_url).unwrap();
114
        a.save_config().unwrap();
115
        write(root, "a/f.txt", b"hello from a\n");
116
        let h1 = a.commit("first", &[]).unwrap();
117
        let r = a.push("cloud", false).unwrap();
118
        assert!(r.iter().all(|x| x.ok), "{plain}: {r:?}");
119
        assert!(store.has("oxygen.toml").unwrap());
120
        assert_eq!(store.list("bundles/").unwrap().len(), 1, "{plain}");
121
122
        let b = Repo::clone(&remote_url, &root.join("b"))
123
            .unwrap_or_else(|e| panic!("{plain}: clone: {e}"));
124
        assert_eq!(
125
            std::fs::read(root.join("b/f.txt")).unwrap(),
126
            b"hello from a\n"
127
        );
128
        assert_eq!(b.bookmark_heads().unwrap(), vec![h1]);
129
        write(root, "b/f.txt", b"hello from a\nand b\n");
130
        b.commit("second", &[]).unwrap();
131
        assert!(b.push("origin", false).unwrap()[0].ok);
132
        let pr = a.pull("cloud").unwrap();
133
        assert!(pr.changed, "{plain}");
134
        assert_eq!(
135
            std::fs::read(root.join("a/f.txt")).unwrap(),
136
            b"hello from a\nand b\n"
137
        );
138
        let total: u64 = store.list("").unwrap().iter().map(|e| e.size).sum();
139
        eprintln!("{plain}: round trip ok, {total} bytes in the store before cleanup");
140
        remove_all(&store, "");
141
        assert!(store.list("").unwrap().is_empty(), "{plain}: cleanup");
142
    }
143
}
144
145
fn write(root: &Path, rel: &str, content: &[u8]) {
146
    let p = root.join(rel);
147
    std::fs::create_dir_all(p.parent().unwrap()).unwrap();
148
    std::fs::write(p, content).unwrap();
149
}