OXYGENHUB
◆ v0.5.2 oxygenhub / crates / hub / tests / storage.rs
12.2 KiB · Rust · last changed The store as its provider's icon; a tidier Projects page 26 days ago source blame history raw
1f78655a Andy McCall ↶ 1
//! Repositories kept in their owner's bucket (oxygenhub#96): the New
2
//! repository form refuses what the hub must not accept and creates
3
//! nothing then; with a real S3 test bucket configured (`OXY_TEST_S3` and
4
//! `OXY_TEST_S3_PROFILE`, keys read from that profile and handed over), a
5
//! repository is made in the bucket, takes a push, shows where it lives,
6
//! passes Settings → Storage → Check, and the test removes what it wrote.
7
8
use axum::body::Body;
9
use axum::http::{header, Request, StatusCode};
10
use http_body_util::BodyExt;
11
use oxy_core::repo::Repo;
12
use oxygenhub::config::Config;
13
use oxygenhub::repos::RepoName;
14
use oxygenhub::{db, Shared};
15
use tower::ServiceExt;
16
17
fn hermetic() {
18
    static ONCE: std::sync::Once = std::sync::Once::new();
19
    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();
20
    ONCE.call_once(|| {
21
        let d = tempfile::tempdir().unwrap();
22
        std::env::set_var("OXY_HOME", d.path());
23
        DIR.set(d).ok();
24
    });
25
}
26
27
struct Hub {
28
    app: Shared,
29
    base: String,
30
    _dir: tempfile::TempDir,
31
}
32
33
async fn hub() -> Hub {
34
    hermetic();
35
    let dir = tempfile::tempdir().unwrap();
36
    let mut config = Config::load(dir.path()).unwrap();
37
    config.auth.push_tokens = vec!["site".into()];
38
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
39
    let port = listener.local_addr().unwrap().port();
40
    config.server.base_url = format!("http://127.0.0.1:{port}");
41
    let app = oxygenhub::build(config).await.unwrap();
42
    let router = oxygenhub::router(app.clone());
43
    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
44
    Hub { app, base: format!("http://127.0.0.1:{port}"), _dir: dir }
45
}
46
47
async fn send(app: &Shared, method: &str, path: &str, cookie: Option<&str>, form: Option<&str>) -> (StatusCode, String, Option<String>) {
48
    let mut req = Request::builder().method(method).uri(path);
49
    if let Some(c) = cookie {
50
        req = req.header(header::COOKIE, format!("oxyhub={c}"));
51
    }
52
    let body = match form {
53
        Some(f) => {
54
            req = req.header(header::CONTENT_TYPE, "application/x-www-form-urlencoded");
55
            Body::from(f.to_string())
56
        }
57
        None => Body::empty(),
58
    };
59
    let res = oxygenhub::router(app.clone()).oneshot(req.body(body).unwrap()).await.unwrap();
60
    let status = res.status();
61
    let cookie = res
62
        .headers()
63
        .get_all(header::SET_COOKIE)
64
        .iter()
65
        .filter_map(|v| v.to_str().ok())
66
        .filter_map(|c| c.split(';').next())
67
        .filter_map(|kv| kv.strip_prefix("oxyhub="))
68
        .map(String::from)
69
        .next();
70
    let bytes = res.into_body().collect().await.unwrap().to_bytes();
71
    (status, String::from_utf8_lossy(&bytes).into_owned(), cookie)
72
}
73
74
fn csrf_of(page: &str) -> String {
75
    let i = page.find("name=\"_csrf\" value=\"").expect("csrf field") + "name=\"_csrf\" value=\"".len();
76
    page[i..].split('"').next().unwrap().to_string()
77
}
78
79
fn enc(s: &str) -> String {
80
    s.bytes()
81
        .map(|b| if b.is_ascii_alphanumeric() || b"-_.~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") })
82
        .collect()
83
}
84
85
async fn andy(app: &Shared) -> String {
86
    let (st, body, cookie) =
87
        send(app, "POST", "/register", None, Some("username=andy&email=andy%40example.com&password=correct-horse-battery&invite=")).await;
88
    assert_eq!(st, StatusCode::SEE_OTHER, "{body}");
89
    cookie.expect("session")
90
}
91
92
async fn new_repo(app: &Shared, me: &str, fields: &str) -> (StatusCode, String) {
93
    let csrf = csrf_of(&send(app, "GET", "/new", Some(me), None).await.1);
94
    let (st, body, _) = send(app, "POST", "/new", Some(me), Some(&format!("_csrf={csrf}&owner=andy&{fields}"))).await;
95
    (st, body)
96
}
97
98
#[tokio::test(flavor = "multi_thread")]
99
async fn a_bucket_the_hub_must_not_use_is_refused_and_nothing_is_made() {
100
    let h = hub().await;
101
    let me = andy(&h.app).await;
102
    for (url, why) in [
103
        ("s3://b/p?profile=default", "?profile= is not allowed"),
104
        ("s3://b/p?endpoint=http%3A%2F%2F169.254.169.254", "?endpoint= is not allowed"),
105
        ("gs://b/p?credentials=%2Fetc%2Fkey.json", "?credentials= is not allowed"),
106
        ("dir:///var/lib/oxygenhub", "a store is s3://"),
107
        ("az://account", "names the account and the container"),
108
    ] {
109
        let (st, body) =
110
            new_repo(&h.app, &me, &format!("name=cloudy&storage=bucket&store_url={}&access_key_id=a&secret_access_key=b", enc(url))).await;
111
        assert_eq!(st, StatusCode::OK, "{url}");
112
        assert!(body.contains(why), "{url}: {body}");
113
    }
114
    let (_, body) = new_repo(&h.app, &me, &format!("name=cloudy&storage=bucket&store_url={}&access_key_id=a", enc("s3://b/p"))).await;
115
    assert!(body.contains("Credentials: credentials need secret_access_key"), "{body}");
116
    let name = RepoName::new("andy", "cloudy").unwrap();
117
    assert!(!h.app.repos.exists(&name));
118
    assert!(db::repos::get(&h.app.db, "andy", "cloudy").await.unwrap().is_none());
119
    // The hub's own disk stays the default.
120
    let (st, body) = new_repo(&h.app, &me, "name=local").await;
121
    assert_eq!(st, StatusCode::SEE_OTHER, "{body}");
122
    let page = send(&h.app, "GET", "/andy/local", Some(&me), None).await.1;
7b26e611 Andy McCall ↶ 123
    assert!(!page.contains("class=\"store store-"), "no store icon on a repository on this disk");
1f78655a Andy McCall ↶ 124
}
125
126
/// The S3 test bucket's URL with only its region kept, and the profile's keys.
127
fn s3_test() -> Option<(String, String, String)> {
128
    let url = std::env::var("OXY_TEST_S3").ok().filter(|u| !u.is_empty())?;
129
    let profile = std::env::var("OXY_TEST_S3_PROFILE").ok().filter(|p| !p.is_empty())?;
130
    let (base, query) = url.split_once('?').unwrap_or((&url, ""));
131
    let region: Vec<&str> = query.split('&').filter(|p| p.starts_with("region=")).collect();
132
    let home = std::path::PathBuf::from(std::env::var("HOME").ok()?);
133
    let text = std::fs::read_to_string(home.join(".aws/credentials")).ok()?;
134
    let (mut section, mut id, mut secret) = (String::new(), None, None);
135
    for line in text.lines().map(str::trim) {
136
        if let Some(n) = line.strip_prefix('[').and_then(|l| l.strip_suffix(']')) {
137
            section = n.to_string();
138
        } else if let Some((k, v)) = line.split_once('=') {
139
            if section == profile {
140
                match k.trim() {
141
                    "aws_access_key_id" => id = Some(v.trim().to_string()),
142
                    "aws_secret_access_key" => secret = Some(v.trim().to_string()),
143
                    _ => {}
144
                }
145
            }
146
        }
147
    }
148
    let base = format!("{base}/hub-{}", std::process::id());
149
    let url = if region.is_empty() { base } else { format!("{base}?{}", region.join("&")) };
150
    Some((url, id?, secret?))
151
}
152
153
#[tokio::test(flavor = "multi_thread")]
154
async fn a_repository_in_a_real_bucket_takes_a_push_and_passes_its_check() {
155
    let Some((url, id, secret)) = s3_test() else {
156
        eprintln!("skipped: OXY_TEST_S3 and OXY_TEST_S3_PROFILE are not set");
157
        return;
158
    };
159
    let h = hub().await;
160
    let me = andy(&h.app).await;
161
    let fields = format!(
162
        "name=cloudy&visibility=private&storage=bucket&store_url={}&access_key_id={}&secret_access_key={}",
163
        enc(&url),
164
        enc(&id),
165
        enc(&secret)
166
    );
167
    let (st, body) = new_repo(&h.app, &me, &fields).await;
168
    let credentials = oxy_strata::Credentials::Aws { access_key_id: id, secret_access_key: secret, session_token: None };
169
    let store = oxy_strata::CloudStore::open_with(&url, Some(&credentials)).unwrap();
170
    let cleanup = || {
171
        use oxy_core::strata::Store as _;
172
        for e in store.list("").unwrap() {
173
            store.delete(&e.key).unwrap();
174
        }
175
    };
176
    if st != StatusCode::SEE_OTHER {
177
        cleanup();
178
        panic!("{body}");
179
    }
180
    let result = std::panic::AssertUnwindSafe(async {
181
        // A push from a client lands in the bucket.
182
        let work = tempfile::tempdir().unwrap();
183
        let mut c = Repo::init(work.path()).unwrap();
184
        c.config.set("remote.origin.url", &format!("{}/andy/cloudy", h.base)).unwrap();
185
        c.config.set("remote.origin.token", "site").unwrap();
186
        c.save_config().unwrap();
187
        std::fs::write(work.path().join("a.txt"), "kept in a bucket\n").unwrap();
188
        let head = c.commit("First", &[]).unwrap();
189
        let pushed = tokio::task::spawn_blocking(move || c.push("origin", false).unwrap()).await.unwrap();
190
        assert!(pushed.iter().all(|r| r.ok), "{pushed:?}");
191
        let hosted = h.app.repos.open(&RepoName::new("andy", "cloudy").unwrap()).unwrap();
192
        assert_eq!(hosted.bookmark_heads().unwrap(), vec![head]);
193
        {
194
            use oxy_core::strata::Store as _;
195
            assert!(store.read_ref("ops/head").unwrap().is_some(), "the operation log is in the bucket");
196
        }
197
        // The pages say where it lives; Settings → Storage checks the bucket.
198
        let page = send(&h.app, "GET", "/andy/cloudy", Some(&me), None).await.1;
7b26e611 Andy McCall ↶ 199
        assert!(page.contains("class=\"store store-s3\"") && page.contains(url.split('?').next().unwrap()), "{page}");
1f78655a Andy McCall ↶ 200
        let settings = send(&h.app, "GET", "/andy/cloudy/settings", Some(&me), None).await.1;
201
        assert!(settings.contains("id=\"storage\""), "{settings}");
202
        let csrf = csrf_of(&settings);
203
        let (_, after, _) =
204
            send(&h.app, "POST", "/andy/cloudy/settings/storage", Some(&me), Some(&format!("_csrf={csrf}&action=check"))).await;
205
        assert!(after.contains("The bucket passed the check: 8 checks passed."), "{after}");
206
        // A second repository cannot start in the same place.
207
        let (st, body) = new_repo(&h.app, &me, &fields.replace("name=cloudy", "name=again")).await;
208
        assert_eq!(st, StatusCode::OK);
209
        assert!(body.contains("already holds"), "{body}");
210
    });
211
    let outcome = futures_catch(result).await;
212
    cleanup();
213
    if let Err(p) = outcome {
214
        std::panic::resume_unwind(p);
215
    }
216
}
217
218
async fn futures_catch<F: std::future::Future<Output = ()>>(f: std::panic::AssertUnwindSafe<F>) -> std::thread::Result<()> {
219
    use futures::FutureExt;
220
    f.catch_unwind().await
221
}
8ddebaa7 Andy McCall ↶ 222
223
#[tokio::test(flavor = "multi_thread")]
224
async fn an_organisation_can_require_its_own_buckets() {
225
    let h = hub().await;
226
    let me = andy(&h.app).await;
227
    let csrf = csrf_of(&send(&h.app, "GET", "/orgs/new", Some(&me), None).await.1);
228
    let (st, body, _) = send(&h.app, "POST", "/orgs/new", Some(&me), Some(&format!("_csrf={csrf}&name=acme&display_name=Acme"))).await;
229
    assert_eq!(st, StatusCode::SEE_OTHER, "{body}");
230
    let settings = send(&h.app, "GET", "/orgs/acme/settings", Some(&me), None).await.1;
231
    assert!(settings.contains("id=\"storage\""));
232
    let csrf = csrf_of(&settings);
233
    let set = |allowed: &str| {
234
        let form = format!("_csrf={csrf}&require_bucket=1&allowed={}", enc(allowed));
235
        let app = h.app.clone();
236
        let me = me.clone();
237
        async move { send(&app, "POST", "/orgs/acme/settings/storage", Some(&me), Some(&form)).await.1 }
238
    };
239
    let page = set("s3://acme-london/repos/?region=eu-west-2").await;
240
    assert!(page.contains("a prefix names a bucket and a path, with no settings"), "{page}");
241
    let page = set("s3://acme-london/repos/\ngs://acme-eu/").await;
242
    assert!(page.contains("Storage policy saved: every new repository in s3://acme-london/repos/, gs://acme-eu/."), "{page}");
243
244
    let new = |fields: String| {
245
        let app = h.app.clone();
246
        let me = me.clone();
247
        async move {
248
            let csrf = csrf_of(&send(&app, "GET", "/new", Some(&me), None).await.1);
249
            send(&app, "POST", "/new", Some(&me), Some(&format!("_csrf={csrf}&owner=acme&{fields}"))).await.1
250
        }
251
    };
252
    let page = new("name=ondisk".into()).await;
253
    assert!(page.contains("acme keeps every new repository in a bucket"), "{page}");
254
    let page = new(format!("name=elsewhere&storage=bucket&store_url={}&access_key_id=a&secret_access_key=b", enc("s3://other/x"))).await;
255
    assert!(page.contains("acme allows only these stores: s3://acme-london/repos/, gs://acme-eu/"), "{page}");
256
    assert!(db::repos::get(&h.app.db, "acme", "ondisk").await.unwrap().is_none());
257
    assert!(db::repos::get(&h.app.db, "acme", "elsewhere").await.unwrap().is_none());
258
    // A user's own repositories are not the organisation's business.
259
    let (st, body) = new_repo(&h.app, &me, "name=mine").await;
260
    assert_eq!(st, StatusCode::SEE_OTHER, "{body}");
261
}