OXYGENHUB
# e30c081a5064 oxygen / crates / oxy-server / tests / http.rs
20.5 KiB · Rust · last changed oxy serve takes a push of more than 2 MB (closes #138) 12 days ago source blame history raw
1
//! End-to-end: a real server on a loopback port, clients over HTTP.
2
3
use oxy_core::repo::{Repo, SignatureStatus};
4
use std::fs;
5
use std::net::TcpListener;
6
use std::path::Path;
7
8
fn write(root: &Path, rel: &str, content: &[u8]) {
9
    let p = root.join(rel);
10
    fs::create_dir_all(p.parent().unwrap()).unwrap();
11
    fs::write(p, content).unwrap();
12
}
13
14
/// Point `$OXY_HOME` at a throwaway directory once per test process so no
15
/// test reads or writes the developer's real user configuration or keys.
16
fn hermetic() {
17
    static ONCE: std::sync::Once = std::sync::Once::new();
18
    static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();
19
    ONCE.call_once(|| {
20
        let d = tempfile::tempdir().unwrap();
21
        std::env::set_var("OXY_HOME", d.path());
22
        DIR.set(d).ok();
23
    });
24
}
25
26
fn start(root: &Path, tokens: Vec<String>) -> String {
27
    start_with(root, tokens, None)
28
}
29
30
fn start_with(root: &Path, tokens: Vec<String>, tls: Option<oxy_server::TlsFiles>) -> String {
31
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
32
    let port = listener.local_addr().unwrap().port();
33
    let scheme = if tls.is_some() { "https" } else { "http" };
34
    let cfg = oxy_server::ServerConfig {
35
        root: root.to_path_buf(),
36
        tokens,
37
        tls,
38
    };
39
    std::thread::spawn(move || oxy_server::run_blocking(listener, cfg).unwrap());
40
    // Give the acceptor a moment; the first request otherwise races the bind.
41
    std::thread::sleep(std::time::Duration::from_millis(50));
42
    format!("{scheme}://127.0.0.1:{port}")
43
}
44
45
/// A self-signed certificate for 127.0.0.1 written to `dir`: (cert, key).
46
fn self_signed(dir: &Path) -> (std::path::PathBuf, std::path::PathBuf) {
47
    fs::create_dir_all(dir).unwrap();
48
    let ck =
49
        rcgen::generate_simple_self_signed(vec!["127.0.0.1".into(), "localhost".into()]).unwrap();
50
    let cert = dir.join("cert.pem");
51
    let key = dir.join("key.pem");
52
    fs::write(&cert, ck.cert.pem()).unwrap();
53
    fs::write(&key, ck.signing_key.serialize_pem()).unwrap();
54
    (cert, key)
55
}
56
57
#[test]
58
fn clone_push_pull_over_http() {
59
    hermetic();
60
    let dir = tempfile::tempdir().unwrap();
61
    let root = dir.path();
62
    let served = root.join("served");
63
    Repo::init(&served.join("origin")).unwrap();
64
    let base = start(&served, vec![]);
65
    let url = format!("{base}/origin");
66
67
    let mut a = Repo::clone(&url, &root.join("a"))
68
        .unwrap()
69
        .with_home(root.join("home"));
70
    let (key, _) = a
71
        .create_identity(
72
            "Ada",
73
            "[email protected]",
74
            oxy_core::repo::IdentityScope::Repo,
75
        )
76
        .unwrap();
77
    write(&root.join("a"), "f.txt", b"hello\n");
78
    let h1 = a.commit("hello", &[]).unwrap();
79
    let r = a.push("origin", false).unwrap();
80
    assert!(r[0].ok, "{}", r[0].reason);
81
82
    let origin = Repo::open(&served.join("origin")).unwrap();
83
    assert_eq!(origin.bookmark_heads().unwrap(), vec![h1]);
84
    assert!(
85
        matches!(origin.authenticate_change(&h1).unwrap(), SignatureStatus::Valid { key: k, sealed: true, .. } if k == key)
86
    );
87
88
    let b = Repo::clone(&url, &root.join("b")).unwrap();
89
    assert_eq!(fs::read(root.join("b/f.txt")).unwrap(), b"hello\n");
90
    write(&root.join("b"), "g.txt", b"g\n");
91
    let h2 = b.commit("g", &[]).unwrap();
92
    assert!(b.push("origin", false).unwrap()[0].ok);
93
94
    let pr = a.pull("origin").unwrap();
95
    assert!(pr.changed && pr.checked_out);
96
    assert_eq!(fs::read(root.join("a/g.txt")).unwrap(), b"g\n");
97
    let mut heads = a.bookmark_heads().unwrap();
98
    heads.sort();
99
    let mut want = vec![h1, h2];
100
    want.sort();
101
    assert_eq!(heads, want);
102
103
    // Unknown repositories and names the server refuses are surfaced as errors.
104
    let err = Repo::clone(&format!("{base}/nope"), &root.join("c"))
105
        .err()
106
        .unwrap()
107
        .to_string();
108
    assert!(err.contains("404"), "{err}");
109
    fs::create_dir_all(served.join(".hidden/.oxy")).unwrap();
110
    assert!(Repo::clone(&format!("{base}/.hidden"), &root.join("d"))
111
        .err()
112
        .unwrap()
113
        .to_string()
114
        .contains("404"));
115
}
116
117
/// A push of more than axum's default 2 MB body is taken: `oxy serve`
118
/// sets its own limit, as OxygenHub does.
119
#[test]
120
fn a_push_of_more_than_two_megabytes_is_taken() {
121
    hermetic();
122
    let dir = tempfile::tempdir().unwrap();
123
    let root = dir.path();
124
    let served = root.join("served");
125
    Repo::init(&served.join("origin")).unwrap();
126
    let url = format!("{}/origin", start(&served, vec![]));
127
    let a = Repo::clone(&url, &root.join("a")).unwrap();
128
    let mut x: u64 = 0x9e3779b97f4a7c15;
129
    let noise: Vec<u8> = (0..3 << 20)
130
        .map(|_| {
131
            x ^= x << 13;
132
            x ^= x >> 7;
133
            x ^= x << 17;
134
            (x >> 24) as u8
135
        })
136
        .collect();
137
    write(&root.join("a"), "big.bin", &noise);
138
    let h = a.commit("three megabytes", &[]).unwrap();
139
    let r = a.push("origin", false).unwrap();
140
    assert!(r[0].ok, "{}", r[0].reason);
141
    let b = Repo::clone(&url, &root.join("b")).unwrap();
142
    assert_eq!(b.bookmark_heads().unwrap(), vec![h]);
143
    assert_eq!(fs::read(root.join("b/big.bin")).unwrap(), noise);
144
}
145
146
#[test]
147
fn objects_endpoint_serves_lazy_blobs() {
148
    hermetic();
149
    let dir = tempfile::tempdir().unwrap();
150
    let root = dir.path();
151
    let served = root.join("served");
152
    let origin = Repo::init(&served.join("origin")).unwrap();
153
    let big: Vec<u8> = (0..300_000u32).map(|i| (i % 253) as u8).collect(); // has NUL bytes: opaque
154
    write(&served.join("origin"), "big.bin", &big);
155
    write(&served.join("origin"), "t.txt", b"t\n");
156
    origin.commit("big", &[]).unwrap();
157
    let base = start(&served, vec![]);
158
    let url = format!("{base}/origin");
159
    let a = Repo::clone_with(&url, &root.join("a"), Some(1000), &[]).unwrap();
160
    assert_eq!(
161
        fs::read(root.join("a/big.bin")).unwrap(),
162
        big,
163
        "fetched through the objects endpoint during checkout"
164
    );
165
    assert!(!a.snapshot("status").unwrap().changed);
166
}
167
168
#[test]
169
fn push_requires_a_token_when_configured() {
170
    hermetic();
171
    let dir = tempfile::tempdir().unwrap();
172
    let root = dir.path();
173
    let served = root.join("served");
174
    Repo::init(&served.join("origin")).unwrap();
175
    let base = start(&served, vec!["s3cret".into()]);
176
    let url = format!("{base}/origin");
177
178
    // Reads are open.
179
    let mut a = Repo::clone(&url, &root.join("a")).unwrap();
180
    write(&root.join("a"), "f.txt", b"x\n");
181
    a.commit("x", &[]).unwrap();
182
    let err = a.push("origin", false).err().unwrap().to_string();
183
    assert!(err.contains("401"), "{err}");
184
185
    a.config.set("remote.origin.token", "wrong").unwrap();
186
    a.save_config().unwrap();
187
    assert!(a
188
        .push("origin", false)
189
        .err()
190
        .unwrap()
191
        .to_string()
192
        .contains("401"));
193
194
    a.config.set("remote.origin.token", "s3cret").unwrap();
195
    a.save_config().unwrap();
196
    assert!(a.push("origin", false).unwrap()[0].ok);
197
    let origin = Repo::open(&served.join("origin")).unwrap();
198
    assert_eq!(origin.bookmark_heads().unwrap().len(), 1);
199
}
200
201
#[test]
202
fn https_with_a_self_signed_certificate() {
203
    hermetic();
204
    let dir = tempfile::tempdir().unwrap();
205
    let root = dir.path();
206
    let served = root.join("served");
207
    Repo::init(&served.join("origin")).unwrap();
208
    let (cert, key) = self_signed(root);
209
    let base = start_with(
210
        &served,
211
        vec![],
212
        Some(oxy_server::TlsFiles {
213
            cert: cert.clone(),
214
            key,
215
        }),
216
    );
217
    let url = format!("{base}/origin");
218
219
    // Without trusting the certificate the clone fails; with `OXY_CA` it works.
220
    let err = Repo::clone(&url, &root.join("untrusted"))
221
        .err()
222
        .unwrap()
223
        .to_string();
224
    assert!(
225
        err.contains("certificate")
226
            || err.contains("tls")
227
            || err.contains("TLS")
228
            || err.contains("handshake"),
229
        "{err}"
230
    );
231
    std::env::set_var("OXY_CA", &cert);
232
    let mut a = Repo::clone(&url, &root.join("a")).unwrap();
233
    std::env::remove_var("OXY_CA");
234
    // Persist the trust for this remote so later commands need no env var.
235
    a.config
236
        .set("remote.origin.ca", cert.to_str().unwrap())
237
        .unwrap();
238
    a.save_config().unwrap();
239
    write(&root.join("a"), "f.txt", b"tls\n");
240
    let h = a.commit("over tls", &[]).unwrap();
241
    assert!(a.push("origin", false).unwrap()[0].ok);
242
    let origin = Repo::open(&served.join("origin")).unwrap();
243
    assert_eq!(origin.bookmark_heads().unwrap(), vec![h]);
244
    // A wrong CA is refused.
245
    let (other_cert, _) = self_signed(&root.join("other"));
246
    a.config
247
        .set("remote.origin.ca", other_cert.to_str().unwrap())
248
        .unwrap();
249
    a.save_config().unwrap();
250
    assert!(a.fetch("origin").is_err());
251
}
252
253
/// Eight clones push independent changes at the same moment. The server
254
/// serialises pushes per repository, so every push either lands or is
255
/// refused because the bookmark moved, and a pull-then-push retry lands it.
256
#[test]
257
fn concurrent_pushes_are_serialised() {
258
    hermetic();
259
    let dir = tempfile::tempdir().unwrap();
260
    let root = dir.path();
261
    let served = root.join("served");
262
    Repo::init(&served.join("origin")).unwrap();
263
    let base = start(&served, vec![]);
264
    let url = format!("{base}/origin");
265
    const N: usize = 8;
266
    let mut clones = Vec::new();
267
    for i in 0..N {
268
        let path = root.join(format!("c{i}"));
269
        Repo::clone(&url, &path).unwrap();
270
        clones.push(path);
271
    }
272
    let barrier = std::sync::Arc::new(std::sync::Barrier::new(N));
273
    let mut handles = Vec::new();
274
    for (i, path) in clones.into_iter().enumerate() {
275
        let barrier = barrier.clone();
276
        handles.push(std::thread::spawn(move || {
277
            let repo = Repo::open(&path).unwrap();
278
            write(
279
                &path,
280
                &format!("file-{i}.txt"),
281
                format!("from clone {i}\n").as_bytes(),
282
            );
283
            let h = repo.commit(&format!("change {i}"), &[]).unwrap();
284
            barrier.wait();
285
            // A push is refused either up front (the remote moved since the
286
            // clone's last fetch) or by the server's compare-and-swap; both
287
            // are answered by pulling and pushing again.
288
            for attempt in 0..(N * 4) {
289
                match repo.push("origin", false) {
290
                    Ok(r) if r.iter().all(|x| x.ok) => return (h, attempt),
291
                    Ok(_) => {}
292
                    Err(e)
293
                        if e.to_string().contains("pull first")
294
                            || e.to_string().contains("moved") => {}
295
                    Err(e) => panic!("clone {i}: {e}"),
296
                }
297
                repo.pull("origin")
298
                    .unwrap_or_else(|e| panic!("clone {i} pull: {e}"));
299
            }
300
            panic!("clone {i} never managed to push");
301
        }));
302
    }
303
    let mut hashes = Vec::new();
304
    let mut total_rejected = 0;
305
    for h in handles {
306
        let (hash, attempts) = h.join().unwrap();
307
        hashes.push(hash);
308
        total_rejected += attempts;
309
    }
310
    let origin = Repo::open(&served.join("origin")).unwrap();
311
    let mut heads = origin.bookmark_heads().unwrap();
312
    heads.sort();
313
    hashes.sort();
314
    assert_eq!(heads, hashes, "every independent change is a head");
315
    let snap = origin.load_state(&heads).unwrap().materialize();
316
    for i in 0..N {
317
        assert!(snap.files.contains_key(&format!("file-{i}.txt")));
318
    }
319
    assert!(!snap.is_conflicted());
320
    eprintln!("{total_rejected} pushes were refused for a moved bookmark and retried");
321
    // The operation log is a single chain: each accepted push builds on the last.
322
    let ops = origin.op_log().unwrap();
323
    assert!(ops.len() > N);
324
}
325
326
#[test]
327
fn a_bare_repository_is_served_and_browsed_without_a_working_copy() {
328
    hermetic();
329
    let dir = tempfile::tempdir().unwrap();
330
    let root = dir.path();
331
    let served = root.join("served");
332
    let hosted = Repo::init_bare(&served.join("lib")).unwrap();
333
    assert!(hosted.is_bare());
334
    assert!(
335
        hosted.working_copy().is_err(),
336
        "a bare repository has no working copy"
337
    );
338
    assert!(!served.join("lib").join(".oxy").join("workspace").exists());
339
    let base = start(&served, vec![]);
340
    let url = format!("{base}/lib");
341
342
    // A client clones the empty repository, commits twice, and pushes.
343
    let work = root.join("work");
344
    let client = Repo::clone(&url, &work).unwrap();
345
    write(&work, "src/lib.rs", b"pub fn one() {}\n");
346
    client.snapshot("test").unwrap();
347
    let c1 = client.commit("one", &[]).unwrap();
348
    write(&work, "src/lib.rs", b"pub fn one() {}\npub fn two() {}\n");
349
    client.snapshot("test").unwrap();
350
    let c2 = client.commit("two", &[]).unwrap();
351
    let results = client.push("origin", false).unwrap();
352
    assert!(results.iter().all(|r| r.ok), "{results:?}");
353
354
    // The host reads history, files and blame from the bookmark's state alone.
355
    let hosted = Repo::open(&served.join("lib")).unwrap();
356
    assert!(hosted.is_bare());
357
    let heads = hosted.bookmark_heads().unwrap();
358
    assert_eq!(heads, vec![c2]);
359
    let log = hosted.log().unwrap();
360
    assert_eq!(log.iter().map(|e| e.hash).collect::<Vec<_>>(), vec![c2, c1]);
361
    assert!(log.iter().all(|e| !e.is_working_copy));
362
    assert_eq!(hosted.log_state(&[c1]).unwrap().len(), 1);
363
    assert_eq!(
364
        hosted.file_at("src/lib.rs", Some(&heads)).unwrap(),
365
        b"pub fn one() {}\npub fn two() {}\n"
366
    );
367
    let blame = hosted.blame_in("src/lib.rs", &heads).unwrap();
368
    assert_eq!(
369
        blame.iter().map(|l| l.hash).collect::<Vec<_>>(),
370
        vec![c1, c2]
371
    );
372
    assert_eq!(hosted.log_for_path("src/lib.rs").unwrap().len(), 2);
373
    assert_eq!(hosted.diff_files(Some(&c2)).unwrap().len(), 1);
374
    assert!(matches!(
375
        hosted.verify_change(&c2).unwrap(),
376
        SignatureStatus::Unsigned | SignatureStatus::Valid { .. }
377
    ));
378
379
    // Snapshotting a bare repository is refused rather than inventing a working copy.
380
    assert!(hosted.snapshot("test").is_err());
381
}
382
383
/// A host embeds the protocol with its own backend: names with an owner
384
/// segment, reads gated per repository, and a push hook.
385
#[test]
386
fn a_custom_backend_embeds_the_protocol() {
387
    use axum::extract::{Path as AxPath, State};
388
    use axum::http::{HeaderMap, Method, StatusCode};
389
    use axum::response::{IntoResponse, Response};
390
    use oxy_server::{Access, Backend, Endpoint, Failure, Server};
391
    use std::sync::{Arc, Mutex};
392
393
    struct Hub {
394
        root: std::path::PathBuf,
395
        pushes: Arc<Mutex<Vec<String>>>,
396
    }
397
    impl Backend for Hub {
398
        fn open(&self, name: &str) -> Result<Repo, Failure> {
399
            let Some((owner, repo)) = name.split_once('/') else {
400
                return Err(Failure(StatusCode::NOT_FOUND, "no such repository".into()));
401
            };
402
            let dir = self.root.join(owner).join(repo);
403
            if !dir.join(".oxy").is_dir() {
404
                return Err(Failure(StatusCode::NOT_FOUND, "no such repository".into()));
405
            }
406
            Repo::open(&dir).map_err(|e| Failure(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))
407
        }
408
        fn authorize(
409
            &self,
410
            name: &str,
411
            access: Access,
412
            headers: &HeaderMap,
413
        ) -> Result<(), Failure> {
414
            // `secret/*` is private: unknown to anyone without the token, even for reads.
415
            let token = oxy_server::bearer_token(headers);
416
            if name.starts_with("secret/") && token != Some("s3cret") {
417
                return Err(Failure(StatusCode::NOT_FOUND, "no such repository".into()));
418
            }
419
            if access == Access::Push && token.is_none() {
420
                return Err(Failure(StatusCode::UNAUTHORIZED, "sign in to push".into()));
421
            }
422
            Ok(())
423
        }
424
        fn after_push(&self, name: &str, results: &[oxy_core::protocol::RefResult]) {
425
            self.pushes.lock().unwrap().push(format!(
426
                "{name}: {}",
427
                results.iter().filter(|r| r.ok).count()
428
            ));
429
        }
430
    }
431
    async fn route(
432
        State(server): State<Arc<Server>>,
433
        AxPath((owner, repo, endpoint)): AxPath<(String, String, String)>,
434
        method: Method,
435
        headers: HeaderMap,
436
        body: axum::body::Bytes,
437
    ) -> Response {
438
        match Endpoint::parse(&endpoint) {
439
            Some(e) => {
440
                server
441
                    .handle(format!("{owner}/{repo}"), e, &method, &headers, body)
442
                    .await
443
            }
444
            None => Failure(StatusCode::NOT_FOUND, "no such endpoint".into()).into_response(),
445
        }
446
    }
447
448
    hermetic();
449
    let dir = tempfile::tempdir().unwrap();
450
    let root = dir.path();
451
    Repo::init_bare(&root.join("repos/andy/app")).unwrap();
452
    Repo::init_bare(&root.join("repos/secret/plans")).unwrap();
453
    let pushes = Arc::new(Mutex::new(Vec::new()));
454
    let server = Server::new(Hub {
455
        root: root.join("repos"),
456
        pushes: pushes.clone(),
457
    });
458
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
459
    let port = listener.local_addr().unwrap().port();
460
    std::thread::spawn(move || {
461
        let rt = tokio::runtime::Builder::new_multi_thread()
462
            .enable_all()
463
            .build()
464
            .unwrap();
465
        rt.block_on(async move {
466
            listener.set_nonblocking(true).unwrap();
467
            let app = axum::Router::new()
468
                .route(
469
                    "/{owner}/{repo}/oxy/v1/{endpoint}",
470
                    axum::routing::any(route),
471
                )
472
                .with_state(server);
473
            axum::serve(tokio::net::TcpListener::from_std(listener).unwrap(), app)
474
                .await
475
                .unwrap();
476
        });
477
    });
478
    std::thread::sleep(std::time::Duration::from_millis(50));
479
    let base = format!("http://127.0.0.1:{port}");
480
481
    // Anonymous clone of the public repository works; the private one does not exist for outsiders.
482
    let work = root.join("work");
483
    let client = Repo::clone(&format!("{base}/andy/app"), &work).unwrap();
484
    assert!(Repo::clone(&format!("{base}/secret/plans"), &root.join("nope")).is_err());
485
486
    // A push needs a token; with one it lands and the hook sees it.
487
    write(&work, "a.txt", b"a\n");
488
    client.snapshot("test").unwrap();
489
    client.commit("a", &[]).unwrap();
490
    assert!(
491
        client.push("origin", false).is_err(),
492
        "anonymous push must be refused"
493
    );
494
    let mut client = client;
495
    client.config.set("remote.origin.token", "s3cret").unwrap();
496
    let results = client.push("origin", false).unwrap();
497
    assert!(results.iter().all(|r| r.ok), "{results:?}");
498
    assert_eq!(pushes.lock().unwrap().as_slice(), ["andy/app: 1"]);
499
500
    let hosted = Repo::open(&root.join("repos/andy/app")).unwrap();
501
    assert_eq!(hosted.bookmark_heads().unwrap().len(), 1);
502
}
503
504
/// A hosted repository created unbound adopts the id of the first
505
/// repository pushed to it; before that it cannot be cloned or fetched.
506
#[test]
507
fn an_unbound_repository_adopts_the_first_pushers_id() {
508
    hermetic();
509
    let dir = tempfile::tempdir().unwrap();
510
    let root = dir.path();
511
    let served = root.join("served");
512
    let hosted = Repo::init_bare_unbound(&served.join("lib")).unwrap();
513
    assert!(hosted.is_unbound());
514
    assert_eq!(hosted.id().unwrap(), [0; 16]);
515
    let base = start(&served, vec![]);
516
    let url = format!("{base}/lib");
517
    assert!(Repo::clone(&url, &root.join("early")).err().unwrap().to_string().contains("unbound"));
518
519
    // An existing repository with its own id and history.
520
    let work = root.join("work");
521
    let mut local = Repo::init(&work).unwrap();
522
    let my_id = local.id().unwrap();
523
    write(&work, "a.txt", b"a\n");
524
    local.snapshot("test").unwrap();
525
    local.commit("a", &[]).unwrap();
526
    local.config.set("remote.origin.url", &url).unwrap();
527
    local.save_config().unwrap();
528
    assert!(local.fetch("origin").err().unwrap().to_string().contains("unbound"), "fetching an unbound remote is refused");
529
    let results = local.push("origin", false).unwrap();
530
    assert!(results.iter().all(|r| r.ok), "{results:?}");
531
532
    let hosted = Repo::open(&served.join("lib")).unwrap();
533
    assert!(!hosted.is_unbound());
534
    assert_eq!(hosted.id().unwrap(), my_id, "bound to the pusher's id");
535
    assert_eq!(hosted.bookmark_heads().unwrap().len(), 1);
536
    // From now on it is an ordinary remote: clones share the id, other repositories are refused.
537
    let cloned = Repo::clone(&url, &root.join("late")).unwrap();
538
    assert_eq!(cloned.id().unwrap(), my_id);
539
    let mut other = Repo::init(&root.join("other")).unwrap();
540
    other.config.set("remote.origin.url", &url).unwrap();
541
    other.save_config().unwrap();
542
    write(&root.join("other"), "b.txt", b"b\n");
543
    other.snapshot("test").unwrap();
544
    other.commit("b", &[]).unwrap();
545
    assert!(other.push("origin", false).err().unwrap().to_string().contains("different repository"));
546
}