OXYGENHUB
◆ v0.9.0 oxygen / crates / oxy-server / src / lib.rs
21.9 KiB · Rust · last changed oxy-server: a host can add a note to a reference a push moved (closes #148) 9 days ago source blame history raw
1
//! Reference HTTP server for the Oxygen wire protocol (spec §15.1, §16).
2
//!
3
//! Serves every repository directory under a root at `/{repo}/oxy/v1/...`,
4
//! over TLS when a certificate and key are given, else plain HTTP for use
5
//! behind a TLS-terminating proxy or on a loopback interface (spec §15.1).
6
//!
7
//! Authentication: when bearer tokens are configured, `push` requires one;
8
//! reads are open. With no tokens configured everything is open (development).
9
//!
10
//! Hosts embed the protocol by implementing [`Backend`] (name resolution
11
//! and authorization) and routing to [`Server::handle`]; `router` is the
12
//! standalone server built on [`DirBackend`].
13
14
pub mod witness;
15
16
use axum::body::Bytes;
17
use axum::extract::{Path, State};
18
use axum::http::{header, HeaderMap, Method, StatusCode};
19
use axum::response::{IntoResponse, Response};
20
use axum::routing::any;
21
use axum::Router;
22
use oxy_core::protocol::{wire, CONTENT_TYPE};
23
use oxy_core::repo::Repo;
24
use std::collections::HashMap;
25
use std::path::PathBuf;
26
use std::sync::{Arc, Mutex};
27
28
#[derive(Clone, Debug)]
29
pub struct ServerConfig {
30
    /// Directory containing one repository per subdirectory.
31
    pub root: PathBuf,
32
    /// Bearer tokens allowed to push; empty means no authentication.
33
    pub tokens: Vec<String>,
34
    /// Serve HTTPS with this certificate chain and private key (PEM files).
35
    pub tls: Option<TlsFiles>,
36
}
37
38
#[derive(Clone, Debug)]
39
pub struct TlsFiles {
40
    pub cert: PathBuf,
41
    pub key: PathBuf,
42
}
43
44
/// Which kind of access a request needs (spec §15.1: reads may be open;
45
/// a push must be authenticated when any credential is configured).
46
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
47
pub enum Access {
48
    Read,
49
    Push,
50
}
51
52
/// What a host provides to the protocol server: how a repository name maps
53
/// to a repository on disk and who may do what. `oxy serve` uses
54
/// [`DirBackend`]; a hub supplies its own with accounts and roles.
55
pub trait Backend: Send + Sync + 'static {
56
    /// Open the repository called `name`, or fail with the status to
57
    /// answer. An unknown name is `404`; a name the caller may not see
58
    /// SHOULD also be `404`, so private repositories are not enumerable.
59
    fn open(&self, name: &str) -> Result<Repo, Failure>;
60
61
    /// Decide whether this request may perform `access` on `name`.
62
    fn authorize(&self, name: &str, access: Access, headers: &HeaderMap) -> Result<(), Failure>;
63
64
    /// Called after a push was processed, with its per-reference results
65
    /// (accepted or refused), the states each accepted reference moved
66
    /// between, and the changes stored, for logs and cache invalidation.
67
    fn after_push(&self, _name: &str, _report: &oxy_core::repo::PushReport, _headers: &HeaderMap) {}
68
69
    /// Notes for the pusher on references this push moved: each
70
    /// `(reference, note)` is added to that accepted reference's result,
71
    /// which a client shows beside it (`Pushed origin/x - note`). For a
72
    /// push that did what the pusher may not expect, say moving the
73
    /// bookmark of a proposal that has landed (oxygenhub#329, #148). Called
74
    /// before [`Backend::after_push`], which sees the notes.
75
    fn push_notes(
76
        &self,
77
        _name: &str,
78
        _report: &oxy_core::repo::PushReport,
79
        _headers: &HeaderMap,
80
    ) -> Vec<(String, String)> {
81
        Vec::new()
82
    }
83
84
    /// Vet a push by what it adds before anything of it is stored or any
85
    /// reference moves (spec §16.1): `bundle` is the push as decoded, its
86
    /// changes with their lines, `updates` the references it would move.
87
    /// A host refuses the whole push by returning a [`Failure`], whose
88
    /// text reaches the client as the push's error, as it is, several
89
    /// lines included; it should answer neither 401, 403 nor 404, which a
90
    /// client takes for a credential it should change. Called for every
91
    /// part of a push sent in parts (each part's changes are its own),
92
    /// outside the repository's push lock, so a slow vet holds up no
93
    /// other push. The default accepts everything at no cost.
94
    fn vet_push(
95
        &self,
96
        _name: &str,
97
        _repo: &Repo,
98
        _bundle: &oxy_core::protocol::Bundle,
99
        _updates: &[oxy_core::protocol::RefUpdate],
100
        _headers: &HeaderMap,
101
    ) -> Result<(), Failure> {
102
        Ok(())
103
    }
104
105
    /// Add the host's transparency log to a refs answer (spec §14.8): its
106
    /// checkpoint, and each attestation's entry and inclusion proof. A host
107
    /// without a log leaves it as it is.
108
    fn add_log(&self, _name: &str, _refs: &mut oxy_core::protocol::RefsResponse) {}
109
110
    /// Tailor a refs answer to the client asking, by its request headers
111
    /// (its `User-Agent` and the features it declares in `Oxy-Features`),
112
    /// after [`Backend::add_log`]. The default leaves out the tags a client
113
    /// that does not declare `certified-tags` would refuse
114
    /// ([`Repo::refs_for_client`], oxygen#131); a host that overrides it
115
    /// should apply that too ([`tailor_refs`]).
116
    fn adjust_refs(
117
        &self,
118
        _name: &str,
119
        repo: &Repo,
120
        refs: &mut oxy_core::protocol::RefsResponse,
121
        headers: &HeaderMap,
122
    ) {
123
        tailor_refs(repo, refs, headers);
124
    }
125
126
    /// Whether tag `tag` of `name` may be withdrawn (oxygenhub#301): called
127
    /// for each withdrawal a push carries, before the repository sees it.
128
    /// A host refuses one it depends on (a forge's release names the tag)
129
    /// with the reason, which is that withdrawal's result. The default
130
    /// allows every one; the policy is checked after, as for any push.
131
    fn vet_tag_withdrawal(&self, _name: &str, _tag: &str) -> Result<(), String> {
132
        Ok(())
133
    }
134
135
    /// The bookmark of `name` that pushes may never delete (its default),
136
    /// when the host keeps one apart from the repository's own config.
137
    fn default_bookmark(&self, _name: &str) -> Option<String> {
138
        None
139
    }
140
}
141
142
/// A protocol server over any [`Backend`], serialising the pushes to each
143
/// repository. Hosts route requests to [`Server::handle`].
144
pub struct Server {
145
    backend: Box<dyn Backend>,
146
    locks: Mutex<HashMap<String, Arc<Mutex<()>>>>,
147
}
148
149
/// An error response: a status and a plain-text body.
150
#[derive(Debug)]
151
pub struct Failure(pub StatusCode, pub String);
152
153
impl IntoResponse for Failure {
154
    fn into_response(self) -> Response {
155
        (self.0, self.1).into_response()
156
    }
157
}
158
159
impl From<oxy_core::repo::RepoError> for Failure {
160
    fn from(e: oxy_core::repo::RepoError) -> Failure {
161
        use oxy_core::repo::RepoError;
162
        // The server's own storage failing is not the client's fault.
163
        let status = match &e {
164
            RepoError::Io(_) | RepoError::Store(_) => StatusCode::INTERNAL_SERVER_ERROR,
165
            _ => StatusCode::BAD_REQUEST,
166
        };
167
        Failure(status, e.to_string())
168
    }
169
}
170
171
/// The wire protocol's endpoints (spec §16.1), named as in the URL.
172
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
173
pub enum Endpoint {
174
    Capabilities,
175
    Refs,
176
    Negotiate,
177
    Fetch,
178
    Objects,
179
    Checkpoint,
180
    CheckpointIndex,
181
    Shallow,
182
    Push,
183
}
184
185
impl Endpoint {
186
    pub fn parse(name: &str) -> Option<Endpoint> {
187
        Some(match name {
188
            "capabilities" => Endpoint::Capabilities,
189
            "refs" => Endpoint::Refs,
190
            "negotiate" => Endpoint::Negotiate,
191
            "fetch" => Endpoint::Fetch,
192
            "objects" => Endpoint::Objects,
193
            "checkpoint" => Endpoint::Checkpoint,
194
            "checkpoint-index" => Endpoint::CheckpointIndex,
195
            "shallow" => Endpoint::Shallow,
196
            "push" => Endpoint::Push,
197
            _ => return None,
198
        })
199
    }
200
201
    pub fn name(self) -> &'static str {
202
        match self {
203
            Endpoint::Capabilities => "capabilities",
204
            Endpoint::Refs => "refs",
205
            Endpoint::Negotiate => "negotiate",
206
            Endpoint::Fetch => "fetch",
207
            Endpoint::Objects => "objects",
208
            Endpoint::Checkpoint => "checkpoint",
209
            Endpoint::CheckpointIndex => "checkpoint-index",
210
            Endpoint::Shallow => "shallow",
211
            Endpoint::Push => "push",
212
        }
213
    }
214
215
    /// The HTTP method the endpoint answers to.
216
    pub fn method(self) -> Method {
217
        match self {
218
            Endpoint::Capabilities | Endpoint::Refs => Method::GET,
219
            _ => Method::POST,
220
        }
221
    }
222
223
    fn access(self) -> Access {
224
        match self {
225
            Endpoint::Push => Access::Push,
226
            _ => Access::Read,
227
        }
228
    }
229
}
230
231
fn ok(body: Vec<u8>) -> Response {
232
    ([(header::CONTENT_TYPE, CONTENT_TYPE)], body).into_response()
233
}
234
235
pub fn valid_repo_name(name: &str) -> bool {
236
    !name.is_empty()
237
        && !name.starts_with('.')
238
        && name
239
            .chars()
240
            .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'))
241
}
242
243
impl Server {
244
    pub fn new(backend: impl Backend) -> Arc<Server> {
245
        Arc::new(Server {
246
            backend: Box::new(backend),
247
            locks: Mutex::new(HashMap::new()),
248
        })
249
    }
250
251
    pub fn backend(&self) -> &dyn Backend {
252
        self.backend.as_ref()
253
    }
254
255
    fn lock_for(&self, name: &str) -> Arc<Mutex<()>> {
256
        self.locks
257
            .lock()
258
            .unwrap()
259
            .entry(name.to_string())
260
            .or_default()
261
            .clone()
262
    }
263
264
    /// Answer one protocol request for repository `name`. `method` is
265
    /// checked against the endpoint; the body is the raw CBOR request.
266
    pub async fn handle(
267
        self: &Arc<Self>,
268
        name: String,
269
        endpoint: Endpoint,
270
        method: &Method,
271
        headers: &HeaderMap,
272
        body: Bytes,
273
    ) -> Response {
274
        if *method != endpoint.method() {
275
            return Failure(
276
                StatusCode::METHOD_NOT_ALLOWED,
277
                format!("{} is {}", endpoint.name(), endpoint.method()),
278
            )
279
            .into_response();
280
        }
281
        if let Err(f) = self.backend.authorize(&name, endpoint.access(), headers) {
282
            return f.into_response();
283
        }
284
        let claimed = headers
285
            .get(oxy_core::protocol::REPO_ID_HEADER)
286
            .and_then(|v| v.to_str().ok())
287
            .and_then(hex_id);
288
        let server = Arc::clone(self);
289
        let request_headers = headers.clone();
290
        let repo = name.clone();
291
        let result = tokio::task::spawn_blocking(move || {
292
            server.respond(&repo, endpoint, body, claimed, &request_headers)
293
        })
294
        .await;
295
        let failure = match result {
296
            Ok(Ok(r)) => return r,
297
            Ok(Err(f)) => f,
298
            Err(e) => Failure(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()),
299
        };
300
        // Name the operation and repository, in the log and for the client.
301
        let f = Failure(
302
            failure.0,
303
            format!("{} of {name}: {}", endpoint.name(), failure.1),
304
        );
305
        eprintln!("oxy-server: {}: {}", f.0.as_u16(), f.1);
306
        f.into_response()
307
    }
308
309
    fn respond(
310
        &self,
311
        name: &str,
312
        endpoint: Endpoint,
313
        body: Bytes,
314
        claimed: Option<[u8; 16]>,
315
        headers: &HeaderMap,
316
    ) -> Result<Response, Failure> {
317
        let r = self.backend.open(name)?;
318
        Ok(match endpoint {
319
            Endpoint::Capabilities => ok(wire::encode_capabilities(r.id()?)),
320
            Endpoint::Refs => {
321
                let mut refs = r.remote_refs()?;
322
                if let Some(d) = self.backend.default_bookmark(name) {
323
                    refs.default_bookmark = Some(d);
324
                }
325
                self.backend.add_log(name, &mut refs);
326
                self.backend.adjust_refs(name, &r, &mut refs, headers);
327
                ok(wire::encode_refs(r.id()?, &refs))
328
            }
329
            Endpoint::Negotiate => {
330
                let (have, want) = wire::decode_negotiate(&body)?;
331
                ok(wire::encode_hashes(&r.negotiate(&have, &want)?))
332
            }
333
            Endpoint::Fetch => {
334
                let (changes, max_blob) = wire::decode_fetch(&body)?;
335
                ok(wire::encode_bundle(
336
                    &r.bundle_for_filtered(&changes, max_blob)?,
337
                ))
338
            }
339
            Endpoint::Objects => {
340
                let refs = wire::decode_hashes(&body)?;
341
                let bundle = oxy_core::protocol::Bundle {
342
                    objects: r.objects_for(&refs)?,
343
                    ..Default::default()
344
                };
345
                ok(wire::encode_bundle(&bundle))
346
            }
347
            Endpoint::Checkpoint => {
348
                let heads = wire::decode_hashes(&body)?;
349
                let lock = self.lock_for(name);
350
                let _guard = lock.lock().unwrap();
351
                let bundle = r.checkpoint_bundle_for(&heads)?;
352
                let _t = oxy_core::timing::phase("serve: encode the checkpoint");
353
                ok(wire::encode_bundle(&bundle))
354
            }
355
            Endpoint::CheckpointIndex => {
356
                let heads = wire::decode_hashes(&body)?;
357
                let lock = self.lock_for(name);
358
                let _guard = lock.lock().unwrap();
359
                ok(wire::encode_bundle(&r.checkpoint_index_for(&heads)?))
360
            }
361
            Endpoint::Shallow => {
362
                let (heads, depth) = wire::decode_shallow(&body)?;
363
                ok(wire::encode_hashes(&r.shallow_changes(&heads, depth)?))
364
            }
365
            Endpoint::Push => {
366
                let (bundle, updates, policy, tags) = wire::decode_push(&body)?;
367
                // The host vets what the push adds before anything is stored
368
                // or bound: a refused push leaves the repository as it was.
369
                self.backend
370
                    .vet_push(name, &r, &bundle, &updates, headers)?;
371
                let lock = self.lock_for(name);
372
                let _guard = lock.lock().unwrap();
373
                if r.is_unbound() {
374
                    // The first push binds the repository to the pusher's id (spec §16.1).
375
                    match claimed {
376
                        Some(id) => r.bind_id(id)?,
377
                        None => {
378
                            return Err(Failure(
379
                                StatusCode::BAD_REQUEST,
380
                                format!(
381
                                    "this repository is unbound; push with a client that sends {}",
382
                                    oxy_core::protocol::REPO_ID_HEADER
383
                                ),
384
                            ))
385
                        }
386
                    }
387
                }
388
                let protected = self.backend.default_bookmark(name);
389
                let (refused, updates): (Vec<_>, Vec<_>) = updates.into_iter().partition(|u| {
390
                    u.new_state.is_zero() && protected.as_deref() == Some(u.name.as_str())
391
                });
392
                let mut kept_tags = Vec::new();
393
                let mut withdrawals_refused = Vec::new();
394
                for t in tags {
395
                    match t
396
                        .is_withdrawal()
397
                        .then(|| self.backend.vet_tag_withdrawal(name, &t.name))
398
                    {
399
                        Some(Err(reason)) => withdrawals_refused.push((t.name, reason)),
400
                        _ => kept_tags.push(t),
401
                    }
402
                }
403
                let mut report = r.receive_push_report(bundle, updates, policy, kept_tags)?;
404
                // The request named references, all refused here: it was no
405
                // push of objects alone, whatever the repository saw.
406
                if !refused.is_empty() || !withdrawals_refused.is_empty() {
407
                    report
408
                        .results
409
                        .retain(|r| r.name != oxy_core::repo::OBJECTS_RESULT);
410
                }
411
                for u in refused {
412
                    report.results.push(oxy_core::protocol::RefResult {
413
                        name: u.name.clone(),
414
                        ok: false,
415
                        reason: format!(
416
                            "{:?} is this repository's default bookmark and cannot be deleted",
417
                            u.name
418
                        ),
419
                    });
420
                }
421
                for (tag, reason) in withdrawals_refused {
422
                    report.results.push(oxy_core::protocol::RefResult {
423
                        name: format!("tags/{tag}"),
424
                        ok: false,
425
                        reason,
426
                    });
427
                }
428
                for r in report.results.iter().filter(|r| !r.ok) {
429
                    eprintln!(
430
                        "oxy-server: push to {name}: {} refused: {}",
431
                        r.name, r.reason
432
                    );
433
                }
434
                for (reference, note) in self.backend.push_notes(name, &report, headers) {
435
                    if let Some(r) = report
436
                        .results
437
                        .iter_mut()
438
                        .find(|r| r.ok && r.name == reference)
439
                    {
440
                        r.reason = if r.reason.is_empty() {
441
                            note
442
                        } else {
443
                            format!("{}; {note}", r.reason)
444
                        };
445
                    }
446
                }
447
                self.backend.after_push(name, &report, headers);
448
                ok(wire::encode_push_response(&report.results))
449
            }
450
        })
451
    }
452
}
453
454
/// The backend of `oxy serve`: one repository per subdirectory of a root,
455
/// open reads, and pushes gated by a list of bearer tokens.
456
pub struct DirBackend {
457
    pub root: PathBuf,
458
    pub tokens: Vec<String>,
459
}
460
461
impl Backend for DirBackend {
462
    fn open(&self, name: &str) -> Result<Repo, Failure> {
463
        if !valid_repo_name(name) {
464
            return Err(Failure(StatusCode::NOT_FOUND, "no such repository".into()));
465
        }
466
        let dir = self.root.join(name);
467
        if !dir.join(".oxy").is_dir() {
468
            return Err(Failure(StatusCode::NOT_FOUND, "no such repository".into()));
469
        }
470
        Repo::open(&dir).map_err(|e| Failure(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))
471
    }
472
473
    fn default_bookmark(&self, name: &str) -> Option<String> {
474
        self.open(name).ok().map(|r| r.config.bookmark.clone())
475
    }
476
477
    fn authorize(&self, _name: &str, access: Access, headers: &HeaderMap) -> Result<(), Failure> {
478
        if access == Access::Read || self.tokens.is_empty() {
479
            return Ok(());
480
        }
481
        match bearer_token(headers) {
482
            Some(t) if self.tokens.iter().any(|x| x == t) => Ok(()),
483
            _ => Err(Failure(
484
                StatusCode::UNAUTHORIZED,
485
                "a valid bearer token is required to push".into(),
486
            )),
487
        }
488
    }
489
}
490
491
fn hex_id(h: &str) -> Option<[u8; 16]> {
492
    let h = h.trim();
493
    if h.len() != 32 {
494
        return None;
495
    }
496
    let mut out = [0u8; 16];
497
    for (i, byte) in out.iter_mut().enumerate() {
498
        *byte = u8::from_str_radix(&h[2 * i..2 * i + 2], 16).ok()?;
499
    }
500
    Some(out)
501
}
502
503
/// The features a request declares in its `Oxy-Features` header, as the
504
/// header's text; `None` when it sent none (a client of the 0.5.0 era).
505
pub fn client_features(headers: &HeaderMap) -> Option<&str> {
506
    headers
507
        .get(oxy_core::protocol::FEATURES_HEADER)
508
        .and_then(|v| v.to_str().ok())
509
}
510
511
/// Tailor `refs` of `repo` to the client whose request carried `headers`:
512
/// the shared rule of [`Repo::refs_for_client`], which every host applies.
513
pub fn tailor_refs(repo: &Repo, refs: &mut oxy_core::protocol::RefsResponse, headers: &HeaderMap) {
514
    repo.refs_for_client(refs, client_features(headers));
515
}
516
517
/// The token of an `Authorization: Bearer` header, if any.
518
pub fn bearer_token(headers: &HeaderMap) -> Option<&str> {
519
    headers
520
        .get(header::AUTHORIZATION)
521
        .and_then(|v| v.to_str().ok())
522
        .and_then(|v| v.strip_prefix("Bearer "))
523
        .map(str::trim)
524
}
525
526
async fn dispatch(
527
    State(server): State<Arc<Server>>,
528
    Path((repo, endpoint)): Path<(String, String)>,
529
    method: Method,
530
    headers: HeaderMap,
531
    body: Bytes,
532
) -> Response {
533
    match Endpoint::parse(&endpoint) {
534
        Some(e) => server.handle(repo, e, &method, &headers, body).await,
535
        None => Failure(StatusCode::NOT_FOUND, "no such endpoint".into()).into_response(),
536
    }
537
}
538
539
/// The largest request `oxy serve` takes: a push part is about
540
/// `push.batch-bytes` (64 MiB), but one change can weigh more; OxygenHub's
541
/// default `push_body_limit` is the same.
542
pub const BODY_LIMIT: usize = if usize::BITS > 32 {
543
    4 << 30
544
} else {
545
    usize::MAX
546
};
547
548
/// The router of `oxy serve`: `/{repo}/oxy/v1/{endpoint}` over a [`DirBackend`].
549
pub fn router(cfg: ServerConfig) -> Router {
550
    let server = Server::new(DirBackend {
551
        root: cfg.root,
552
        tokens: cfg.tokens,
553
    });
554
    // Without a limit of its own a request takes axum's 2 MB, and a larger
555
    // push was cut off with no word in the log ("Broken pipe" at the client).
556
    Router::new()
557
        .route("/{repo}/oxy/v1/{endpoint}", any(dispatch))
558
        .layer(axum::extract::DefaultBodyLimit::max(BODY_LIMIT))
559
        .with_state(server)
560
}
561
562
/// Serve on an already-bound listener until the process ends; HTTPS when
563
/// `cfg.tls` is set.
564
pub async fn serve(listener: std::net::TcpListener, cfg: ServerConfig) -> std::io::Result<()> {
565
    listener.set_nonblocking(true)?;
566
    match cfg.tls.clone() {
567
        Some(tls) => {
568
            let _ = rustls::crypto::ring::default_provider().install_default();
569
            let rustls_cfg =
570
                axum_server::tls_rustls::RustlsConfig::from_pem_file(&tls.cert, &tls.key)
571
                    .await
572
                    .map_err(|e| std::io::Error::other(format!("loading TLS files: {e}")))?;
573
            axum_server::from_tcp_rustls(listener, rustls_cfg)?
574
                .serve(router(cfg).into_make_service())
575
                .await
576
        }
577
        None => {
578
            let listener = tokio::net::TcpListener::from_std(listener)?;
579
            axum::serve(listener, router(cfg)).await
580
        }
581
    }
582
}
583
584
/// Build a runtime and serve; for the CLI and for tests on a background thread.
585
pub fn run_blocking(listener: std::net::TcpListener, cfg: ServerConfig) -> std::io::Result<()> {
586
    let rt = tokio::runtime::Builder::new_multi_thread()
587
        .enable_all()
588
        .build()?;
589
    rt.block_on(serve(listener, cfg))
590
}