OXYGENHUB
# 835ceb9b3ea7 oxygen / crates / oxy-core / src / protocol.rs
69.8 KiB · Rust · last changed Fetch every bookmark in one exchange, and none when nothing is new: a no-op pull of 160 bookmarks 169 s to 3 s, a clone 324 s to 26 s (closes #158) 4 days ago source blame history raw
5b9c6ad4 Andy McCall ↶ 1
//! Transport-agnostic protocol (spec §16): what a client asks of a remote and
2
//! what a server does with a push. `LocalRemote` serves another repository
750251b2 Andy McCall ↶ 3
//! directory in-process, `HttpRemote` speaks to an `oxy-server` over HTTP(S),
4
//! and the server side reuses `Repo::receive_push` and friends unchanged.
5
//!
6
//! Wire messages are deterministic CBOR (`wire` module); over HTTP each is a
7
//! request or response body with content type `application/oxy-cbor`.
5b9c6ad4 Andy McCall ↶ 8
9
use crate::attest::Attest;
750251b2 Andy McCall ↶ 10
use crate::cbor::{self, Value};
5b9c6ad4 Andy McCall ↶ 11
use crate::hash::{ChangeHash, ObjectRef, ObjectType};
12
use crate::identity::Signature;
750251b2 Andy McCall ↶ 13
use crate::objects::{self as o, Change, ObjectError};
83e0677b Andy McCall ↶ 14
use crate::policy::Policy;
5b9c6ad4 Andy McCall ↶ 15
use crate::repo::{Repo, RepoError, Result};
2b5ee371 Andy McCall ↶ 16
use std::path::{Path, PathBuf};
9f9e72b9 Andy McCall ↶ 17
use std::sync::{Arc, Mutex};
5b9c6ad4 Andy McCall ↶ 18
750251b2 Andy McCall ↶ 19
pub const CONTENT_TYPE: &str = "application/oxy-cbor";
20
pub const PROTOCOL_VERSION: u64 = 1;
21
5b9c6ad4 Andy McCall ↶ 22
/// An attestation together with the signatures over its ObjectRef.
23
#[derive(Clone, Debug, PartialEq, Eq)]
24
pub struct SignedAttest {
25
    pub attest: Attest,
26
    pub sigs: Vec<Signature>,
27
}
28
83e0677b Andy McCall ↶ 29
/// A policy together with the root signatures over its ObjectRef.
30
#[derive(Clone, Debug, PartialEq, Eq)]
31
pub struct SignedPolicy {
32
    pub policy: Policy,
33
    pub sigs: Vec<Signature>,
34
}
35
28fc23d7 Andy McCall ↶ 36
impl SignedPolicy {
37
    /// The proposal-file encoding (`wire::policy_value`).
38
    pub fn encode(&self) -> Vec<u8> {
39
        crate::cbor::encode(&wire::policy_value(self))
40
    }
41
    pub fn decode(b: &[u8]) -> Result<SignedPolicy> {
75167672 Andy McCall 42
        wire::policy_from(
43
            &crate::cbor::decode(b).map_err(|e| RepoError::Object(ObjectError::from(e)))?,
44
        )
28fc23d7 Andy McCall ↶ 45
    }
46
}
47
132b884b Andy McCall ↶ 48
/// What `refs` returns: every bookmark, every tag, and the current policy.
83e0677b Andy McCall ↶ 49
#[derive(Clone, Debug, Default)]
50
pub struct RefsResponse {
51
    pub refs: Vec<RemoteRef>,
52
    pub policy: Option<SignedPolicy>,
132b884b Andy McCall ↶ 53
    pub tags: Vec<TagRef>,
28fc23d7 Andy McCall ↶ 54
    /// Earlier policies, ascending by serial, so a client behind by more
55
    /// than one update can verify each step (spec §14.5).
56
    pub history: Vec<SignedPolicy>,
14edbaaf Andy McCall ↶ 57
    /// The bookmark a clone checks out (spec §16.1), when the remote names one.
58
    pub default_bookmark: Option<String>,
4b9d0677 Andy McCall ↶ 59
    /// The host's latest transparency-log checkpoint, a signed note (spec
60
    /// §14.8), against which `RemoteRef::logged` proofs are made (key 6,
61
    /// only when the host keeps a log).
62
    pub log: Option<String>,
028545df Andy McCall ↶ 63
    /// The witnesses the host asks to cosign its checkpoints: each one's
64
    /// name in cosignatures and its identity's bytes (key 7, only when it
65
    /// has any), which a client pins (spec §14.8, *Integrated time*).
66
    pub witnesses: Vec<(String, Vec<u8>)>,
b63fc7a7 Andy McCall ↶ 67
    /// Tags withdrawn here (spec §10.1; key 8, only when there are any):
68
    /// each with its `untag` attestation as `attestation`, and that
69
    /// attestation's acceptance as `accepted` when a short-lived key
70
    /// signed it. Older clients ignore them and keep the tag they have.
71
    pub withdrawn: Vec<TagRef>,
83e0677b Andy McCall ↶ 72
}
73
4b9d0677 Andy McCall ↶ 74
/// Where a reference attestation sits in the host's transparency log
75
/// (spec §14.8): its entry number, the entry, and the inclusion proof
76
/// against the checkpoint served beside it.
77
#[derive(Clone, Debug, PartialEq, Eq)]
78
pub struct Logged {
79
    pub index: u64,
80
    pub entry: Vec<u8>,
81
    pub proof: Vec<[u8; 32]>,
82
}
83
028545df Andy McCall ↶ 84
/// A checkpoint witnesses cosigned that includes a logged entry (spec
85
/// §14.8, *Integrated time*): the cosigned note, and the entry's inclusion
86
/// proof against it.
87
#[derive(Clone, Debug, PartialEq, Eq)]
88
pub struct Integration {
89
    pub note: String,
90
    pub proof: Vec<[u8; 32]>,
91
}
92
132b884b Andy McCall ↶ 93
/// A tag as a remote reports it (spec §10.1).
94
#[derive(Clone, Debug, PartialEq, Eq)]
95
pub struct TagRef {
96
    pub name: String,
97
    pub state: ObjectRef,
98
    pub heads: Vec<ChangeHash>,
99
    pub attestation: Option<SignedAttest>,
1ec6fdbc Andy McCall ↶ 100
    /// The acceptance of `attestation` by its signer's certificate issuer
101
    /// when a short-lived key signed it (spec §14.7; key 4, only when there
102
    /// is one): the time a fetch checks the certificate at.
103
    pub accepted: Option<SignedAttest>,
104
    /// The attestation's place in the host's transparency log (key 5,
105
    /// only when it has one), as a bookmark's.
106
    pub logged: Option<Logged>,
107
    /// The cosigned checkpoints that first included `logged`'s entry (key
108
    /// 6, only when there are any).
109
    pub integration: Vec<Integration>,
110
}
111
112
impl TagRef {
b63fc7a7 Andy McCall ↶ 113
    /// Whether this is a tag's withdrawal (its attestation an `untag`), as
114
    /// a push carries one (oxygenhub#301).
115
    pub fn is_withdrawal(&self) -> bool {
116
        self.attestation
117
            .as_ref()
118
            .is_some_and(|a| matches!(a.attest.body, crate::attest::AttestBody::Untag { .. }))
119
    }
120
1ec6fdbc Andy McCall ↶ 121
    /// A tag with no acceptance or log place (yet).
122
    pub fn new(
123
        name: String,
124
        state: ObjectRef,
125
        heads: Vec<ChangeHash>,
126
        attestation: Option<SignedAttest>,
127
    ) -> TagRef {
128
        TagRef {
129
            name,
130
            state,
131
            heads,
132
            attestation,
133
            accepted: None,
134
            logged: None,
135
            integration: vec![],
136
        }
137
    }
132b884b Andy McCall ↶ 138
}
139
5b9c6ad4 Andy McCall ↶ 140
/// One reference as a remote reports it.
141
#[derive(Clone, Debug)]
142
pub struct RemoteRef {
143
    pub name: String,
144
    pub state: ObjectRef,
145
    pub heads: Vec<ChangeHash>,
4161262e Andy McCall ↶ 146
    /// The latest attestation, only when it describes `state`. An unattested
147
    /// push (accepted while no trust policy exists) leaves it `None`.
5b9c6ad4 Andy McCall ↶ 148
    pub attestation: Option<SignedAttest>,
4161262e Andy McCall ↶ 149
    /// Serial of the latest attestation ever recorded, stale or not, so the
150
    /// next attestation continues the sequence.
151
    pub last_serial: Option<u64>,
f53cbe7a Andy McCall ↶ 152
    /// The host's acceptance of `attestation` when a short-lived key signed
153
    /// it (spec §14.7): the time its certificate is checked at.
154
    pub accepted: Option<SignedAttest>,
4b9d0677 Andy McCall ↶ 155
    /// The attestation's place in the host's transparency log (key 6, only
156
    /// when it has one).
157
    pub logged: Option<Logged>,
028545df Andy McCall ↶ 158
    /// The cosigned checkpoints that first included `logged`'s entry (key
159
    /// 7, only when there are any): what its integrated time is read from.
160
    pub integration: Vec<Integration>,
5b9c6ad4 Andy McCall ↶ 161
}
162
163
/// Everything needed to add a set of changes to a repository.
164
#[derive(Clone, Debug, Default)]
165
pub struct Bundle {
166
    /// In dependency order, each with its individual signatures.
167
    pub changes: Vec<(Change, Vec<Signature>)>,
168
    /// Seals covering the changes.
169
    pub attests: Vec<SignedAttest>,
170
    /// Chunks, blobs, identities, and state objects referenced by the changes.
171
    pub objects: Vec<(ObjectType, Vec<u8>)>,
28fc23d7 Andy McCall ↶ 172
    /// Policies between the receiver's and the one sent with the push,
173
    /// ascending by serial (spec §14.5).
174
    pub policies: Vec<SignedPolicy>,
598009a7 Andy McCall ↶ 175
    /// Each change's hash, in the order of `changes`, when whoever made
176
    /// the bundle knows it: the store it came from, or the bytes it was
177
    /// decoded from. Never read from the wire. Empty when unknown.
178
    pub hashes: Vec<ChangeHash>,
2c75672f Andy McCall ↶ 179
    /// Changes as stored, with their signatures, in dependency order, for a
180
    /// fetch between repositories on one machine: their text blobs and
181
    /// chunks are among `objects`. Never on the wire.
182
    pub raw_changes: Vec<(Vec<u8>, Vec<Signature>)>,
f0a517d2 Andy McCall ↶ 183
    /// Certificates of short-lived keys that signed what the bundle carries
184
    /// (spec §14.7), their issuers' identities among `objects`.
185
    pub certificates: Vec<crate::cert::SignedCertificate>,
5b9c6ad4 Andy McCall ↶ 186
}
187
188
impl Bundle {
189
    pub fn is_empty(&self) -> bool {
190
        self.changes.is_empty() && self.attests.is_empty() && self.objects.is_empty()
191
    }
192
}
193
194
/// A requested bookmark move (spec §10.2, §16.3).
195
#[derive(Clone, Debug)]
196
pub struct RefUpdate {
197
    pub name: String,
198
    pub old_state: ObjectRef,
199
    pub new_state: ObjectRef,
200
    pub force: bool,
201
    pub attestation: Option<SignedAttest>,
202
}
203
204
#[derive(Clone, Debug, PartialEq, Eq)]
205
pub struct RefResult {
206
    pub name: String,
207
    pub ok: bool,
208
    pub reason: String,
209
}
210
211
pub trait Remote {
212
    fn repo_id(&self) -> Result<[u8; 16]>;
83e0677b Andy McCall ↶ 213
    fn refs(&self) -> Result<RefsResponse>;
5b9c6ad4 Andy McCall ↶ 214
    /// Changes in the closure of `want` but not of `have`, in dependency order.
215
    fn negotiate(&self, have: &[ChangeHash], want: &[ChangeHash]) -> Result<Vec<ChangeHash>>;
b3c5a59a Andy McCall ↶ 216
    /// The changes with their signatures, seals, identities, and blobs; blobs
217
    /// larger than `max_blob` bytes are left out for lazy fetching (spec §13.5).
218
    fn fetch(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle>;
219
    /// Individual objects by reference (spec §16.1 `objects`); a blob comes
220
    /// with its chunks.
221
    fn objects(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>>;
2863456a Andy McCall ↶ 222
    /// A checkpoint of exactly `heads` (spec §16.1 `checkpoint`): the
223
    /// checkpoint object, its payload blob, and the blob's chunks.
224
    fn checkpoint(&self, heads: &[ChangeHash]) -> Result<Bundle>;
80b069fa Andy McCall ↶ 225
    /// A checkpoint of exactly `heads` without its graph blobs (spec §16.1
226
    /// `checkpoint-index`, oxygen#137), which the client then asks for in
227
    /// parts with `objects`. A remote that cannot answer it sends the
228
    /// whole checkpoint, which does as well.
229
    fn checkpoint_index(&self, heads: &[ChangeHash]) -> Result<Bundle> {
230
        self.checkpoint(heads)
231
    }
2863456a Andy McCall ↶ 232
    /// The changes within `depth` dependency steps of `heads` (spec §17).
233
    fn shallow(&self, heads: &[ChangeHash], depth: u32) -> Result<Vec<ChangeHash>>;
132b884b Andy McCall ↶ 234
    /// Push changes, bookmark updates, new tags, and optionally a newer
235
    /// policy. The policy is applied first; a result named `policy` reports
236
    /// it, and each tag reports as `tags/<name>`.
75167672 Andy McCall 237
    fn push(
238
        &self,
239
        bundle: Bundle,
240
        updates: Vec<RefUpdate>,
241
        policy: Option<SignedPolicy>,
242
        tags: Vec<TagRef>,
243
    ) -> Result<Vec<RefResult>>;
5b9c6ad4 Andy McCall ↶ 244
}
245
750251b2 Andy McCall ↶ 246
// ---------------------------------------------------------------- local
247
5b9c6ad4 Andy McCall ↶ 248
/// Another repository on this machine, served in-process.
249
pub struct LocalRemote {
250
    pub repo: Repo,
84b9e9ac Andy McCall ↶ 251
    /// The pushing repository's id, which binds an unbound target on the
252
    /// first push, as `Oxy-Repo-Id` does over HTTP.
253
    pub claimed_id: Option<[u8; 16]>,
5b9c6ad4 Andy McCall ↶ 254
}
255
256
impl LocalRemote {
257
    pub fn open(path: &Path) -> Result<LocalRemote> {
75167672 Andy McCall 258
        Ok(LocalRemote {
259
            repo: Repo::open(path)?,
84b9e9ac Andy McCall ↶ 260
            claimed_id: None,
75167672 Andy McCall 261
        })
5b9c6ad4 Andy McCall ↶ 262
    }
263
}
264
2f0b4b06 Andy McCall ↶ 265
// Bundles a local remote served on this thread, for tests that count
266
// round trips (oxygen#158).
267
#[cfg(test)]
268
thread_local! {
269
    pub(crate) static LOCAL_FETCHES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
270
}
271
5b9c6ad4 Andy McCall ↶ 272
impl Remote for LocalRemote {
273
    fn repo_id(&self) -> Result<[u8; 16]> {
274
        self.repo.id()
275
    }
83e0677b Andy McCall ↶ 276
    fn refs(&self) -> Result<RefsResponse> {
5b9c6ad4 Andy McCall ↶ 277
        self.repo.remote_refs()
278
    }
279
    fn negotiate(&self, have: &[ChangeHash], want: &[ChangeHash]) -> Result<Vec<ChangeHash>> {
280
        self.repo.negotiate(have, want)
281
    }
b3c5a59a Andy McCall ↶ 282
    fn fetch(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle> {
2f0b4b06 Andy McCall ↶ 283
        #[cfg(test)]
284
        LOCAL_FETCHES.with(|n| n.set(n.get() + 1));
2c75672f Andy McCall ↶ 285
        self.repo.bundle_raw_for(changes, max_blob)
b3c5a59a Andy McCall ↶ 286
    }
287
    fn objects(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>> {
288
        self.repo.objects_for(refs)
5b9c6ad4 Andy McCall ↶ 289
    }
2863456a Andy McCall ↶ 290
    fn checkpoint(&self, heads: &[ChangeHash]) -> Result<Bundle> {
291
        self.repo.checkpoint_bundle_for(heads)
80b069fa Andy McCall ↶ 292
    }
293
    fn checkpoint_index(&self, heads: &[ChangeHash]) -> Result<Bundle> {
294
        self.repo.checkpoint_index_for(heads)
2863456a Andy McCall ↶ 295
    }
296
    fn shallow(&self, heads: &[ChangeHash], depth: u32) -> Result<Vec<ChangeHash>> {
297
        self.repo.shallow_changes(heads, depth)
298
    }
75167672 Andy McCall 299
    fn push(
300
        &self,
301
        bundle: Bundle,
302
        updates: Vec<RefUpdate>,
303
        policy: Option<SignedPolicy>,
304
        tags: Vec<TagRef>,
305
    ) -> Result<Vec<RefResult>> {
84b9e9ac Andy McCall ↶ 306
        if self.repo.is_unbound() {
307
            match self.claimed_id {
308
                Some(id) => self.repo.bind_id(id)?,
eae59556 Andy McCall ↶ 309
                None => {
310
                    return Err(RepoError::Msg(
311
                        "the remote is unbound and this client did not say which repository it is"
312
                            .into(),
313
                    ))
314
                }
84b9e9ac Andy McCall ↶ 315
            }
316
        }
132b884b Andy McCall ↶ 317
        self.repo.receive_push(bundle, updates, policy, tags)
5b9c6ad4 Andy McCall ↶ 318
    }
319
}
320
750251b2 Andy McCall ↶ 321
// ---------------------------------------------------------------- http
322
2b5ee371 Andy McCall ↶ 323
/// How to reach a remote: credentials and trust.
324
#[derive(Clone, Debug, Default)]
325
pub struct RemoteOptions {
326
    /// Bearer token for pushes.
327
    pub token: Option<String>,
328
    /// PEM file with extra root certificates to trust for `https://` remotes
329
    /// (self-hosted servers); the system web roots are always trusted.
330
    pub ca: Option<PathBuf>,
75167672 Andy McCall 331
    /// The local repository's id, sent as `Oxy-Repo-Id` so an unbound
332
    /// hosted repository can adopt it on the first push (spec §16.1).
333
    pub repo_id: Option<[u8; 16]>,
9f9e72b9 Andy McCall ↶ 334
    /// Where a remote kept on a store (`dir://`, `s3://`, …) keeps its
335
    /// mirror and what it has seen: the remote's directory under
336
    /// `.oxy/remotes`. None (a clone, before there is a repository) means
337
    /// a temporary directory.
338
    pub cache_dir: Option<PathBuf>,
68a92484 Andy McCall ↶ 339
    /// Other tokens for the same hub, tried in turn when `token` is
569802eb Andy McCall ↶ 340
    /// refused (401, 403 or 404): the credential store's sign-in when
341
    /// `token` is the remote's own (oxygen#155), or those of enclosing
342
    /// checkouts when it came from the credential store (oxygen#112).
68a92484 Andy McCall ↶ 343
    pub fallback_tokens: Vec<String>,
0588de69 Andy McCall ↶ 344
    /// `token` is the remote's own: fall back only when the hub says it is
345
    /// not a token at all (HTTP 401: expired or unknown), never when it is
346
    /// one that may not do this (403, 404), so a narrowly scoped token is
347
    /// not quietly swapped for a wider sign-in (oxygen#155).
348
    pub fallback_on_401_only: bool,
2b5ee371 Andy McCall ↶ 349
}
350
9f9e72b9 Andy McCall ↶ 351
/// Opens a store for a URL it understands, or passes (`None`). Stores that
352
/// need more than the standard library (the cloud ones) are added by the
353
/// program that links them, with [`register_store_opener`].
354
pub type StoreOpener =
355
    fn(&str, &RemoteOptions) -> Option<crate::strata::Result<Box<dyn crate::strata::Store>>>;
356
357
static STORE_OPENERS: Mutex<Vec<StoreOpener>> = Mutex::new(Vec::new());
358
359
pub fn register_store_opener(opener: StoreOpener) {
360
    STORE_OPENERS
361
        .lock()
362
        .unwrap_or_else(|p| p.into_inner())
363
        .push(opener);
364
}
365
366
/// A store for `url`, from `dir://` or any registered opener.
367
pub fn open_store(
368
    url: &str,
369
    options: &RemoteOptions,
370
) -> Option<crate::strata::Result<Box<dyn crate::strata::Store>>> {
371
    if let Some(path) = url.strip_prefix("dir://") {
372
        return Some(
373
            crate::strata::LocalStore::open(path)
374
                .map(|s| Box::new(s) as Box<dyn crate::strata::Store>),
375
        );
376
    }
377
    let openers = STORE_OPENERS.lock().unwrap_or_else(|p| p.into_inner());
378
    openers.iter().find_map(|open| open(url, options))
379
}
380
75167672 Andy McCall 381
/// The request header carrying the client's repository id.
382
pub const REPO_ID_HEADER: &str = "Oxy-Repo-Id";
383
384
/// The id of a hosted repository that has not adopted one yet.
385
pub const UNBOUND_ID: [u8; 16] = [0; 16];
386
ee130dd9 Andy McCall ↶ 387
/// The request header in which a client declares the protocol features it
388
/// understands beyond the 0.5.0-era set: a comma-separated list, which a
389
/// server may tailor its answers to (docs/protocol.md, *Client
390
/// capabilities*). A request without it is a client of that era.
391
pub const FEATURES_HEADER: &str = "Oxy-Features";
392
393
/// The feature of a client that authorizes a tag signed by a short-lived,
394
/// certified key by its certificate at the time the host accepted it
395
/// (oxygen#129). A client without it refuses such a tag, and with it the
396
/// whole fetch, so servers leave those tags out of its `refs` (oxygen#131).
397
pub const FEATURE_CERTIFIED_TAGS: &str = "certified-tags";
398
399
/// The features this client declares in [`FEATURES_HEADER`].
400
pub const CLIENT_FEATURES: &[&str] = &[FEATURE_CERTIFIED_TAGS];
401
402
/// The `User-Agent` every request of this client sends: `oxy/<version>`.
403
pub fn user_agent() -> String {
404
    format!("oxy/{}", env!("CARGO_PKG_VERSION"))
405
}
406
407
/// Whether a [`FEATURES_HEADER`] value (absent: `None`) declares `feature`.
408
/// Entries are comma-separated, compared without surrounding blanks and
409
/// regardless of case; unknown ones are ignored.
410
pub fn declares_feature(header: Option<&str>, feature: &str) -> bool {
411
    header.is_some_and(|h| h.split(',').any(|f| f.trim().eq_ignore_ascii_case(feature)))
412
}
413
750251b2 Andy McCall ↶ 414
/// A remote served by `oxy-server` at `<base>/oxy/v1/...`.
415
pub struct HttpRemote {
416
    base: String,
68a92484 Andy McCall ↶ 417
    /// The token in use, and those still to try when it is refused.
418
    token: Mutex<Option<String>>,
419
    fallbacks: Mutex<Vec<String>>,
0588de69 Andy McCall ↶ 420
    fallback_on_401_only: bool,
75167672 Andy McCall 421
    repo_id: Option<[u8; 16]>,
750251b2 Andy McCall ↶ 422
    agent: ureq::Agent,
423
}
424
2b5ee371 Andy McCall ↶ 425
/// A TLS client configuration trusting the web roots plus `ca`, if given.
426
fn tls_client_config(ca: Option<&Path>) -> Result<Arc<rustls::ClientConfig>> {
427
    let mut roots = rustls::RootCertStore::empty();
428
    roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
429
    if let Some(path) = ca {
75167672 Andy McCall 430
        let pem = std::fs::read(path)
431
            .map_err(|e| RepoError::Msg(format!("reading CA file {}: {e}", path.display())))?;
2b5ee371 Andy McCall ↶ 432
        let certs = rustls_pemfile::certs(&mut pem.as_slice())
433
            .collect::<std::result::Result<Vec<_>, _>>()
434
            .map_err(|e| RepoError::Msg(format!("parsing CA file {}: {e}", path.display())))?;
435
        if certs.is_empty() {
75167672 Andy McCall 436
            return Err(RepoError::Msg(format!(
437
                "CA file {} holds no certificates",
438
                path.display()
439
            )));
2b5ee371 Andy McCall ↶ 440
        }
441
        for c in certs {
75167672 Andy McCall 442
            roots
443
                .add(c)
444
                .map_err(|e| RepoError::Msg(format!("CA file {}: {e}", path.display())))?;
2b5ee371 Andy McCall ↶ 445
        }
446
    }
75167672 Andy McCall 447
    let cfg = rustls::ClientConfig::builder_with_provider(Arc::new(
448
        rustls::crypto::ring::default_provider(),
449
    ))
450
    .with_safe_default_protocol_versions()
451
    .map_err(|e| RepoError::Msg(e.to_string()))?
452
    .with_root_certificates(roots)
453
    .with_no_client_auth();
2b5ee371 Andy McCall ↶ 454
    Ok(Arc::new(cfg))
455
}
456
750251b2 Andy McCall ↶ 457
impl HttpRemote {
2b5ee371 Andy McCall ↶ 458
    pub fn new(base: &str, options: &RemoteOptions) -> Result<HttpRemote> {
ff1f62fe Andy McCall ↶ 459
        refuse_plain_token(base, options)?;
75167672 Andy McCall 460
        let agent = ureq::AgentBuilder::new()
461
            .tls_config(tls_client_config(options.ca.as_deref())?)
ee130dd9 Andy McCall ↶ 462
            .user_agent(&user_agent())
75167672 Andy McCall 463
            .build();
464
        Ok(HttpRemote {
465
            base: base.trim_end_matches('/').to_string(),
68a92484 Andy McCall ↶ 466
            token: Mutex::new(options.token.clone()),
467
            fallbacks: Mutex::new(options.fallback_tokens.iter().rev().cloned().collect()),
0588de69 Andy McCall ↶ 468
            fallback_on_401_only: options.fallback_on_401_only,
75167672 Andy McCall 469
            repo_id: options.repo_id,
470
            agent,
471
        })
750251b2 Andy McCall ↶ 472
    }
473
68a92484 Andy McCall ↶ 474
    /// A request, tried again with the next fallback token while the
475
    /// hub refuses the one in use (oxygen#112).
750251b2 Andy McCall ↶ 476
    fn call(&self, post: bool, endpoint: &str, body: Option<&[u8]>) -> Result<Vec<u8>> {
68a92484 Andy McCall ↶ 477
        loop {
478
            let token = self.token.lock().unwrap_or_else(|p| p.into_inner()).clone();
479
            match self.call_once(post, endpoint, body, token.as_deref()) {
0588de69 Andy McCall ↶ 480
                Err(RepoError::Msg(m))
481
                    if token.is_some()
482
                        && refused(&m)
483
                        && (!self.fallback_on_401_only || m.contains(": HTTP 401")) =>
484
                {
68a92484 Andy McCall ↶ 485
                    let next = self
486
                        .fallbacks
487
                        .lock()
488
                        .unwrap_or_else(|p| p.into_inner())
489
                        .pop();
490
                    let Some(next) = next else {
491
                        return Err(RepoError::Msg(m));
492
                    };
493
                    crate::credentials::warn_once(&format!(
569802eb Andy McCall ↶ 494
                        "{}: the token in use was refused; trying another this computer has for the hub \
495
                         (a stale remote.NAME.token goes with `oxy config --unset remote.NAME.token`)",
68a92484 Andy McCall ↶ 496
                        self.base
497
                    ));
498
                    *self.token.lock().unwrap_or_else(|p| p.into_inner()) = Some(next);
499
                }
500
                other => return other,
501
            }
502
        }
503
    }
504
505
    fn call_once(
506
        &self,
507
        post: bool,
508
        endpoint: &str,
509
        body: Option<&[u8]>,
510
        token: Option<&str>,
511
    ) -> Result<Vec<u8>> {
750251b2 Andy McCall ↶ 512
        let url = format!("{}/oxy/v1/{endpoint}", self.base);
75167672 Andy McCall 513
        let mut req = if post {
514
            self.agent.post(&url)
515
        } else {
516
            self.agent.get(&url)
517
        };
ee130dd9 Andy McCall ↶ 518
        req = req
519
            .set("Accept", CONTENT_TYPE)
520
            .set(FEATURES_HEADER, &CLIENT_FEATURES.join(","));
68a92484 Andy McCall ↶ 521
        if let Some(t) = token {
750251b2 Andy McCall ↶ 522
            req = req.set("Authorization", &format!("Bearer {t}"));
523
        }
75167672 Andy McCall 524
        if let Some(id) = &self.repo_id {
525
            req = req.set(REPO_ID_HEADER, &hex::encode(id));
526
        }
750251b2 Andy McCall ↶ 527
        let resp = match body {
528
            Some(b) => req.set("Content-Type", CONTENT_TYPE).send_bytes(b),
529
            None => req.call(),
530
        };
531
        match resp {
532
            Ok(r) => {
533
                let mut out = Vec::new();
75167672 Andy McCall 534
                r.into_reader()
535
                    .read_to_end(&mut out)
536
                    .map_err(|e| RepoError::Msg(format!("{url}: {e}")))?;
750251b2 Andy McCall ↶ 537
                Ok(out)
538
            }
539
            Err(ureq::Error::Status(code, r)) => {
540
                let text = r.into_string().unwrap_or_default();
67a4fe1a Andy McCall ↶ 541
                // The hint comes from the status alone: the server's text
542
                // (a host's reason for refusing a push, several lines
d3b27090 Andy McCall ↶ 543
                // perhaps) is shown as it is, never read for hints. A
544
                // proxy's HTML error page is shown by its title only
545
                // (oxygen#133).
67a4fe1a Andy McCall ↶ 546
                let status = format!("{url}: HTTP {code}");
d3b27090 Andy McCall ↶ 547
                let msg = format!("{status}: {}", plain_error_text(&text));
67a4fe1a Andy McCall ↶ 548
                Err(RepoError::Msg(match hint_for(&status) {
549
                    Some(h) => format!("{msg}\n  hint: {h}"),
550
                    None => msg,
551
                }))
750251b2 Andy McCall ↶ 552
            }
3e5772cc Andy McCall ↶ 553
            // The transport error already names the URL.
554
            Err(e) => Err(RepoError::Msg(with_hint(e.to_string()))),
750251b2 Andy McCall ↶ 555
        }
556
    }
68a92484 Andy McCall ↶ 557
}
558
559
/// Whether a failed request's message says the hub refused its token:
560
/// HTTP 401, 403, or 404 (a hub hides a private repository from a token
561
/// that may not read it).
562
fn refused(message: &str) -> bool {
563
    [": HTTP 401", ": HTTP 403", ": HTTP 404"]
564
        .iter()
565
        .any(|c| message.contains(c))
750251b2 Andy McCall ↶ 566
}
567
ff1f62fe Andy McCall ↶ 568
/// Refuse to send a token where anyone on the way could read it: plain
569
/// `http` to another machine (see [`crate::credentials::token_allowed`]).
570
fn refuse_plain_token(url: &str, options: &RemoteOptions) -> Result<()> {
571
    if options.token.is_some() && !crate::credentials::token_allowed(url) {
572
        return Err(RepoError::Msg(format!(
573
            "{url}: refusing to send a token over plain http; use https"
574
        )));
575
    }
576
    Ok(())
577
}
578
3e5772cc Andy McCall ↶ 579
/// What to do about a failed request, recognised from its message.
580
pub fn hint_for(message: &str) -> Option<&'static str> {
581
    let m = message.to_ascii_lowercase();
582
    let hint = if m.contains("http 401") {
ff1f62fe Andy McCall ↶ 583
        "the hub needs you signed in, or did not accept the token: run `oxy auth login HUB` (pushing needs repo:write)"
3e5772cc Andy McCall ↶ 584
    } else if m.contains("http 403") {
585
        "the token or account lacks permission: pushing needs a token with repo:write and write access to the repository"
d3b27090 Andy McCall ↶ 586
    } else if m.contains("http 413") {
587
        "the request was larger than the server, or a proxy in front of it, takes at once (Cloudflare's limit is 100 MB): push in smaller parts with `oxy config push.batch-bytes 33554432`, then push again"
3e5772cc Andy McCall ↶ 588
    } else if m.contains("http 404") {
ff1f62fe Andy McCall ↶ 589
        "nothing at this URL that you can see: check the owner and repository name; a private repository needs you signed in (`oxy auth login HUB`)"
3e5772cc Andy McCall ↶ 590
    } else if m.contains("unknownissuer")
591
        || m.contains("unknown issuer")
592
        || m.contains("certificate")
593
        || m.contains("invalidcertificate")
594
    {
595
        "the server's certificate is not trusted: for a hub with a self-signed certificate, download it from /.well-known/oxyhub-ca.pem on the hub and run `oxy config remote.<remote>.ca FILE`"
596
    } else if m.contains("connection refused") {
597
        "nothing is listening there: check the URL and port, and that the server is running"
598
    } else if m.contains("dns")
599
        || m.contains("failed to lookup")
600
        || m.contains("name or service not known")
601
    {
602
        "the host name does not resolve: check the URL"
603
    } else if m.contains("timed out") || m.contains("timeout") {
604
        "the server did not answer in time: check the network, or try again"
605
    } else {
606
        return None;
607
    };
608
    Some(hint)
609
}
610
d3b27090 Andy McCall ↶ 611
/// An error response's text as it is, unless it is an HTML page (a proxy's,
612
/// such as Cloudflare's 413): then its title, or a note that it was one.
613
fn plain_error_text(text: &str) -> String {
614
    let t = text.trim();
615
    let lower = t.to_ascii_lowercase();
616
    if !(lower.starts_with("<!doctype html") || lower.starts_with("<html")) {
617
        return t.to_string();
618
    }
619
    let title = lower
620
        .find("<title>")
621
        .and_then(|i| {
622
            lower[i + 7..]
623
                .find("</title>")
624
                .map(|j| t[i + 7..i + 7 + j].trim())
625
        })
626
        .filter(|s| !s.is_empty());
627
    match title {
628
        Some(s) => format!("{s} (an HTML error page, from a proxy in front of the server perhaps)"),
629
        None => "an HTML error page (from a proxy in front of the server perhaps)".into(),
630
    }
631
}
632
3e5772cc Andy McCall ↶ 633
/// `message`, followed by what to do about it when that is known.
634
pub fn with_hint(message: String) -> String {
635
    match hint_for(&message) {
636
        Some(h) => format!("{message}\n  hint: {h}"),
637
        None => message,
ff1f62fe Andy McCall ↶ 638
    }
639
}
640
641
/// A JSON request to a hub API (`GET`, `POST` or `DELETE`), with the same
642
/// TLS trust and token rules as remotes. Returns the status and body, an
643
/// error status included; only transport failures are errors.
644
pub fn http_json(
645
    method: &str,
646
    url: &str,
647
    body: Option<&str>,
648
    options: &RemoteOptions,
649
) -> Result<(u16, String)> {
650
    refuse_plain_token(url, options)?;
651
    let agent = ureq::AgentBuilder::new()
652
        .tls_config(tls_client_config(options.ca.as_deref())?)
653
        .timeout(std::time::Duration::from_secs(60))
ee130dd9 Andy McCall ↶ 654
        .user_agent(&user_agent())
ff1f62fe Andy McCall ↶ 655
        .build();
656
    let mut req = agent.request(method, url).set("Accept", "application/json");
657
    if let Some(t) = &options.token {
658
        req = req.set("Authorization", &format!("Bearer {t}"));
659
    }
660
    let resp = match body {
661
        Some(b) => req.set("Content-Type", "application/json").send_string(b),
662
        None => req.call(),
663
    };
664
    match resp {
665
        Ok(r) => {
666
            let code = r.status();
667
            Ok((code, r.into_string().unwrap_or_default()))
668
        }
669
        Err(ureq::Error::Status(code, r)) => Ok((code, r.into_string().unwrap_or_default())),
670
        Err(e) => Err(RepoError::Msg(with_hint(e.to_string()))),
3e5772cc Andy McCall ↶ 671
    }
672
}
673
d82e4409 Andy McCall ↶ 674
/// An authenticated GET of any URL on a host, with the same TLS trust and
675
/// token as remotes use (for hub APIs: proposals). Returns the body.
676
pub fn http_get(url: &str, options: &RemoteOptions) -> Result<Vec<u8>> {
ff1f62fe Andy McCall ↶ 677
    refuse_plain_token(url, options)?;
d82e4409 Andy McCall ↶ 678
    let agent = ureq::AgentBuilder::new()
679
        .tls_config(tls_client_config(options.ca.as_deref())?)
ee130dd9 Andy McCall ↶ 680
        .user_agent(&user_agent())
d82e4409 Andy McCall ↶ 681
        .build();
682
    let mut req = agent.get(url).set("Accept", "application/json");
683
    if let Some(t) = &options.token {
684
        req = req.set("Authorization", &format!("Bearer {t}"));
685
    }
686
    match req.call() {
687
        Ok(r) => {
688
            let mut out = Vec::new();
689
            r.into_reader()
690
                .read_to_end(&mut out)
691
                .map_err(|e| RepoError::Msg(format!("{url}: {e}")))?;
692
            Ok(out)
693
        }
694
        Err(ureq::Error::Status(code, r)) => {
695
            let text = r.into_string().unwrap_or_default();
3e5772cc Andy McCall ↶ 696
            Err(RepoError::Msg(with_hint(format!(
4213c41d Andy McCall ↶ 697
                "{url}: HTTP {code}: {}",
698
                text.trim()
3e5772cc Andy McCall ↶ 699
            ))))
d82e4409 Andy McCall ↶ 700
        }
3e5772cc Andy McCall ↶ 701
        // The transport error already names the URL.
702
        Err(e) => Err(RepoError::Msg(with_hint(e.to_string()))),
d82e4409 Andy McCall ↶ 703
    }
704
}
705
750251b2 Andy McCall ↶ 706
impl Remote for HttpRemote {
707
    fn repo_id(&self) -> Result<[u8; 16]> {
708
        wire::decode_capabilities(&self.call(false, "capabilities", None)?)
709
    }
83e0677b Andy McCall ↶ 710
    fn refs(&self) -> Result<RefsResponse> {
750251b2 Andy McCall ↶ 711
        Ok(wire::decode_refs(&self.call(false, "refs", None)?)?.1)
712
    }
713
    fn negotiate(&self, have: &[ChangeHash], want: &[ChangeHash]) -> Result<Vec<ChangeHash>> {
75167672 Andy McCall 714
        wire::decode_hashes(&self.call(
715
            true,
716
            "negotiate",
717
            Some(&wire::encode_negotiate(have, want)),
718
        )?)
750251b2 Andy McCall ↶ 719
    }
b3c5a59a Andy McCall ↶ 720
    fn fetch(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle> {
75167672 Andy McCall 721
        wire::decode_bundle(&self.call(
722
            true,
723
            "fetch",
724
            Some(&wire::encode_fetch(changes, max_blob)),
725
        )?)
750251b2 Andy McCall ↶ 726
    }
b3c5a59a Andy McCall ↶ 727
    fn objects(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>> {
75167672 Andy McCall 728
        Ok(
729
            wire::decode_bundle(&self.call(true, "objects", Some(&wire::encode_hashes(refs)))?)?
730
                .objects,
731
        )
b3c5a59a Andy McCall ↶ 732
    }
2863456a Andy McCall ↶ 733
    fn checkpoint(&self, heads: &[ChangeHash]) -> Result<Bundle> {
f3811eb2 Andy McCall ↶ 734
        let body = {
735
            let _t = crate::timing::phase("remote: checkpoint answer");
736
            self.call(true, "checkpoint", Some(&wire::encode_hashes(heads)))?
737
        };
738
        let _t = crate::timing::phase("remote: decode the checkpoint");
80b069fa Andy McCall ↶ 739
        wire::decode_bundle(&body)
740
    }
741
    fn checkpoint_index(&self, heads: &[ChangeHash]) -> Result<Bundle> {
742
        let body = {
743
            let _t = crate::timing::phase("remote: checkpoint index");
744
            match self.call(true, "checkpoint-index", Some(&wire::encode_hashes(heads))) {
745
                Ok(b) => b,
746
                // A server from before checkpoint-index: the whole answer.
747
                Err(RepoError::Msg(m)) if m.contains("HTTP 404") => return self.checkpoint(heads),
748
                Err(e) => return Err(e),
749
            }
750
        };
f3811eb2 Andy McCall ↶ 751
        wire::decode_bundle(&body)
2863456a Andy McCall ↶ 752
    }
753
    fn shallow(&self, heads: &[ChangeHash], depth: u32) -> Result<Vec<ChangeHash>> {
75167672 Andy McCall 754
        wire::decode_hashes(&self.call(
755
            true,
756
            "shallow",
757
            Some(&wire::encode_shallow(heads, depth)),
758
        )?)
2863456a Andy McCall ↶ 759
    }
75167672 Andy McCall 760
    fn push(
761
        &self,
762
        bundle: Bundle,
763
        updates: Vec<RefUpdate>,
764
        policy: Option<SignedPolicy>,
765
        tags: Vec<TagRef>,
766
    ) -> Result<Vec<RefResult>> {
767
        wire::decode_push_response(&self.call(
768
            true,
769
            "push",
770
            Some(&wire::encode_push(
771
                &bundle,
772
                &updates,
773
                policy.as_ref(),
774
                &tags,
775
            )),
776
        )?)
750251b2 Andy McCall ↶ 777
    }
778
}
779
827281ee Andy McCall ↶ 780
/// A remote's URL as it is stored: an `http(s)://` URL as given, a local
781
/// path made absolute (with its `file://` kept), so it still names the same
782
/// repository from inside the clone (oxygen#57).
783
pub fn stored_url(url: &str) -> String {
8d803563 Andy McCall ↶ 784
    // Only a path is made absolute: a plain one, or one behind file:// or
785
    // dir://. Every other URL (a hub, a bucket) is kept as written.
9f9e72b9 Andy McCall ↶ 786
    let (prefix, path) = match (url.strip_prefix("file://"), url.strip_prefix("dir://")) {
787
        (Some(p), _) => ("file://", p),
788
        (_, Some(p)) => ("dir://", p),
8d803563 Andy McCall ↶ 789
        _ if url.contains("://") => return url.to_string(),
9f9e72b9 Andy McCall ↶ 790
        _ => ("", url),
827281ee Andy McCall ↶ 791
    };
792
    match std::path::absolute(path) {
793
        Ok(abs) => format!("{prefix}{}", abs.display()),
794
        Err(_) => url.to_string(),
795
    }
796
}
797
9f9e72b9 Andy McCall ↶ 798
/// Open a remote by URL: a local path (optionally `file://`), `http(s)://`
799
/// (a hub), or a store (`dir://`, and whatever openers are registered).
2b5ee371 Andy McCall ↶ 800
pub fn open_remote(url: &str, options: &RemoteOptions) -> Result<Box<dyn Remote>> {
750251b2 Andy McCall ↶ 801
    if url.starts_with("http://") || url.starts_with("https://") {
2b5ee371 Andy McCall ↶ 802
        return Ok(Box::new(HttpRemote::new(url, options)?));
9f9e72b9 Andy McCall ↶ 803
    }
804
    if let Some(store) = open_store(url, options) {
805
        return crate::repo::strata::open_store_remote(store?, options);
750251b2 Andy McCall ↶ 806
    }
5b9c6ad4 Andy McCall ↶ 807
    let path = url.strip_prefix("file://").unwrap_or(url);
808
    if path.contains("://") {
750251b2 Andy McCall ↶ 809
        return Err(RepoError::Msg(format!("unsupported remote URL {url:?}")));
5b9c6ad4 Andy McCall ↶ 810
    }
84b9e9ac Andy McCall ↶ 811
    let mut local = LocalRemote::open(Path::new(path))?;
812
    local.claimed_id = options.repo_id;
813
    Ok(Box::new(local))
5b9c6ad4 Andy McCall ↶ 814
}
750251b2 Andy McCall ↶ 815
816
// ---------------------------------------------------------------- wire
817
818
/// CBOR encodings of the protocol messages (spec §16.1).
819
pub mod wire {
820
    use super::*;
821
822
    fn schema(msg: &str) -> RepoError {
823
        RepoError::Object(ObjectError::Schema(msg.into()))
824
    }
825
    fn sigs_value(sigs: &[Signature]) -> Value {
826
        Value::Array(sigs.iter().map(|s| s.to_value()).collect())
827
    }
828
    fn sigs_from(v: &Value) -> Result<Vec<Signature>> {
75167672 Andy McCall 829
        Ok(o::array(v, "signatures")?
830
            .iter()
831
            .map(Signature::from_value)
832
            .collect::<std::result::Result<Vec<_>, _>>()?)
750251b2 Andy McCall ↶ 833
    }
834
    fn hashes_value(hs: &[ObjectRef]) -> Value {
835
        Value::Array(hs.iter().map(|h| Value::Bytes(h.0.to_vec())).collect())
836
    }
837
    fn hashes_from(v: &Value) -> Result<Vec<ObjectRef>> {
75167672 Andy McCall 838
        Ok(o::array(v, "hashes")?
839
            .iter()
840
            .map(|h| o::oref(h, "hash"))
841
            .collect::<std::result::Result<Vec<_>, _>>()?)
750251b2 Andy McCall ↶ 842
    }
843
    fn attest_value(sa: &SignedAttest) -> Value {
844
        Value::Array(vec![Value::Bytes(sa.attest.encode()), sigs_value(&sa.sigs)])
845
    }
846
    fn attest_from(v: &Value) -> Result<SignedAttest> {
847
        let a = o::array(v, "signed attest")?;
848
        if a.len() != 2 {
849
            return Err(schema("signed attest: expected [attest, sigs]"));
850
        }
75167672 Andy McCall 851
        Ok(SignedAttest {
852
            attest: Attest::decode(o::bytes(&a[0], "attest")?)?,
853
            sigs: sigs_from(&a[1])?,
854
        })
750251b2 Andy McCall ↶ 855
    }
856
    fn opt_attest_value(sa: &Option<SignedAttest>) -> Value {
857
        sa.as_ref().map(attest_value).unwrap_or(Value::Null)
858
    }
859
    fn opt_attest_from(v: &Value) -> Result<Option<SignedAttest>> {
75167672 Andy McCall 860
        if v.is_null() {
861
            Ok(None)
862
        } else {
863
            attest_from(v).map(Some)
864
        }
750251b2 Andy McCall ↶ 865
    }
28fc23d7 Andy McCall ↶ 866
    /// `[policy bytes, [signature]]`: also the format of a policy proposal
867
    /// file exchanged between root holders (spec §14.5).
868
    pub fn policy_value(sp: &SignedPolicy) -> Value {
869
        Value::Array(vec![Value::Bytes(sp.policy.encode()), sigs_value(&sp.sigs)])
83e0677b Andy McCall ↶ 870
    }
28fc23d7 Andy McCall ↶ 871
    pub fn policy_from(v: &Value) -> Result<SignedPolicy> {
83e0677b Andy McCall ↶ 872
        let a = o::array(v, "signed policy")?;
873
        if a.len() != 2 {
874
            return Err(schema("signed policy: expected [policy, sigs]"));
875
        }
75167672 Andy McCall 876
        Ok(SignedPolicy {
877
            policy: Policy::decode(o::bytes(&a[0], "policy")?)?,
878
            sigs: sigs_from(&a[1])?,
879
        })
28fc23d7 Andy McCall ↶ 880
    }
881
    fn opt_policy_value(sp: Option<&SignedPolicy>) -> Value {
882
        sp.map(policy_value).unwrap_or(Value::Null)
883
    }
884
    fn opt_policy_from(v: &Value) -> Result<Option<SignedPolicy>> {
75167672 Andy McCall 885
        if v.is_null() {
886
            Ok(None)
887
        } else {
888
            policy_from(v).map(Some)
889
        }
83e0677b Andy McCall ↶ 890
    }
750251b2 Andy McCall ↶ 891
    fn map<'a>(v: &'a Value, what: &str) -> Result<&'a o::Map> {
892
        Ok(o::map(v, what)?)
893
    }
894
    fn field<'a>(m: &'a o::Map, k: u64, what: &str) -> Result<&'a Value> {
895
        Ok(o::field(m, k, what)?)
896
    }
897
6f8aeb99 Andy McCall ↶ 898
    /// A bundle's value. Each change's text blobs and their chunks (spec
899
    /// §6.1) are added to its objects when they are not already there, so a
900
    /// bundle can always be read on its own.
750251b2 Andy McCall ↶ 901
    pub fn bundle_value(b: &Bundle) -> Value {
2c75672f Andy McCall ↶ 902
        let mut changes = Vec::with_capacity(b.raw_changes.len() + b.changes.len());
903
        // Changes as stored are already their encoding, their texts among
904
        // the objects.
905
        for (bytes, s) in &b.raw_changes {
906
            changes.push(Value::Array(vec![
907
                Value::Bytes(bytes.clone()),
908
                sigs_value(s),
909
            ]));
910
        }
6f8aeb99 Andy McCall ↶ 911
        let mut texts: Vec<(ObjectType, Vec<u8>)> = Vec::new();
912
        let mut listed: Option<std::collections::HashSet<ObjectRef>> = None;
913
        for (c, s) in &b.changes {
914
            let (bytes, stored) = c.encode_parts();
915
            for t in stored {
916
                let listed = listed.get_or_insert_with(|| {
917
                    b.objects
918
                        .iter()
919
                        .map(|(ty, body)| crate::hash::hash_object(*ty, body))
920
                        .collect()
921
                });
922
                if !listed.insert(t.blob.hash()) {
923
                    continue;
924
                }
925
                for chunk in crate::chunk::chunks(&t.content) {
926
                    if listed.insert(crate::hash::hash_object(ObjectType::Chunk, chunk)) {
927
                        texts.push((ObjectType::Chunk, chunk.to_vec()));
928
                    }
929
                }
930
                texts.push((ObjectType::Blob, t.blob.encode()));
931
            }
932
            changes.push(Value::Array(vec![Value::Bytes(bytes), sigs_value(s)]));
933
        }
f0a517d2 Andy McCall ↶ 934
        let mut v = Value::map([
6f8aeb99 Andy McCall ↶ 935
            (0, Value::Array(changes)),
750251b2 Andy McCall ↶ 936
            (
75167672 Andy McCall 937
                1,
938
                Value::Array(b.attests.iter().map(attest_value).collect()),
939
            ),
940
            (
750251b2 Andy McCall ↶ 941
                2,
942
                Value::Array(
75167672 Andy McCall 943
                    b.objects
944
                        .iter()
6f8aeb99 Andy McCall ↶ 945
                        .chain(&texts)
75167672 Andy McCall 946
                        .map(|(t, body)| {
947
                            Value::Array(vec![
948
                                Value::Text(t.name().into()),
949
                                Value::Bytes(body.clone()),
950
                            ])
951
                        })
952
                        .collect(),
750251b2 Andy McCall ↶ 953
                ),
954
            ),
75167672 Andy McCall 955
            (
956
                3,
957
                Value::Array(b.policies.iter().map(policy_value).collect()),
958
            ),
f0a517d2 Andy McCall ↶ 959
        ]);
960
        // Only when there are any; older peers ignore the field.
961
        if !b.certificates.is_empty() {
962
            if let Value::Map(m) = &mut v {
963
                m.insert(
964
                    4,
965
                    Value::Array(
966
                        b.certificates
967
                            .iter()
968
                            .map(|c| Value::Bytes(c.encode()))
969
                            .collect(),
970
                    ),
971
                );
972
            }
973
        }
974
        v
750251b2 Andy McCall ↶ 975
    }
976
977
    pub fn bundle_from_value(v: &Value) -> Result<Bundle> {
978
        let m = map(v, "bundle")?;
979
        let mut b = Bundle::default();
980
        for ob in o::array(field(m, 2, "objects")?, "objects")? {
981
            let pair = o::array(ob, "object")?;
982
            if pair.len() != 2 {
983
                return Err(schema("object: expected [type, body]"));
984
            }
75167672 Andy McCall 985
            let ty = ObjectType::from_name(o::text(&pair[0], "object type")?)
986
                .ok_or_else(|| schema("unknown object type"))?;
987
            b.objects
988
                .push((ty, o::bytes(&pair[1], "object body")?.to_vec()));
6f8aeb99 Andy McCall ↶ 989
        }
990
        // Stored inserts are read from the bundle's own objects.
991
        let mut by_ref: Option<std::collections::HashMap<ObjectRef, usize>> = None;
992
        for c in o::array(field(m, 0, "changes")?, "changes")? {
993
            let pair = o::array(c, "change entry")?;
994
            if pair.len() != 2 {
995
                return Err(schema("change entry: expected [change, sigs]"));
996
            }
598009a7 Andy McCall ↶ 997
            let encoded = o::bytes(&pair[0], "change")?;
998
            b.hashes
999
                .push(crate::hash::hash_object(ObjectType::Change, encoded));
0f93f49c Andy McCall ↶ 1000
            let change = Change::decode_with(encoded, &mut |text, count, form| {
6f8aeb99 Andy McCall ↶ 1001
                let by_ref = by_ref.get_or_insert_with(|| {
1002
                    b.objects
1003
                        .iter()
1004
                        .enumerate()
1005
                        .map(|(i, (ty, body))| (crate::hash::hash_object(*ty, body), i))
1006
                        .collect()
1007
                });
1008
                let get =
1009
                    |r: &ObjectRef, want: ObjectType| match by_ref.get(r).map(|i| &b.objects[*i]) {
1010
                        Some((ty, body)) if *ty == want => Ok(body),
1011
                        _ => Err(ObjectError::Schema(format!(
1012
                            "the bundle lacks {} {}",
1013
                            want.name(),
1014
                            r.short(12)
1015
                        ))),
1016
                    };
1017
                let blob = crate::objects::Blob::decode(get(text, ObjectType::Blob)?)?;
1018
                let mut content = Vec::with_capacity(blob.length as usize);
1019
                for chunk in &blob.chunks {
1020
                    content.extend_from_slice(get(chunk, ObjectType::Chunk)?);
1021
                }
0f93f49c Andy McCall ↶ 1022
                crate::objects::text_lines(&content, count, form)
6f8aeb99 Andy McCall ↶ 1023
            })?;
1024
            b.changes.push((change, sigs_from(&pair[1])?));
1025
        }
1026
        for a in o::array(field(m, 1, "attests")?, "attests")? {
1027
            b.attests.push(attest_from(a)?);
750251b2 Andy McCall ↶ 1028
        }
28fc23d7 Andy McCall ↶ 1029
        if let Some(ps) = m.get(&3) {
1030
            for p in o::array(ps, "policies")? {
1031
                b.policies.push(policy_from(p)?);
f0a517d2 Andy McCall ↶ 1032
            }
1033
        }
1034
        if let Some(cs) = m.get(&4) {
1035
            for c in o::array(cs, "certificates")? {
1036
                b.certificates.push(
1037
                    crate::cert::SignedCertificate::decode(o::bytes(c, "certificate")?)
1038
                        .map_err(|e| schema(&format!("certificate: {e}")))?,
1039
                );
28fc23d7 Andy McCall ↶ 1040
            }
1041
        }
750251b2 Andy McCall ↶ 1042
        Ok(b)
1043
    }
1044
1045
    pub fn encode_bundle(b: &Bundle) -> Vec<u8> {
1046
        cbor::encode(&bundle_value(b))
1047
    }
1048
    pub fn decode_bundle(bytes: &[u8]) -> Result<Bundle> {
1049
        bundle_from_value(&cbor::decode(bytes).map_err(ObjectError::from)?)
1050
    }
1051
1052
    pub fn encode_capabilities(id: [u8; 16]) -> Vec<u8> {
1053
        cbor::encode(&Value::map([
1054
            (0, Value::Array(vec![Value::Uint(PROTOCOL_VERSION)])),
1055
            (1, Value::Bytes(id.to_vec())),
1056
            (2, Value::Array(vec![Value::Text("seal".into())])),
1057
        ]))
1058
    }
1059
    pub fn decode_capabilities(bytes: &[u8]) -> Result<[u8; 16]> {
1060
        let v = cbor::decode(bytes).map_err(ObjectError::from)?;
1061
        let m = map(&v, "capabilities")?;
1062
        let versions = o::array(field(m, 0, "versions")?, "versions")?;
75167672 Andy McCall 1063
        if !versions
1064
            .iter()
1065
            .any(|x| x.as_uint() == Some(PROTOCOL_VERSION))
1066
        {
1067
            return Err(RepoError::Msg(
1068
                "remote does not speak protocol version 1".into(),
1069
            ));
750251b2 Andy McCall ↶ 1070
        }
75167672 Andy McCall 1071
        o::bytes(field(m, 1, "repo id")?, "repo id")?
1072
            .try_into()
1073
            .map_err(|_| schema("repo id: expected 16 bytes"))
750251b2 Andy McCall ↶ 1074
    }
1075
132b884b Andy McCall ↶ 1076
    fn tags_value(tags: &[TagRef]) -> Value {
1077
        Value::Array(
1078
            tags.iter()
1079
                .map(|t| {
1ec6fdbc Andy McCall ↶ 1080
                    let mut v = Value::map([
132b884b Andy McCall ↶ 1081
                        (0, Value::Text(t.name.clone())),
1082
                        (1, Value::Bytes(t.state.0.to_vec())),
1083
                        (2, hashes_value(&t.heads)),
1084
                        (3, opt_attest_value(&t.attestation)),
1ec6fdbc Andy McCall ↶ 1085
                    ]);
1086
                    // Only when there is one; older peers ignore them
1087
                    // (oxygen#129).
1088
                    if let Value::Map(m) = &mut v {
1089
                        if t.accepted.is_some() {
1090
                            m.insert(4, opt_attest_value(&t.accepted));
1091
                        }
1092
                        if let Some(l) = &t.logged {
1093
                            m.insert(5, logged_value(l));
1094
                        }
1095
                        if !t.integration.is_empty() {
1096
                            m.insert(6, integration_value(&t.integration));
1097
                        }
1098
                    }
1099
                    v
132b884b Andy McCall ↶ 1100
                })
1101
                .collect(),
1102
        )
1103
    }
1104
    fn tags_from(v: &Value) -> Result<Vec<TagRef>> {
1105
        let mut out = Vec::new();
1106
        for t in o::array(v, "tags")? {
1107
            let tm = map(t, "tag")?;
1108
            out.push(TagRef {
1109
                name: o::text(field(tm, 0, "name")?, "name")?.to_owned(),
1110
                state: o::oref(field(tm, 1, "state")?, "state")?,
1111
                heads: hashes_from(field(tm, 2, "heads")?)?,
1112
                attestation: opt_attest_from(field(tm, 3, "attestation")?)?,
1ec6fdbc Andy McCall ↶ 1113
                accepted: match tm.get(&4) {
1114
                    Some(a) => opt_attest_from(a)?,
1115
                    None => None,
1116
                },
1117
                logged: tm.get(&5).map(logged_from).transpose()?,
1118
                integration: match tm.get(&6) {
1119
                    Some(l) => integration_from(l)?,
1120
                    None => Vec::new(),
1121
                },
132b884b Andy McCall ↶ 1122
            });
1ec6fdbc Andy McCall ↶ 1123
        }
1124
        Ok(out)
1125
    }
1126
1127
    /// The tags of a refs answer as oxy 0.5.0 read them (and the runners
1128
    /// built with it), keys 0 to 3 only: what an older reader makes of an
1129
    /// answer with keys 4 to 6.
1130
    #[cfg(test)]
1131
    pub(super) fn tags_as_0_5_read_them(bytes: &[u8]) -> Result<Vec<TagRef>> {
1132
        let v = cbor::decode(bytes).map_err(ObjectError::from)?;
1133
        let m = map(&v, "refs")?;
1134
        let mut out = Vec::new();
1135
        for t in o::array(field(m, 3, "tags")?, "tags")? {
1136
            let tm = map(t, "tag")?;
1137
            out.push(TagRef::new(
1138
                o::text(field(tm, 0, "name")?, "name")?.to_owned(),
1139
                o::oref(field(tm, 1, "state")?, "state")?,
1140
                hashes_from(field(tm, 2, "heads")?)?,
1141
                opt_attest_from(field(tm, 3, "attestation")?)?,
1142
            ));
132b884b Andy McCall ↶ 1143
        }
1144
        Ok(out)
1ec6fdbc Andy McCall ↶ 1145
    }
1146
1147
    /// A log place, `[entry number, entry bytes, [proof hash]]` (spec §14.8).
1148
    fn logged_value(l: &Logged) -> Value {
1149
        Value::Array(vec![
1150
            Value::Uint(l.index),
1151
            Value::Bytes(l.entry.clone()),
1152
            proof_value(&l.proof),
1153
        ])
132b884b Andy McCall ↶ 1154
    }
1ec6fdbc Andy McCall ↶ 1155
    fn logged_from(v: &Value) -> Result<Logged> {
1156
        let a = o::array(v, "logged")?;
1157
        if a.len() != 3 {
1158
            return Err(schema("logged: expected [index, entry, proof]"));
1159
        }
1160
        Ok(Logged {
1161
            index: o::uint(&a[0], "index")?,
1162
            entry: o::bytes(&a[1], "entry")?.to_vec(),
1163
            proof: proof_from(&a[2])?,
1164
        })
1165
    }
132b884b Andy McCall ↶ 1166
1ec6fdbc Andy McCall ↶ 1167
    /// Cosigned checkpoints holding an entry, `[[note, [proof hash]]]`.
1168
    fn integration_value(integration: &[Integration]) -> Value {
1169
        Value::Array(
1170
            integration
1171
                .iter()
1172
                .map(|i| Value::Array(vec![Value::Text(i.note.clone()), proof_value(&i.proof)]))
1173
                .collect(),
1174
        )
1175
    }
1176
    fn integration_from(v: &Value) -> Result<Vec<Integration>> {
1177
        let mut out = Vec::new();
1178
        for i in o::array(v, "integration")? {
1179
            let a = o::array(i, "integration")?;
1180
            if a.len() != 2 {
1181
                return Err(schema("integration: expected [note, proof]"));
1182
            }
1183
            out.push(Integration {
1184
                note: o::text(&a[0], "note")?.to_owned(),
1185
                proof: proof_from(&a[1])?,
1186
            });
1187
        }
1188
        Ok(out)
1189
    }
1190
83e0677b Andy McCall ↶ 1191
    pub fn encode_refs(id: [u8; 16], resp: &RefsResponse) -> Vec<u8> {
4b9d0677 Andy McCall ↶ 1192
        let mut v = Value::map([
750251b2 Andy McCall ↶ 1193
            (
1194
                0,
1195
                Value::Array(
83e0677b Andy McCall ↶ 1196
                    resp.refs
1197
                        .iter()
750251b2 Andy McCall ↶ 1198
                        .map(|r| {
f53cbe7a Andy McCall ↶ 1199
                            let mut v = Value::map([
750251b2 Andy McCall ↶ 1200
                                (0, Value::Text(r.name.clone())),
1201
                                (1, Value::Bytes(r.state.0.to_vec())),
1202
                                (2, hashes_value(&r.heads)),
1203
                                (3, opt_attest_value(&r.attestation)),
1204
                                (4, r.last_serial.map(Value::Uint).unwrap_or(Value::Null)),
f53cbe7a Andy McCall ↶ 1205
                            ]);
1206
                            // Only when there is one; older peers ignore it.
4b9d0677 Andy McCall ↶ 1207
                            if let Value::Map(m) = &mut v {
1208
                                if r.accepted.is_some() {
f53cbe7a Andy McCall ↶ 1209
                                    m.insert(5, opt_attest_value(&r.accepted));
4b9d0677 Andy McCall ↶ 1210
                                }
1211
                                if let Some(l) = &r.logged {
1ec6fdbc Andy McCall ↶ 1212
                                    m.insert(6, logged_value(l));
f53cbe7a Andy McCall ↶ 1213
                                }
028545df Andy McCall ↶ 1214
                                if !r.integration.is_empty() {
1ec6fdbc Andy McCall ↶ 1215
                                    m.insert(7, integration_value(&r.integration));
028545df Andy McCall ↶ 1216
                                }
f53cbe7a Andy McCall ↶ 1217
                            }
1218
                            v
750251b2 Andy McCall ↶ 1219
                        })
1220
                        .collect(),
1221
                ),
1222
            ),
1223
            (1, Value::Bytes(id.to_vec())),
83e0677b Andy McCall ↶ 1224
            (2, opt_policy_value(resp.policy.as_ref())),
132b884b Andy McCall ↶ 1225
            (3, tags_value(&resp.tags)),
75167672 Andy McCall 1226
            (
1227
                4,
1228
                Value::Array(resp.history.iter().map(policy_value).collect()),
1229
            ),
14edbaaf Andy McCall ↶ 1230
            (
1231
                5,
1232
                resp.default_bookmark
1233
                    .clone()
1234
                    .map(Value::Text)
1235
                    .unwrap_or(Value::Null),
1236
            ),
4b9d0677 Andy McCall ↶ 1237
        ]);
028545df Andy McCall ↶ 1238
        if let Value::Map(m) = &mut v {
1239
            if let Some(log) = &resp.log {
1240
                m.insert(6, Value::Text(log.clone()));
b63fc7a7 Andy McCall ↶ 1241
            }
1242
            if !resp.withdrawn.is_empty() {
1243
                m.insert(8, tags_value(&resp.withdrawn));
028545df Andy McCall ↶ 1244
            }
1245
            if !resp.witnesses.is_empty() {
1246
                m.insert(
1247
                    7,
1248
                    Value::Array(
1249
                        resp.witnesses
1250
                            .iter()
1251
                            .map(|(name, id)| {
1252
                                Value::Array(vec![
1253
                                    Value::Text(name.clone()),
1254
                                    Value::Bytes(id.clone()),
1255
                                ])
1256
                            })
1257
                            .collect(),
1258
                    ),
1259
                );
1260
            }
4b9d0677 Andy McCall ↶ 1261
        }
1262
        cbor::encode(&v)
750251b2 Andy McCall ↶ 1263
    }
83e0677b Andy McCall ↶ 1264
    pub fn decode_refs(bytes: &[u8]) -> Result<([u8; 16], RefsResponse)> {
750251b2 Andy McCall ↶ 1265
        let v = cbor::decode(bytes).map_err(ObjectError::from)?;
1266
        let m = map(&v, "refs")?;
75167672 Andy McCall 1267
        let id: [u8; 16] = o::bytes(field(m, 1, "repo id")?, "repo id")?
1268
            .try_into()
1269
            .map_err(|_| schema("repo id: expected 16 bytes"))?;
750251b2 Andy McCall ↶ 1270
        let mut refs = Vec::new();
1271
        for r in o::array(field(m, 0, "refs")?, "refs")? {
1272
            let rm = map(r, "ref")?;
1273
            let ls = field(rm, 4, "last serial")?;
1274
            refs.push(RemoteRef {
1275
                name: o::text(field(rm, 0, "name")?, "name")?.to_owned(),
1276
                state: o::oref(field(rm, 1, "state")?, "state")?,
1277
                heads: hashes_from(field(rm, 2, "heads")?)?,
1278
                attestation: opt_attest_from(field(rm, 3, "attestation")?)?,
75167672 Andy McCall 1279
                last_serial: if ls.is_null() {
1280
                    None
1281
                } else {
1282
                    Some(o::uint(ls, "last serial")?)
f53cbe7a Andy McCall ↶ 1283
                },
1284
                accepted: match rm.get(&5) {
1285
                    Some(a) => opt_attest_from(a)?,
1286
                    None => None,
75167672 Andy McCall 1287
                },
1ec6fdbc Andy McCall ↶ 1288
                logged: rm.get(&6).map(logged_from).transpose()?,
028545df Andy McCall ↶ 1289
                integration: match rm.get(&7) {
1ec6fdbc Andy McCall ↶ 1290
                    Some(l) => integration_from(l)?,
028545df Andy McCall ↶ 1291
                    None => Vec::new(),
1292
                },
750251b2 Andy McCall ↶ 1293
            });
1294
        }
83e0677b Andy McCall ↶ 1295
        let policy = match m.get(&2) {
1296
            Some(p) => opt_policy_from(p)?,
1297
            None => None,
1298
        };
132b884b Andy McCall ↶ 1299
        let tags = match m.get(&3) {
1300
            Some(t) => tags_from(t)?,
1301
            None => Vec::new(),
1302
        };
28fc23d7 Andy McCall ↶ 1303
        let mut history = Vec::new();
1304
        if let Some(h) = m.get(&4) {
1305
            for p in o::array(h, "history")? {
1306
                history.push(policy_from(p)?);
1307
            }
1308
        }
14edbaaf Andy McCall ↶ 1309
        let default_bookmark = match m.get(&5) {
4b9d0677 Andy McCall ↶ 1310
            Some(Value::Text(t)) => Some(t.clone()),
1311
            _ => None,
1312
        };
1313
        let log = match m.get(&6) {
14edbaaf Andy McCall ↶ 1314
            Some(Value::Text(t)) => Some(t.clone()),
1315
            _ => None,
1316
        };
028545df Andy McCall ↶ 1317
        let mut witnesses = Vec::new();
1318
        if let Some(w) = m.get(&7) {
1319
            for x in o::array(w, "witnesses")? {
1320
                let a = o::array(x, "witness")?;
1321
                if a.len() != 2 {
1322
                    return Err(schema("witness: expected [name, identity]"));
1323
                }
1324
                witnesses.push((
1325
                    o::text(&a[0], "witness name")?.to_owned(),
1326
                    o::bytes(&a[1], "witness identity")?.to_vec(),
1327
                ));
1328
            }
1329
        }
b63fc7a7 Andy McCall ↶ 1330
        let withdrawn = match m.get(&8) {
1331
            Some(t) => tags_from(t)?,
1332
            None => Vec::new(),
1333
        };
75167672 Andy McCall 1334
        Ok((
1335
            id,
1336
            RefsResponse {
1337
                refs,
1338
                policy,
1339
                tags,
1340
                history,
14edbaaf Andy McCall ↶ 1341
                default_bookmark,
4b9d0677 Andy McCall ↶ 1342
                log,
028545df Andy McCall ↶ 1343
                witnesses,
b63fc7a7 Andy McCall ↶ 1344
                withdrawn,
75167672 Andy McCall 1345
            },
1346
        ))
750251b2 Andy McCall ↶ 1347
    }
1348
028545df Andy McCall ↶ 1349
    /// A proof's hashes, from the leaf upwards (spec §14.8).
1350
    fn proof_value(proof: &[[u8; 32]]) -> Value {
1351
        Value::Array(proof.iter().map(|h| Value::Bytes(h.to_vec())).collect())
1352
    }
1353
1354
    fn proof_from(v: &Value) -> Result<Vec<[u8; 32]>> {
1355
        let mut proof = Vec::new();
1356
        for h in o::array(v, "proof")? {
1357
            proof.push(
1358
                o::bytes(h, "proof hash")?
1359
                    .try_into()
1360
                    .map_err(|_| schema("proof hash: expected 32 bytes"))?,
1361
            );
1362
        }
1363
        Ok(proof)
1364
    }
1365
750251b2 Andy McCall ↶ 1366
    pub fn encode_negotiate(have: &[ChangeHash], want: &[ChangeHash]) -> Vec<u8> {
75167672 Andy McCall 1367
        cbor::encode(&Value::map([
1368
            (0, hashes_value(have)),
1369
            (1, hashes_value(want)),
1370
        ]))
750251b2 Andy McCall ↶ 1371
    }
1372
    pub fn decode_negotiate(bytes: &[u8]) -> Result<(Vec<ChangeHash>, Vec<ChangeHash>)> {
1373
        let v = cbor::decode(bytes).map_err(ObjectError::from)?;
1374
        let m = map(&v, "negotiate")?;
75167672 Andy McCall 1375
        Ok((
1376
            hashes_from(field(m, 0, "have")?)?,
1377
            hashes_from(field(m, 1, "want")?)?,
1378
        ))
750251b2 Andy McCall ↶ 1379
    }
1380
b3c5a59a Andy McCall ↶ 1381
    /// `fetch` request: `{0: [changes], 1: max blob size or null}`.
1382
    pub fn encode_fetch(hs: &[ChangeHash], max_blob: Option<u64>) -> Vec<u8> {
75167672 Andy McCall 1383
        cbor::encode(&Value::map([
1384
            (0, hashes_value(hs)),
1385
            (1, max_blob.map(Value::Uint).unwrap_or(Value::Null)),
1386
        ]))
b3c5a59a Andy McCall ↶ 1387
    }
1388
    pub fn decode_fetch(bytes: &[u8]) -> Result<(Vec<ChangeHash>, Option<u64>)> {
1389
        let v = cbor::decode(bytes).map_err(ObjectError::from)?;
1390
        let m = map(&v, "fetch")?;
1391
        let max = match m.get(&1) {
1392
            Some(x) if !x.is_null() => Some(o::uint(x, "max blob")?),
1393
            _ => None,
1394
        };
1395
        Ok((hashes_from(field(m, 0, "changes")?)?, max))
1396
    }
1397
2863456a Andy McCall ↶ 1398
    /// `shallow` request: `{0: [heads], 1: depth}`.
1399
    pub fn encode_shallow(hs: &[ChangeHash], depth: u32) -> Vec<u8> {
75167672 Andy McCall 1400
        cbor::encode(&Value::map([
1401
            (0, hashes_value(hs)),
1402
            (1, Value::Uint(depth as u64)),
1403
        ]))
2863456a Andy McCall ↶ 1404
    }
1405
    pub fn decode_shallow(bytes: &[u8]) -> Result<(Vec<ChangeHash>, u32)> {
1406
        let v = cbor::decode(bytes).map_err(ObjectError::from)?;
1407
        let m = map(&v, "shallow")?;
75167672 Andy McCall 1408
        Ok((
1409
            hashes_from(field(m, 0, "heads")?)?,
1410
            o::u32_(field(m, 1, "depth")?, "depth")?,
1411
        ))
2863456a Andy McCall ↶ 1412
    }
1413
750251b2 Andy McCall ↶ 1414
    pub fn encode_hashes(hs: &[ChangeHash]) -> Vec<u8> {
1415
        cbor::encode(&Value::map([(0, hashes_value(hs))]))
1416
    }
1417
    pub fn decode_hashes(bytes: &[u8]) -> Result<Vec<ChangeHash>> {
1418
        let v = cbor::decode(bytes).map_err(ObjectError::from)?;
1419
        hashes_from(field(map(&v, "hashes")?, 0, "hashes")?)
1420
    }
1421
75167672 Andy McCall 1422
    pub fn encode_push(
1423
        bundle: &Bundle,
1424
        updates: &[RefUpdate],
1425
        policy: Option<&SignedPolicy>,
1426
        tags: &[TagRef],
1427
    ) -> Vec<u8> {
750251b2 Andy McCall ↶ 1428
        cbor::encode(&Value::map([
1429
            (0, bundle_value(bundle)),
83e0677b Andy McCall ↶ 1430
            (2, opt_policy_value(policy)),
132b884b Andy McCall ↶ 1431
            (3, tags_value(tags)),
750251b2 Andy McCall ↶ 1432
            (
1433
                1,
1434
                Value::Array(
1435
                    updates
1436
                        .iter()
1437
                        .map(|u| {
1438
                            Value::map([
1439
                                (0, Value::Text(u.name.clone())),
1440
                                (1, Value::Bytes(u.old_state.0.to_vec())),
1441
                                (2, Value::Bytes(u.new_state.0.to_vec())),
1442
                                (3, Value::Bool(u.force)),
1443
                                (4, opt_attest_value(&u.attestation)),
1444
                            ])
1445
                        })
1446
                        .collect(),
1447
                ),
1448
            ),
1449
        ]))
1450
    }
132b884b Andy McCall ↶ 1451
    /// A decoded push: bundle, bookmark updates, policy, tags.
1452
    pub type PushParts = (Bundle, Vec<RefUpdate>, Option<SignedPolicy>, Vec<TagRef>);
1453
1454
    pub fn decode_push(bytes: &[u8]) -> Result<PushParts> {
750251b2 Andy McCall ↶ 1455
        let v = cbor::decode(bytes).map_err(ObjectError::from)?;
1456
        let m = map(&v, "push")?;
1457
        let bundle = bundle_from_value(field(m, 0, "bundle")?)?;
1458
        let mut updates = Vec::new();
1459
        for u in o::array(field(m, 1, "updates")?, "updates")? {
1460
            let um = map(u, "update")?;
1461
            updates.push(RefUpdate {
1462
                name: o::text(field(um, 0, "name")?, "name")?.to_owned(),
1463
                old_state: o::oref(field(um, 1, "old_state")?, "old_state")?,
1464
                new_state: o::oref(field(um, 2, "new_state")?, "new_state")?,
75167672 Andy McCall 1465
                force: field(um, 3, "force")?
1466
                    .as_bool()
1467
                    .ok_or_else(|| schema("force: expected bool"))?,
750251b2 Andy McCall ↶ 1468
                attestation: opt_attest_from(field(um, 4, "attestation")?)?,
1469
            });
1470
        }
83e0677b Andy McCall ↶ 1471
        let policy = match m.get(&2) {
1472
            Some(p) => opt_policy_from(p)?,
1473
            None => None,
1474
        };
132b884b Andy McCall ↶ 1475
        let tags = match m.get(&3) {
1476
            Some(t) => tags_from(t)?,
1477
            None => Vec::new(),
1478
        };
1479
        Ok((bundle, updates, policy, tags))
750251b2 Andy McCall ↶ 1480
    }
1481
1482
    pub fn encode_push_response(results: &[RefResult]) -> Vec<u8> {
1483
        cbor::encode(&Value::map([(
1484
            0,
1485
            Value::Array(
1486
                results
1487
                    .iter()
75167672 Andy McCall 1488
                    .map(|r| {
1489
                        Value::map([
1490
                            (0, Value::Text(r.name.clone())),
1491
                            (1, Value::Bool(r.ok)),
1492
                            (2, Value::Text(r.reason.clone())),
1493
                        ])
1494
                    })
750251b2 Andy McCall ↶ 1495
                    .collect(),
1496
            ),
1497
        )]))
1498
    }
1499
    pub fn decode_push_response(bytes: &[u8]) -> Result<Vec<RefResult>> {
1500
        let v = cbor::decode(bytes).map_err(ObjectError::from)?;
1501
        let m = map(&v, "push response")?;
1502
        let mut out = Vec::new();
1503
        for r in o::array(field(m, 0, "results")?, "results")? {
1504
            let rm = map(r, "result")?;
1505
            out.push(RefResult {
1506
                name: o::text(field(rm, 0, "name")?, "name")?.to_owned(),
75167672 Andy McCall 1507
                ok: field(rm, 1, "ok")?
1508
                    .as_bool()
1509
                    .ok_or_else(|| schema("ok: expected bool"))?,
750251b2 Andy McCall ↶ 1510
                reason: o::text(field(rm, 2, "reason")?, "reason")?.to_owned(),
1511
            });
1512
        }
1513
        Ok(out)
1514
    }
1515
}
1516
1517
#[cfg(test)]
1518
mod tests {
1519
    use super::*;
1520
    use crate::attest::AttestBody;
1521
    use crate::objects::Hlc;
d3b27090 Andy McCall ↶ 1522
1523
    /// oxygen#133: a proxy's HTML error page is shown by its title, and a
1524
    /// server's own text as it is; a 413 says to push in smaller parts.
1525
    #[test]
1526
    fn html_error_pages_are_shown_by_their_title() {
1527
        let cloudflare = "<html>\r\n<head><title>413 Request Entity Too Large</title></head>\r\n<body><center><h1>413</h1></center><hr><center>cloudflare</center></body></html>";
1528
        assert_eq!(
1529
            plain_error_text(cloudflare),
1530
            "413 Request Entity Too Large (an HTML error page, from a proxy in front of the server perhaps)"
1531
        );
1532
        assert_eq!(
1533
            plain_error_text("  refused: a secret\nat a.txt:3 "),
1534
            "refused: a secret\nat a.txt:3"
1535
        );
1536
        assert!(hint_for("https://hub/x: HTTP 413: too large")
1537
            .unwrap()
1538
            .contains("push.batch-bytes"));
1539
    }
ee130dd9 Andy McCall ↶ 1540
1541
    #[test]
1542
    fn client_features_are_declared_and_read() {
1543
        assert_eq!(CLIENT_FEATURES.join(","), "certified-tags");
1544
        assert!(user_agent().starts_with("oxy/"));
1545
        assert!(declares_feature(
1546
            Some("certified-tags"),
1547
            FEATURE_CERTIFIED_TAGS
1548
        ));
1549
        assert!(declares_feature(
1550
            Some("later-thing, Certified-Tags ,x"),
1551
            FEATURE_CERTIFIED_TAGS
1552
        ));
1553
        assert!(!declares_feature(
1554
            Some("certified-tags-v2"),
1555
            FEATURE_CERTIFIED_TAGS
1556
        ));
1557
        assert!(!declares_feature(Some(""), FEATURE_CERTIFIED_TAGS));
1558
        assert!(!declares_feature(None, FEATURE_CERTIFIED_TAGS));
1559
    }
750251b2 Andy McCall ↶ 1560
1561
    #[test]
1562
    fn wire_round_trips() {
1563
        let c = crate::objects::tests::sample_change();
75167672 Andy McCall 1564
        let sk = crate::identity::SecretKey {
1565
            ed_seed: [1; 32],
1566
            ml_seed: [2; 32],
1567
        };
750251b2 Andy McCall ↶ 1568
        let id = sk.identity("a", "b", 0);
1569
        let key = id.hash();
1570
        let sig = sk.sign(crate::identity::Domain::Change, &key, &c.hash());
1571
        let seal = Attest::seal([3; 16], vec![c.hash()]);
1572
        let seal_sig = sk.sign(crate::identity::Domain::Attest, &key, &seal.hash());
28fc23d7 Andy McCall ↶ 1573
        let policy = Policy::single_root([3; 16], id.clone(), 99);
1574
        let psig = sk.sign(crate::identity::Domain::Policy, &key, &policy.hash());
750251b2 Andy McCall ↶ 1575
        let bundle = Bundle {
1576
            changes: vec![(c.clone(), vec![sig.clone()])],
75167672 Andy McCall 1577
            attests: vec![SignedAttest {
1578
                attest: seal.clone(),
1579
                sigs: vec![seal_sig],
1580
            }],
1581
            objects: vec![
1582
                (ObjectType::Identity, id.encode()),
1583
                (ObjectType::Chunk, b"xyz".to_vec()),
1584
            ],
1585
            policies: vec![SignedPolicy {
1586
                policy: policy.clone(),
1587
                sigs: vec![psig.clone()],
1588
            }],
598009a7 Andy McCall ↶ 1589
            hashes: vec![],
2c75672f Andy McCall ↶ 1590
            raw_changes: vec![],
f0a517d2 Andy McCall ↶ 1591
            certificates: vec![],
750251b2 Andy McCall ↶ 1592
        };
1593
        let d = wire::decode_bundle(&wire::encode_bundle(&bundle)).unwrap();
598009a7 Andy McCall ↶ 1594
        assert_eq!(
1595
            d.hashes,
1596
            vec![c.hash()],
1597
            "the receiver hashes what it was sent"
1598
        );
750251b2 Andy McCall ↶ 1599
        assert_eq!(d.changes, bundle.changes);
1600
        assert_eq!(d.attests, bundle.attests);
1601
        assert_eq!(d.objects, bundle.objects);
28fc23d7 Andy McCall ↶ 1602
        assert_eq!(d.policies, bundle.policies);
f0a517d2 Andy McCall ↶ 1603
        assert!(d.certificates.is_empty());
1604
        // Certificates (keyless, oxygenhub#241) travel as field 4, only
1605
        // when there are any: a bundle without them encodes as before.
1606
        let plain = wire::encode_bundle(&bundle);
1607
        let mut with = bundle.clone();
1608
        let issuer_sk = crate::identity::SecretKey::generate();
1609
        let issuer = issuer_sk.identity("hub", "h@x", 1);
1610
        with.certificates.push(crate::cert::SignedCertificate::sign(
1611
            crate::cert::Certificate {
1612
                key: id.clone(),
1613
                issuer: issuer.hash(),
1614
                subject: "agent:a@b".into(),
1615
                claims: vec![],
1616
                scope: crate::identity::Scope::CHANGES,
1617
                not_before: 1,
1618
                expires: 2,
a2387d54 Andy McCall ↶ 1619
                grant: None,
f0a517d2 Andy McCall ↶ 1620
            },
1621
            &issuer,
1622
            &issuer_sk,
1623
        ));
1624
        let encoded = wire::encode_bundle(&with);
1625
        assert_ne!(encoded, plain);
1626
        assert_eq!(
1627
            wire::decode_bundle(&encoded).unwrap().certificates,
1628
            with.certificates
1629
        );
1630
        with.certificates.clear();
1631
        assert_eq!(wire::encode_bundle(&with), plain);
750251b2 Andy McCall ↶ 1632
75167672 Andy McCall 1633
        let att = Attest {
1634
            repo: [3; 16],
1635
            hlc: Hlc::default(),
1636
            body: AttestBody::Ref {
1637
                name: "main".into(),
1638
                old_state: ObjectRef([1; 32]),
1639
                new_state: ObjectRef([2; 32]),
1640
                serial: 4,
1641
            },
1642
        };
750251b2 Andy McCall ↶ 1643
        let refs = vec![
75167672 Andy McCall 1644
            RemoteRef {
1645
                name: "main".into(),
1646
                state: ObjectRef([2; 32]),
1647
                heads: vec![c.hash()],
1648
                attestation: Some(SignedAttest {
1649
                    attest: att.clone(),
1650
                    sigs: vec![sig.clone()],
1651
                }),
1652
                last_serial: Some(4),
f53cbe7a Andy McCall ↶ 1653
                accepted: None,
4b9d0677 Andy McCall ↶ 1654
                logged: None,
028545df Andy McCall ↶ 1655
                integration: vec![],
75167672 Andy McCall 1656
            },
1657
            RemoteRef {
1658
                name: "dev".into(),
1659
                state: ObjectRef([5; 32]),
1660
                heads: vec![],
1661
                attestation: None,
1662
                last_serial: None,
f53cbe7a Andy McCall ↶ 1663
                accepted: None,
4b9d0677 Andy McCall ↶ 1664
                logged: None,
028545df Andy McCall ↶ 1665
                integration: vec![],
75167672 Andy McCall 1666
            },
750251b2 Andy McCall ↶ 1667
        ];
1ec6fdbc Andy McCall ↶ 1668
        let tag = TagRef::new("v1".into(), ObjectRef([2; 32]), vec![c.hash()], None);
4b9d0677 Andy McCall ↶ 1669
        let mut resp = RefsResponse {
75167672 Andy McCall 1670
            refs,
1671
            policy: Some(SignedPolicy {
1672
                policy: policy.clone(),
1673
                sigs: vec![psig.clone()],
1674
            }),
1675
            tags: vec![tag.clone()],
1676
            history: vec![SignedPolicy {
1677
                policy: policy.clone(),
1678
                sigs: vec![],
1679
            }],
14edbaaf Andy McCall ↶ 1680
            default_bookmark: Some("develop".into()),
4b9d0677 Andy McCall ↶ 1681
            log: Some("hub.example/log\n2\nroot\n\n- sig\n".into()),
028545df Andy McCall ↶ 1682
            witnesses: vec![],
b63fc7a7 Andy McCall ↶ 1683
            withdrawn: vec![],
75167672 Andy McCall 1684
        };
028545df Andy McCall ↶ 1685
        // Without integrated times (spec §14.8) nothing is added: an old
1686
        // peer's answer, and what it reads.
1687
        let old = wire::encode_refs([3; 16], &resp);
1688
        let back = wire::decode_refs(&old).unwrap().1;
1689
        assert!(back.witnesses.is_empty() && back.refs.iter().all(|r| r.integration.is_empty()));
4b9d0677 Andy McCall ↶ 1690
        resp.refs[1].logged = Some(Logged {
1691
            index: 41,
1692
            entry: vec![1, 2, 3],
1693
            proof: vec![[9; 32], [8; 32]],
1694
        });
028545df Andy McCall ↶ 1695
        resp.refs[1].integration = vec![Integration {
1696
            note: "hub.example/log\n42\nroot\n\n- sig\n".into(),
1697
            proof: vec![[7; 32]],
1698
        }];
1699
        resp.witnesses = vec![("w.example".into(), vec![4, 5, 6])];
1700
        let new = wire::encode_refs([3; 16], &resp);
1701
        let (id_back, back) = wire::decode_refs(&new).unwrap();
4b9d0677 Andy McCall ↶ 1702
        assert_eq!(back.log, resp.log);
1703
        assert_eq!(back.refs[1].logged, resp.refs[1].logged);
028545df Andy McCall ↶ 1704
        assert_eq!(back.refs[1].integration, resp.refs[1].integration);
1705
        assert_eq!(back.witnesses, resp.witnesses);
4b9d0677 Andy McCall ↶ 1706
        assert_eq!(back.refs[0].logged, None);
028545df Andy McCall ↶ 1707
        // The new keys are 7s an old reader never asks for.
1708
        let crate::cbor::Value::Map(m) = crate::cbor::decode(&new).unwrap() else {
1709
            panic!()
1710
        };
1711
        assert!(m.contains_key(&7));
750251b2 Andy McCall ↶ 1712
        assert_eq!(id_back, [3; 16]);
83e0677b Andy McCall ↶ 1713
        assert_eq!(back.refs.len(), 2);
1714
        assert_eq!(back.refs[0].attestation.as_ref().unwrap().attest, att);
1715
        assert_eq!(back.refs[1].last_serial, None);
1716
        assert_eq!(back.policy.as_ref().unwrap().policy, policy);
28fc23d7 Andy McCall ↶ 1717
        assert_eq!(back.history, resp.history);
132b884b Andy McCall ↶ 1718
        assert_eq!(back.tags, vec![tag.clone()]);
1ec6fdbc Andy McCall ↶ 1719
1720
        // A tag's acceptance and log place (oxygen#129): keys 4 to 6, only
1721
        // when there are any, so a tag without them encodes as before.
1722
        let crate::cbor::Value::Map(m) = crate::cbor::decode(&new).unwrap() else {
1723
            panic!()
1724
        };
1725
        let crate::cbor::Value::Array(tags) = &m[&3] else {
1726
            panic!()
1727
        };
1728
        let crate::cbor::Value::Map(plain) = &tags[0] else {
1729
            panic!()
1730
        };
1731
        assert_eq!(plain.keys().copied().collect::<Vec<_>>(), vec![0, 1, 2, 3]);
1732
        let mut full = tag.clone();
1733
        full.name = "v2".into();
1734
        full.attestation = Some(SignedAttest {
1735
            attest: att.clone(),
1736
            sigs: vec![sig.clone()],
1737
        });
1738
        full.accepted = full.attestation.clone();
1739
        full.logged = Some(Logged {
1740
            index: 7,
1741
            entry: vec![4],
1742
            proof: vec![[3; 32]],
1743
        });
1744
        full.integration = vec![Integration {
1745
            note: "hub.example/log\n8\nroot\n\n- sig\n".into(),
1746
            proof: vec![],
1747
        }];
1748
        resp.tags = vec![full.clone(), tag.clone()];
1749
        let bytes = wire::encode_refs([3; 16], &resp);
1750
        assert_eq!(wire::decode_refs(&bytes).unwrap().1.tags, resp.tags);
1751
        // In a push too.
1752
        let pushed = wire::encode_push(&Bundle::default(), &[], None, &resp.tags);
1753
        assert_eq!(wire::decode_push(&pushed).unwrap().3, resp.tags);
1754
        // A reader that knows only keys 0 to 3 (oxy 0.5.0 and older, and
1755
        // the runners built with it) reads the same tags from the new answer.
1756
        let stripped: Vec<TagRef> = resp
1757
            .tags
1758
            .iter()
1759
            .map(|t| {
1760
                TagRef::new(
1761
                    t.name.clone(),
1762
                    t.state,
1763
                    t.heads.clone(),
1764
                    t.attestation.clone(),
1765
                )
1766
            })
1767
            .collect();
1768
        assert_eq!(wire::tags_as_0_5_read_them(&bytes).unwrap(), stripped);
750251b2 Andy McCall ↶ 1769
75167672 Andy McCall 1770
        let (h, w) = wire::decode_negotiate(&wire::encode_negotiate(
1771
            &[ObjectRef([1; 32])],
1772
            &[ObjectRef([2; 32])],
1773
        ))
1774
        .unwrap();
750251b2 Andy McCall ↶ 1775
        assert_eq!((h, w), (vec![ObjectRef([1; 32])], vec![ObjectRef([2; 32])]));
75167672 Andy McCall 1776
        assert_eq!(
1777
            wire::decode_hashes(&wire::encode_hashes(&[c.hash()])).unwrap(),
1778
            vec![c.hash()]
1779
        );
1780
        assert_eq!(
1781
            wire::decode_fetch(&wire::encode_fetch(&[c.hash()], Some(4096))).unwrap(),
1782
            (vec![c.hash()], Some(4096))
1783
        );
1784
        assert_eq!(
1785
            wire::decode_fetch(&wire::encode_fetch(&[], None)).unwrap(),
1786
            (vec![], None)
1787
        );
1788
        assert_eq!(
1789
            wire::decode_shallow(&wire::encode_shallow(&[c.hash()], 3)).unwrap(),
1790
            (vec![c.hash()], 3)
1791
        );
b3c5a59a Andy McCall ↶ 1792
        // An old-style fetch request without the filter field still decodes.
75167672 Andy McCall 1793
        assert_eq!(
1794
            wire::decode_fetch(&wire::encode_hashes(&[c.hash()])).unwrap(),
1795
            (vec![c.hash()], None)
1796
        );
750251b2 Andy McCall ↶ 1797
75167672 Andy McCall 1798
        let up = RefUpdate {
1799
            name: "main".into(),
1800
            old_state: ObjectRef([1; 32]),
1801
            new_state: ObjectRef([2; 32]),
1802
            force: true,
1803
            attestation: Some(SignedAttest {
1804
                attest: att,
1805
                sigs: vec![sig],
1806
            }),
1807
        };
1808
        let sp = SignedPolicy {
1809
            policy,
1810
            sigs: vec![psig],
1811
        };
1812
        let (b2, ups, pol, tags) = wire::decode_push(&wire::encode_push(
1813
            &bundle,
1814
            std::slice::from_ref(&up),
1815
            Some(&sp),
1816
            std::slice::from_ref(&tag),
1817
        ))
1818
        .unwrap();
83e0677b Andy McCall ↶ 1819
        assert_eq!(pol, Some(sp));
132b884b Andy McCall ↶ 1820
        assert_eq!(tags, vec![tag]);
750251b2 Andy McCall ↶ 1821
        assert_eq!(b2.changes.len(), 1);
1822
        assert_eq!(ups[0].name, "main");
1823
        assert!(ups[0].force);
1824
        assert_eq!(ups[0].attestation, up.attestation);
75167672 Andy McCall 1825
        let res = vec![RefResult {
1826
            name: "main".into(),
1827
            ok: false,
1828
            reason: "stale".into(),
1829
        }];
1830
        assert_eq!(
1831
            wire::decode_push_response(&wire::encode_push_response(&res)).unwrap(),
1832
            res
1833
        );
1834
        assert_eq!(
1835
            wire::decode_capabilities(&wire::encode_capabilities([7; 16])).unwrap(),
1836
            [7; 16]
1837
        );
750251b2 Andy McCall ↶ 1838
    }
1839
}