OXYGENHUB
# 2b5ee371060e oxygen / crates / oxy-core / src / repo / sync.rs
22.7 KiB · Rust · last changed TLS: oxy serve --tls-cert/--tls-key via rustls; remote.<name>.ca and OXY_CA for self-hosted roots 1 month ago source blame history raw
1
//! Serving and talking to remotes (spec §10.2, §16).
2
3
use super::*;
4
5
#[derive(Debug, Clone, PartialEq, Eq)]
6
pub struct PullReport {
7
    pub heads: Vec<ChangeHash>,
8
    pub changed: bool,
9
    pub checked_out: bool,
10
}
11
12
impl Repo {
13
    // ------------------------------------------------------------ serving (spec §16)
14
15
    fn ref_attest_path(&self, bookmark: &str) -> PathBuf {
16
        self.oxy.join("attest").join(bookmark)
17
    }
18
19
    /// The latest reference attestation stored for a bookmark.
20
    pub fn latest_ref_attest(&self, bookmark: &str) -> Result<Option<(ObjectRef, SignedAttest)>> {
21
        match fs::read_to_string(self.ref_attest_path(bookmark)) {
22
            Ok(h) => {
23
                let r = ObjectRef::from_hex(h.trim()).ok_or_else(|| RepoError::Msg("corrupt attestation pointer".into()))?;
24
                Ok(Some((r, SignedAttest { attest: self.store.get_attest(&r)?, sigs: self.store.signatures(&r)? })))
25
            }
26
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
27
            Err(e) => Err(e.into()),
28
        }
29
    }
30
31
    fn set_ref_attest(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
32
        fs::create_dir_all(self.oxy.join("attest"))?;
33
        write_atomic(&self.ref_attest_path(bookmark), r.to_hex().as_bytes())
34
    }
35
36
    pub fn remote_refs(&self) -> Result<Vec<RemoteRef>> {
37
        let view = self.view()?;
38
        let mut out = Vec::new();
39
        for (name, state) in &view.bookmarks {
40
            let latest = self.latest_ref_attest(name)?.map(|(_, a)| a);
41
            let last_serial = latest.as_ref().and_then(|a| match a.attest.body {
42
                AttestBody::Ref { serial, .. } => Some(serial),
43
                _ => None,
44
            });
45
            let attestation = latest.filter(|a| matches!(&a.attest.body, AttestBody::Ref { new_state, .. } if new_state == state));
46
            out.push(RemoteRef { name: name.clone(), state: *state, heads: self.store.get_state(state)?.heads, attestation, last_serial });
47
        }
48
        Ok(out)
49
    }
50
51
    /// Every change reachable from `heads` that is in the store.
52
    fn closure_in_store(&self, heads: &[ChangeHash]) -> Result<BTreeSet<ChangeHash>> {
53
        let mut seen = BTreeSet::new();
54
        let mut stack: Vec<ChangeHash> = heads.to_vec();
55
        while let Some(h) = stack.pop() {
56
            if seen.contains(&h) || !self.store.has(&h) {
57
                continue;
58
            }
59
            seen.insert(h);
60
            stack.extend(self.store.get_change(&h)?.deps);
61
        }
62
        Ok(seen)
63
    }
64
65
    /// `closure(want) \ closure(have)`, dependencies first (spec §16.2).
66
    pub fn negotiate(&self, have: &[ChangeHash], want: &[ChangeHash]) -> Result<Vec<ChangeHash>> {
67
        let have = self.closure_in_store(have)?;
68
        let mut out = Vec::new();
69
        let mut done: BTreeSet<ChangeHash> = have.clone();
70
        // Iterative post-order DFS.
71
        let mut stack: Vec<(ChangeHash, bool)> = want.iter().rev().map(|h| (*h, false)).collect();
72
        while let Some((h, expanded)) = stack.pop() {
73
            if done.contains(&h) {
74
                continue;
75
            }
76
            if !self.store.has(&h) {
77
                return fail(format!("change {} is not in this repository", h.short(12)));
78
            }
79
            if expanded {
80
                done.insert(h);
81
                out.push(h);
82
                continue;
83
            }
84
            stack.push((h, true));
85
            for d in self.store.get_change(&h)?.deps {
86
                if !done.contains(&d) {
87
                    stack.push((d, false));
88
                }
89
            }
90
        }
91
        Ok(out)
92
    }
93
94
    /// Bundle changes (in the given dependency order) with their signatures,
95
    /// seals, identities, blobs, chunks.
96
    pub fn bundle_for(&self, changes: &[ChangeHash]) -> Result<Bundle> {
97
        let mut bundle = Bundle::default();
98
        let mut objects: BTreeSet<ObjectRef> = BTreeSet::new();
99
        let mut seals: BTreeSet<ObjectRef> = BTreeSet::new();
100
        let add_object = |r: &ObjectRef, objects: &mut BTreeSet<ObjectRef>| {
101
            if !r.is_zero() && self.store.has(r) {
102
                objects.insert(*r);
103
            }
104
        };
105
        for h in changes {
106
            let c = self.store.get_change(h)?;
107
            let sigs = self.store.signatures(h)?;
108
            add_object(&c.author, &mut objects);
109
            for s in &sigs {
110
                add_object(&s.key, &mut objects);
111
            }
112
            for e in &c.edits {
113
                if let Edit::Replace { old, new, .. } = e {
114
                    for b in [old, new].into_iter().flatten() {
115
                        if self.store.has(b) {
116
                            objects.insert(*b);
117
                            if let Ok(blob) = Blob::decode(&self.store.get_typed(b, ObjectType::Blob)?) {
118
                                objects.extend(blob.chunks.iter().copied());
119
                            }
120
                        }
121
                    }
122
                }
123
            }
124
            for s in self.store.seals_for(h)? {
125
                seals.insert(s);
126
            }
127
            bundle.changes.push((c, sigs));
128
        }
129
        for r in seals {
130
            let attest = self.store.get_attest(&r)?;
131
            let sigs = self.store.signatures(&r)?;
132
            for s in &sigs {
133
                add_object(&s.key, &mut objects);
134
            }
135
            bundle.attests.push(SignedAttest { attest, sigs });
136
        }
137
        for r in objects {
138
            let (ty, body) = self.store.get(&r)?;
139
            bundle.objects.push((ty, body));
140
        }
141
        Ok(bundle)
142
    }
143
144
    /// Store a bundle's contents. With `strict`, every change must
145
    /// authenticate (signature or seal by its author or a delegate).
146
    pub fn import_bundle(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
147
        for (ty, body) in &bundle.objects {
148
            match ty {
149
                ObjectType::Chunk | ObjectType::Blob | ObjectType::Identity | ObjectType::State => {
150
                    self.store.put(*ty, body)?;
151
                }
152
                other => return fail(format!("bundle may not carry {} objects", other.name())),
153
            }
154
        }
155
        for sa in &bundle.attests {
156
            let r = self.store.put_attest(&sa.attest)?;
157
            for sig in &sa.sigs {
158
                self.store.put_signature(&r, sig)?;
159
            }
160
            if let AttestBody::Seal { hashes } = &sa.attest.body {
161
                for h in hashes {
162
                    self.store.index_seal(h, &r)?;
163
                }
164
            }
165
        }
166
        let incoming: BTreeSet<ChangeHash> = bundle.changes.iter().map(|(c, _)| c.hash()).collect();
167
        let mut imported = Vec::new();
168
        for (c, sigs) in &bundle.changes {
169
            let h = c.hash();
170
            for d in &c.deps {
171
                if !self.store.has(d) && !incoming.contains(d) {
172
                    return fail(format!("change {} depends on {}, which is neither stored nor in the bundle", h.short(12), d.short(12)));
173
                }
174
            }
175
            self.store.put_change(c)?;
176
            for sig in sigs {
177
                self.store.put_signature(&h, sig)?;
178
            }
179
            imported.push(h);
180
        }
181
        if strict {
182
            for h in &imported {
183
                if !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. }) {
184
                    return fail(format!("change {} is not authenticated by its author", h.short(12)));
185
                }
186
            }
187
        }
188
        Ok(imported)
189
    }
190
191
    /// Verify a reference attestation against an update it claims to describe.
192
    /// `allow_equal` lets a client re-read the attestation it already has; a
193
    /// server receiving a push requires a strictly greater serial.
194
    fn check_ref_attest(&self, sa: &SignedAttest, name: &str, old_state: &ObjectRef, new_state: &ObjectRef, last_serial: Option<u64>, allow_equal: bool) -> std::result::Result<u64, String> {
195
        let AttestBody::Ref { name: n, old_state: o, new_state: nw, serial } = &sa.attest.body else {
196
            return Err("attestation is not a ref attestation".into());
197
        };
198
        if n != name || o != old_state || nw != new_state {
199
            return Err("attestation does not describe this update".into());
200
        }
201
        if sa.attest.repo != self.id().map_err(|e| e.to_string())? {
202
            return Err("attestation is for a different repository".into());
203
        }
204
        if let Some(last) = last_serial {
205
            if *serial < last || (*serial == last && !allow_equal) {
206
                return Err(format!("attestation serial {serial} is not greater than the last seen {last}"));
207
            }
208
        }
209
        let r = sa.attest.hash();
210
        let sig = sa.sigs.first().ok_or("attestation carries no signature")?;
211
        let id = self.store.get_identity(&sig.key).map_err(|_| format!("unknown key {}", sig.key.short(12)))?;
212
        verify_signature(&id, sig, Domain::Attest, &r).map_err(|e| e.to_string())?;
213
        Ok(*serial)
214
    }
215
216
    /// Accept a push (spec §16.3). Without a trust policy every repository
217
    /// is unverified, so unsigned changes and unattested updates are accepted;
218
    /// attestations that are present must still be valid.
219
    pub fn receive_push(&self, bundle: Bundle, updates: Vec<RefUpdate>) -> Result<Vec<RefResult>> {
220
        self.import_bundle(&bundle, false)?;
221
        let (op_ref, op) = self.head_op()?;
222
        let mut view = op.view.clone();
223
        let mut results = Vec::new();
224
        let mut names = Vec::new();
225
        for u in updates {
226
            let outcome = self.apply_update(&mut view, &u);
227
            let (ok, reason) = match outcome {
228
                Ok(()) => (true, String::new()),
229
                Err(e) => (false, e),
230
            };
231
            if ok {
232
                names.push(u.name.clone());
233
            }
234
            results.push(RefResult { name: u.name, ok, reason });
235
        }
236
        if !names.is_empty() {
237
            self.record_op(&[op_ref], &format!("push received: {}", names.join(" ")), view)?;
238
        }
239
        Ok(results)
240
    }
241
242
    fn apply_update(&self, view: &mut View, u: &RefUpdate) -> std::result::Result<(), String> {
243
        let empty = self.store.put_state(&StateObject::new(vec![])).map_err(|e| e.to_string())?;
244
        let current = view.bookmarks.get(&u.name).copied().unwrap_or(empty);
245
        if current != u.old_state {
246
            return Err("stale: the remote bookmark has moved; pull first".into());
247
        }
248
        let new_state = self.store.get_state(&u.new_state).map_err(|_| "new state object is missing".to_string())?;
249
        for h in &new_state.heads {
250
            if !self.store.has(h) {
251
                return Err(format!("head {} is missing", h.short(12)));
252
            }
253
        }
254
        if !u.force {
255
            let old_heads = self.store.get_state(&current).map_err(|e| e.to_string())?.heads;
256
            let closure = self.closure_in_store(&new_state.heads).map_err(|e| e.to_string())?;
257
            if let Some(h) = old_heads.iter().find(|h| !closure.contains(h)) {
258
                return Err(format!("not a fast-forward: current head {} would be dropped (use --force)", h.short(12)));
259
            }
260
        }
261
        if let Some(sa) = &u.attestation {
262
            let last = self.latest_ref_attest(&u.name).map_err(|e| e.to_string())?.and_then(|(_, a)| match a.attest.body {
263
                AttestBody::Ref { serial, .. } => Some(serial),
264
                _ => None,
265
            });
266
            self.check_ref_attest(sa, &u.name, &u.old_state, &u.new_state, last, false)?;
267
            let r = self.store.put_attest(&sa.attest).map_err(|e| e.to_string())?;
268
            for sig in &sa.sigs {
269
                self.store.put_signature(&r, sig).map_err(|e| e.to_string())?;
270
            }
271
            self.set_ref_attest(&u.name, &r).map_err(|e| e.to_string())?;
272
        }
273
        view.bookmarks.insert(u.name.clone(), u.new_state);
274
        Ok(())
275
    }
276
277
    // ------------------------------------------------------------ client side
278
279
    fn remote_url(&self, name: &str) -> Result<String> {
280
        self.config.remotes.get(name).cloned().ok_or_else(|| RepoError::Msg(format!("no remote named {name:?}; add one with `oxy remote add`")))
281
    }
282
283
    /// Credentials and trust for a remote: `remote.<name>.token` (else
284
    /// `$OXY_TOKEN`) and `remote.<name>.ca` (else `$OXY_CA`).
285
    fn remote_options(&self, name: &str) -> RemoteOptions {
286
        RemoteOptions {
287
            token: self.config.remote_tokens.get(name).cloned().or_else(|| std::env::var("OXY_TOKEN").ok()),
288
            ca: self.config.remote_cas.get(name).map(PathBuf::from).or_else(|| std::env::var("OXY_CA").ok().map(PathBuf::from)),
289
        }
290
    }
291
292
    fn open_named_remote(&self, name: &str) -> Result<Box<dyn crate::protocol::Remote>> {
293
        open_remote(&self.remote_url(name)?, &self.remote_options(name))
294
    }
295
296
    fn serial_path(&self, remote: &str, bookmark: &str) -> PathBuf {
297
        self.oxy.join("remotes").join(remote).join(format!("{bookmark}.serial"))
298
    }
299
300
    fn last_seen_serial(&self, remote: &str, bookmark: &str) -> Option<u64> {
301
        fs::read_to_string(self.serial_path(remote, bookmark)).ok().and_then(|s| s.trim().parse().ok())
302
    }
303
304
    fn record_serial(&self, remote: &str, bookmark: &str, serial: u64) -> Result<()> {
305
        let p = self.serial_path(remote, bookmark);
306
        fs::create_dir_all(p.parent().unwrap())?;
307
        write_atomic(&p, serial.to_string().as_bytes())
308
    }
309
310
    /// Every head this repository knows: bookmarks, remote tracking, working copy.
311
    fn known_heads(&self, view: &View) -> Result<Vec<ChangeHash>> {
312
        let mut have: BTreeSet<ChangeHash> = BTreeSet::new();
313
        for st in view.bookmarks.values().chain(view.remotes.values().flat_map(|b| b.values())) {
314
            if let Ok(s) = self.store.get_state(st) {
315
                have.extend(s.heads);
316
            }
317
        }
318
        have.extend(view.workspaces.values().copied());
319
        Ok(have.into_iter().collect())
320
    }
321
322
    /// Fetch every bookmark of a remote into `remotes/<name>/bookmarks/*` (spec §10.2).
323
    pub fn fetch(&self, remote_name: &str) -> Result<Vec<(String, Vec<ChangeHash>, usize)>> {
324
        let remote = self.open_named_remote(remote_name)?;
325
        if remote.repo_id()? != self.id()? {
326
            return fail(format!("remote {remote_name:?} is a different repository (repository IDs differ)"));
327
        }
328
        let (op_ref, op) = self.head_op()?;
329
        let mut view = op.view.clone();
330
        let have = self.known_heads(&view)?;
331
        let mut report = Vec::new();
332
        for r in remote.refs()? {
333
            let needed = remote.negotiate(&have, &r.heads)?;
334
            let bundle = remote.fetch(&needed)?;
335
            let imported = self.import_bundle(&bundle, false)?;
336
            for h in &r.heads {
337
                if !self.store.has(h) {
338
                    return fail(format!("remote did not send head {}", h.short(12)));
339
                }
340
            }
341
            let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
342
            if state != r.state {
343
                return fail(format!("remote state {} does not match its heads", r.state.short(12)));
344
            }
345
            if let Some(sa) = &r.attestation {
346
                let AttestBody::Ref { old_state, new_state, .. } = &sa.attest.body else {
347
                    return fail("remote sent a non-ref attestation for a bookmark");
348
                };
349
                let (old_state, new_state) = (*old_state, *new_state);
350
                let serial = self
351
                    .check_ref_attest(sa, &r.name, &old_state, &new_state, self.last_seen_serial(remote_name, &r.name), true)
352
                    .map_err(|e| RepoError::Msg(format!("attestation for {}/{}: {e}", remote_name, r.name)))?;
353
                if new_state != r.state {
354
                    return fail(format!("attestation for {}/{} does not cover its current state", remote_name, r.name));
355
                }
356
                let ar = self.store.put_attest(&sa.attest)?;
357
                for sig in &sa.sigs {
358
                    self.store.put_signature(&ar, sig)?;
359
                }
360
                self.record_serial(remote_name, &r.name, serial)?;
361
            }
362
            view.remotes.entry(remote_name.to_string()).or_default().insert(r.name.clone(), state);
363
            report.push((r.name, r.heads, imported.len()));
364
        }
365
        if view != op.view {
366
            self.record_op(&[op_ref], &format!("oxy fetch {remote_name}"), view)?;
367
        }
368
        Ok(report)
369
    }
370
371
    /// Fetch, then take the union of the local bookmark's state with the
372
    /// remote's: no merge change, conflicts become reified (spec §9.4).
373
    pub fn pull(&self, remote_name: &str) -> Result<PullReport> {
374
        self.fetch(remote_name)?;
375
        let (op_ref, op) = self.head_op()?;
376
        let bookmark = self.config.bookmark.clone();
377
        let local = self.heads_of(&op.view, &bookmark)?;
378
        let Some(remote_state) = op.view.remotes.get(remote_name).and_then(|b| b.get(&bookmark)).copied() else {
379
            return Ok(PullReport { heads: local, changed: false, checked_out: false });
380
        };
381
        let remote_heads = self.store.get_state(&remote_state)?.heads;
382
        let mut all = local.clone();
383
        all.extend(remote_heads);
384
        let st = self.load_state(&all)?;
385
        let mut heads = st.maximal(&all);
386
        heads.sort();
387
        let mut sorted_local = local.clone();
388
        sorted_local.sort();
389
        if heads == sorted_local {
390
            return Ok(PullReport { heads, changed: false, checked_out: false });
391
        }
392
        let mut view = op.view.clone();
393
        view.bookmarks.insert(bookmark.clone(), self.store.put_state(&StateObject::new(heads.clone()))?);
394
        self.record_op(&[op_ref], &format!("oxy pull {remote_name}"), view)?;
395
        // Follow along if the working copy was clean and on the old heads.
396
        let snap = self.snapshot("oxy pull")?;
397
        let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
398
        let mut base = snap.base.maximal(&snap.wc.deps);
399
        base.sort();
400
        let mut checked_out = false;
401
        if clean && base == sorted_local {
402
            self.checkout(&heads, false, "oxy pull (checkout)")?;
403
            checked_out = true;
404
        }
405
        Ok(PullReport { heads, changed: true, checked_out })
406
    }
407
408
    /// Push the current bookmark (spec §16.3), sealing this author's unsigned
409
    /// changes and attesting the reference when a key is available.
410
    pub fn push(&self, remote_name: &str, force: bool) -> Result<Vec<RefResult>> {
411
        let remote = self.open_named_remote(remote_name)?;
412
        if remote.repo_id()? != self.id()? {
413
            return fail(format!("remote {remote_name:?} is a different repository (repository IDs differ)"));
414
        }
415
        let (op_ref, op) = self.head_op()?;
416
        let bookmark = self.config.bookmark.clone();
417
        let local_heads = self.heads_of(&op.view, &bookmark)?;
418
        let refs = remote.refs()?;
419
        let empty = self.store.put_state(&StateObject::new(vec![]))?;
420
        let (remote_state, remote_heads, last_serial) = match refs.iter().find(|r| r.name == bookmark) {
421
            Some(r) => (r.state, r.heads.clone(), r.last_serial),
422
            None => (empty, vec![], None),
423
        };
424
        let new_state = self.store.put_state(&StateObject::new(local_heads.clone()))?;
425
        if new_state == remote_state {
426
            return Ok(vec![RefResult { name: bookmark, ok: true, reason: "already up to date".into() }]);
427
        }
428
        for h in &remote_heads {
429
            if !self.store.has(h) {
430
                return fail(format!("the remote has change {} which you do not; pull first", h.short(12)));
431
            }
432
        }
433
        if !force {
434
            let closure = self.closure_in_store(&local_heads)?;
435
            if let Some(h) = remote_heads.iter().find(|h| !closure.contains(h)) {
436
                return fail(format!("not a fast-forward: remote head {} is not in your state; pull first or use --force", h.short(12)));
437
            }
438
        }
439
        let to_send = self.negotiate(&remote_heads, &local_heads)?;
440
        let mut bundle = self.bundle_for(&to_send)?;
441
        let key = self.secret_key()?.map(|(k, _)| k);
442
        if let Some(key) = key {
443
            // Seal this author's changes that carry no valid authenticator.
444
            let mut unsigned = Vec::new();
445
            for h in &to_send {
446
                let c = self.store.get_change(h)?;
447
                if c.author == key && !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. }) {
448
                    unsigned.push(*h);
449
                }
450
            }
451
            for chunk in unsigned.chunks(crate::attest::MAX_SEAL_HASHES) {
452
                let sa = self.sign_attest(&Attest::seal(self.id()?, chunk.to_vec()))?;
453
                bundle.attests.push(sa);
454
            }
455
            bundle = self.bundle_for(&to_send)?;
456
        }
457
        bundle.objects.push((ObjectType::State, StateObject::new(local_heads.clone()).encode()));
458
        let attestation = match key {
459
            Some(_) => Some(self.sign_attest(&Attest {
460
                repo: self.id()?,
461
                hlc: Hlc::now(),
462
                body: AttestBody::Ref { name: bookmark.clone(), old_state: remote_state, new_state, serial: last_serial.unwrap_or(0) + 1 },
463
            })?),
464
            None => None,
465
        };
466
        let serial = attestation.as_ref().and_then(|a| match a.attest.body {
467
            AttestBody::Ref { serial, .. } => Some(serial),
468
            _ => None,
469
        });
470
        let results = remote.push(bundle, vec![RefUpdate { name: bookmark.clone(), old_state: remote_state, new_state, force, attestation }])?;
471
        if results.iter().any(|r| r.ok) {
472
            let mut view = op.view.clone();
473
            view.remotes.entry(remote_name.to_string()).or_default().insert(bookmark.clone(), new_state);
474
            self.record_op(&[op_ref], &format!("oxy push {remote_name}"), view)?;
475
            if let Some(s) = serial {
476
                self.record_serial(remote_name, &bookmark, s)?;
477
            }
478
        }
479
        Ok(results)
480
    }
481
482
    /// Clone a remote into `dest`: same repository ID, `origin` remote,
483
    /// bookmark set to the remote's, working copy checked out.
484
    pub fn clone(url: &str, dest: &Path) -> Result<Repo> {
485
        let options = RemoteOptions {
486
            token: std::env::var("OXY_TOKEN").ok(),
487
            ca: std::env::var("OXY_CA").ok().map(PathBuf::from),
488
        };
489
        let remote = open_remote(url, &options)?;
490
        let id = remote.repo_id()?;
491
        let mut repo = Repo::init_with_id(dest, id)?;
492
        repo.config.set("remote.origin.url", url)?;
493
        repo.save_config()?;
494
        repo.fetch("origin")?;
495
        let (op_ref, op) = repo.head_op()?;
496
        let bookmark = repo.config.bookmark.clone();
497
        if let Some(state) = op.view.remotes.get("origin").and_then(|b| b.get(&bookmark)).copied() {
498
            let heads = repo.store.get_state(&state)?.heads;
499
            let mut view = op.view.clone();
500
            view.bookmarks.insert(bookmark, state);
501
            repo.record_op(&[op_ref], &format!("oxy clone {url}"), view)?;
502
            repo.checkout(&heads, false, "oxy clone (checkout)")?;
503
        }
504
        Ok(repo)
505
    }
506
}