OXYGENHUB
# 957fee30826e oxygen / crates / oxy-core / src / repo / sync.rs
45.1 KiB · Rust · last changed Bookmarks: any number per repository, per-workspace current bookmark (bookmark create|set|delete|list|switch), oxy merge as union of heads, bookmark names resolve everywhere, push -b (spec 0.8 section 10.1) 1 month ago source blame history raw
1
//! Serving and talking to remotes (spec §10.2, §16).
2
3
use super::*;
4
5
/// What a fetch brought in.
6
#[derive(Debug, Clone, PartialEq, Eq, Default)]
7
pub struct FetchReport {
8
    /// Bookmark name, its heads, and how many changes were new.
9
    pub refs: Vec<(String, Vec<ChangeHash>, usize)>,
10
    /// Tags adopted from the remote.
11
    pub new_tags: usize,
12
    /// The remote has never presented a policy, so nothing was verified
13
    /// against one (spec §14.5: warn loudly).
14
    pub unverified: bool,
15
}
16
17
#[derive(Debug, Clone, PartialEq, Eq)]
18
pub struct PullReport {
19
    pub heads: Vec<ChangeHash>,
20
    pub changed: bool,
21
    pub checked_out: bool,
22
}
23
24
impl Repo {
25
    // ------------------------------------------------------------ serving (spec §16)
26
27
    fn ref_attest_path(&self, bookmark: &str) -> PathBuf {
28
        self.oxy.join("attest").join(bookmark)
29
    }
30
31
    /// The latest reference attestation stored for a bookmark.
32
    pub fn latest_ref_attest(&self, bookmark: &str) -> Result<Option<(ObjectRef, SignedAttest)>> {
33
        match fs::read_to_string(self.ref_attest_path(bookmark)) {
34
            Ok(h) => {
35
                let r = ObjectRef::from_hex(h.trim()).ok_or_else(|| RepoError::Msg("corrupt attestation pointer".into()))?;
36
                Ok(Some((r, SignedAttest { attest: self.store.get_attest(&r)?, sigs: self.store.signatures(&r)? })))
37
            }
38
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
39
            Err(e) => Err(e.into()),
40
        }
41
    }
42
43
    fn set_ref_attest(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
44
        fs::create_dir_all(self.oxy.join("attest"))?;
45
        write_atomic(&self.ref_attest_path(bookmark), r.to_hex().as_bytes())
46
    }
47
48
    pub fn remote_refs(&self) -> Result<RefsResponse> {
49
        let view = self.view()?;
50
        let mut out = Vec::new();
51
        for (name, state) in &view.bookmarks {
52
            let latest = self.latest_ref_attest(name)?.map(|(_, a)| a);
53
            let last_serial = latest.as_ref().and_then(|a| match a.attest.body {
54
                AttestBody::Ref { serial, .. } => Some(serial),
55
                _ => None,
56
            });
57
            let attestation = latest.filter(|a| matches!(&a.attest.body, AttestBody::Ref { new_state, .. } if new_state == state));
58
            out.push(RemoteRef { name: name.clone(), state: *state, heads: self.store.get_state(state)?.heads, attestation, last_serial });
59
        }
60
        let policy = self.current_policy()?.map(|(_, sp)| sp);
61
        let history = self.policy_history()?.into_iter().filter(|sp| policy.as_ref().is_some_and(|p| sp.policy.serial < p.policy.serial)).collect();
62
        Ok(RefsResponse { refs: out, policy, tags: self.tag_refs()?, history })
63
    }
64
65
    /// Every change reachable from `heads` that is in the store.
66
    fn closure_in_store(&self, heads: &[ChangeHash]) -> Result<BTreeSet<ChangeHash>> {
67
        let mut seen = BTreeSet::new();
68
        let mut stack: Vec<ChangeHash> = heads.to_vec();
69
        while let Some(h) = stack.pop() {
70
            if seen.contains(&h) || !self.store.has(&h) {
71
                continue;
72
            }
73
            seen.insert(h);
74
            stack.extend(self.store.get_change(&h)?.deps);
75
        }
76
        Ok(seen)
77
    }
78
79
    /// The changes within `depth` dependency steps of `heads` (spec §17),
80
    /// nearest first; depth 1 is the heads themselves.
81
    pub fn shallow_changes(&self, heads: &[ChangeHash], depth: u32) -> Result<Vec<ChangeHash>> {
82
        let mut out = Vec::new();
83
        let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
84
        let mut layer: Vec<ChangeHash> = heads.to_vec();
85
        for _ in 0..depth.max(1) {
86
            let mut next = Vec::new();
87
            for h in layer {
88
                if !seen.insert(h) || !self.store.has(&h) {
89
                    continue;
90
                }
91
                let c = self.store.get_change(&h)?;
92
                next.extend(c.deps.iter().copied());
93
                out.push(h);
94
            }
95
            if next.is_empty() {
96
                break;
97
            }
98
            layer = next;
99
        }
100
        out.reverse();
101
        Ok(out)
102
    }
103
104
    fn shallow_path(&self) -> PathBuf {
105
        self.oxy.join("shallow")
106
    }
107
108
    /// The shallow boundary: dependencies this repository does not hold (spec §17).
109
    pub fn boundary(&self) -> BTreeSet<ChangeHash> {
110
        fs::read_to_string(self.shallow_path()).map(|t| t.lines().filter_map(ObjectRef::from_hex).collect()).unwrap_or_default()
111
    }
112
113
    pub fn is_shallow(&self) -> bool {
114
        !self.boundary().is_empty()
115
    }
116
117
    fn write_boundary(&self, boundary: &BTreeSet<ChangeHash>) -> Result<()> {
118
        if boundary.is_empty() {
119
            let _ = fs::remove_file(self.shallow_path());
120
            return Ok(());
121
        }
122
        let text: String = boundary.iter().map(|h| format!("{}\n", h.to_hex())).collect();
123
        write_atomic(&self.shallow_path(), text.as_bytes())
124
    }
125
126
    /// Recompute the boundary from what is stored: every dependency of a
127
    /// stored change that is not itself stored.
128
    fn refresh_boundary(&self, imported: &[ChangeHash]) -> Result<()> {
129
        let mut boundary = self.boundary();
130
        boundary.retain(|h| !self.store.has(h));
131
        for h in imported {
132
            for d in self.store.get_change(h)?.deps {
133
                if !self.store.has(&d) {
134
                    boundary.insert(d);
135
                }
136
            }
137
        }
138
        self.write_boundary(&boundary)
139
    }
140
141
    /// Fetch `depth` more levels of history below the shallow boundary.
142
    pub fn deepen(&self, remote_name: &str, depth: u32) -> Result<usize> {
143
        let boundary = self.boundary();
144
        if boundary.is_empty() {
145
            return Ok(0);
146
        }
147
        let remote = self.open_named_remote(remote_name)?;
148
        let wanted = remote.shallow(&boundary.iter().copied().collect::<Vec<_>>(), depth)?;
149
        let bundle = remote.fetch(&wanted, self.config.max_blob)?;
150
        let imported = self.import_bundle_shallow(&bundle, false)?;
151
        if self.config.max_blob.is_some() {
152
            self.register_missing_blobs(&imported, remote_name)?;
153
        }
154
        self.refresh_boundary(&imported)?;
155
        Ok(imported.len())
156
    }
157
158
    /// `closure(want) \ closure(have)`, dependencies first (spec §16.2).
159
    pub fn negotiate(&self, have: &[ChangeHash], want: &[ChangeHash]) -> Result<Vec<ChangeHash>> {
160
        let have = self.closure_in_store(have)?;
161
        // In a shallow repository the walk stops at the boundary, whose
162
        // changes the remote is assumed to hold.
163
        let boundary = self.boundary();
164
        let mut out = Vec::new();
165
        let mut done: BTreeSet<ChangeHash> = have.clone();
166
        done.extend(boundary.iter().copied());
167
        // Iterative post-order DFS.
168
        let mut stack: Vec<(ChangeHash, bool)> = want.iter().rev().map(|h| (*h, false)).collect();
169
        while let Some((h, expanded)) = stack.pop() {
170
            if done.contains(&h) {
171
                continue;
172
            }
173
            if !self.store.has(&h) {
174
                return fail(format!("change {} is not in this repository", h.short(12)));
175
            }
176
            if expanded {
177
                done.insert(h);
178
                out.push(h);
179
                continue;
180
            }
181
            stack.push((h, true));
182
            for d in self.store.get_change(&h)?.deps {
183
                if !done.contains(&d) {
184
                    stack.push((d, false));
185
                }
186
            }
187
        }
188
        Ok(out)
189
    }
190
191
    /// Bundle changes (in the given dependency order) with their signatures,
192
    /// seals, identities, blobs, chunks.
193
    pub fn bundle_for(&self, changes: &[ChangeHash]) -> Result<Bundle> {
194
        self.bundle_for_filtered(changes, None)
195
    }
196
197
    /// Objects by reference for the `objects` endpoint; a blob brings its chunks.
198
    pub fn objects_for(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>> {
199
        let mut out = Vec::new();
200
        for r in refs {
201
            let (ty, body) = match self.store.get(r) {
202
                Ok(x) => x,
203
                Err(StoreError::NotFound(_)) => continue,
204
                Err(e) => return Err(e.into()),
205
            };
206
            if ty == ObjectType::Blob {
207
                for c in Blob::decode(&body)?.chunks {
208
                    if let Ok((cty, cb)) = self.store.get(&c) {
209
                        out.push((cty, cb));
210
                    }
211
                }
212
            }
213
            out.push((ty, body));
214
        }
215
        Ok(out)
216
    }
217
218
    /// Like `bundle_for`, leaving out blobs larger than `max_blob` bytes
219
    /// (and their chunks) so the receiver fetches them lazily (spec §13.5).
220
    pub fn bundle_for_filtered(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle> {
221
        let mut bundle = Bundle::default();
222
        let mut objects: BTreeSet<ObjectRef> = BTreeSet::new();
223
        let mut seals: BTreeSet<ObjectRef> = BTreeSet::new();
224
        let add_object = |r: &ObjectRef, objects: &mut BTreeSet<ObjectRef>| {
225
            if !r.is_zero() && self.store.has(r) {
226
                objects.insert(*r);
227
            }
228
        };
229
        for h in changes {
230
            let c = self.store.get_change(h)?;
231
            let sigs = self.store.signatures(h)?;
232
            add_object(&c.author, &mut objects);
233
            for s in &sigs {
234
                add_object(&s.key, &mut objects);
235
            }
236
            for e in &c.edits {
237
                if let Edit::Replace { old, new, .. } = e {
238
                    for b in [old, new].into_iter().flatten() {
239
                        if self.store.has(b) {
240
                            if let Ok(blob) = Blob::decode(&self.store.get_typed(b, ObjectType::Blob)?) {
241
                                if max_blob.is_some_and(|m| blob.length > m) {
242
                                    continue;
243
                                }
244
                                objects.insert(*b);
245
                                objects.extend(blob.chunks.iter().copied());
246
                            }
247
                        }
248
                    }
249
                }
250
            }
251
            for s in self.store.seals_for(h)? {
252
                seals.insert(s);
253
            }
254
            bundle.changes.push((c, sigs));
255
        }
256
        for r in seals {
257
            let attest = self.store.get_attest(&r)?;
258
            let sigs = self.store.signatures(&r)?;
259
            for s in &sigs {
260
                add_object(&s.key, &mut objects);
261
            }
262
            bundle.attests.push(SignedAttest { attest, sigs });
263
        }
264
        for r in objects {
265
            let (ty, body) = self.store.get(&r)?;
266
            bundle.objects.push((ty, body));
267
        }
268
        Ok(bundle)
269
    }
270
271
    /// Store a bundle's contents. With `strict`, every change must
272
    /// authenticate (signature or seal by its author or a delegate).
273
    pub fn import_bundle(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
274
        self.import_bundle_inner(bundle, strict, false)
275
    }
276
277
    /// Like `import_bundle`, accepting changes whose dependencies are absent
278
    /// (they join the shallow boundary).
279
    pub fn import_bundle_shallow(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
280
        self.import_bundle_inner(bundle, strict, true)
281
    }
282
283
    fn import_bundle_inner(&self, bundle: &Bundle, strict: bool, shallow: bool) -> Result<Vec<ChangeHash>> {
284
        let batch = self.store.begin_batch()?;
285
        let result = self.import_bundle_batched(bundle, strict, shallow);
286
        batch.finish()?;
287
        result
288
    }
289
290
    fn import_bundle_batched(&self, bundle: &Bundle, strict: bool, shallow: bool) -> Result<Vec<ChangeHash>> {
291
        for (ty, body) in &bundle.objects {
292
            match ty {
293
                ObjectType::Chunk | ObjectType::Blob | ObjectType::Identity | ObjectType::State | ObjectType::Checkpoint => {
294
                    self.store.put(*ty, body)?;
295
                }
296
                other => return fail(format!("bundle may not carry {} objects", other.name())),
297
            }
298
        }
299
        for sa in &bundle.attests {
300
            let r = self.store.put_attest(&sa.attest)?;
301
            for sig in &sa.sigs {
302
                self.store.put_signature(&r, sig)?;
303
            }
304
            if let AttestBody::Seal { hashes } = &sa.attest.body {
305
                for h in hashes {
306
                    self.store.index_seal(h, &r)?;
307
                }
308
            }
309
        }
310
        let incoming: BTreeSet<ChangeHash> = bundle.changes.iter().map(|(c, _)| c.hash()).collect();
311
        let boundary = self.boundary();
312
        let mut imported = Vec::new();
313
        for (c, sigs) in &bundle.changes {
314
            let h = c.hash();
315
            for d in &c.deps {
316
                if !self.store.has(d) && !incoming.contains(d) && !shallow && !boundary.contains(d) {
317
                    return fail(format!("change {} depends on {}, which is neither stored nor in the bundle", h.short(12), d.short(12)));
318
                }
319
            }
320
            self.store.put_change(c)?;
321
            for sig in sigs {
322
                self.store.put_signature(&h, sig)?;
323
            }
324
            imported.push(h);
325
        }
326
        if strict {
327
            for h in &imported {
328
                if !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. }) {
329
                    return fail(format!("change {} is not authenticated by its author", h.short(12)));
330
                }
331
            }
332
        }
333
        Ok(imported)
334
    }
335
336
    /// Verify a reference attestation against an update it claims to describe.
337
    /// `allow_equal` lets a client re-read the attestation it already has; a
338
    /// server receiving a push requires a strictly greater serial.
339
    fn check_ref_attest(&self, sa: &SignedAttest, name: &str, old_state: &ObjectRef, new_state: &ObjectRef, last_serial: Option<u64>, allow_equal: bool) -> std::result::Result<u64, String> {
340
        let AttestBody::Ref { name: n, old_state: o, new_state: nw, serial } = &sa.attest.body else {
341
            return Err("attestation is not a ref attestation".into());
342
        };
343
        if n != name || o != old_state || nw != new_state {
344
            return Err("attestation does not describe this update".into());
345
        }
346
        if sa.attest.repo != self.id().map_err(|e| e.to_string())? {
347
            return Err("attestation is for a different repository".into());
348
        }
349
        if let Some(last) = last_serial {
350
            if *serial < last || (*serial == last && !allow_equal) {
351
                return Err(format!("attestation serial {serial} is not greater than the last seen {last}"));
352
            }
353
        }
354
        let r = sa.attest.hash();
355
        let sig = sa.sigs.first().ok_or("attestation carries no signature")?;
356
        let id = self.store.get_identity(&sig.key).map_err(|_| format!("unknown key {}", sig.key.short(12)))?;
357
        verify_signature(&id, sig, Domain::Attest, &r).map_err(|e| e.to_string())?;
358
        Ok(*serial)
359
    }
360
361
    /// Accept a push (spec §16.4). Without a trust policy every repository
362
    /// is unverified, so unsigned changes and unattested updates are accepted;
363
    /// attestations that are present must still be valid. With a policy,
364
    /// every change and update is checked against it. A newer policy in the
365
    /// push is applied first when it is signed by the current roots.
366
    pub fn receive_push(&self, bundle: Bundle, updates: Vec<RefUpdate>, policy: Option<SignedPolicy>, tags: Vec<crate::protocol::TagRef>) -> Result<Vec<RefResult>> {
367
        let mut results = Vec::new();
368
        if let Some(sp) = policy {
369
            match self.adopt_policy_chain(&bundle.policies, &sp, "push received") {
370
                Ok(()) => {
371
                    results.push(RefResult { name: "policy".into(), ok: true, reason: format!("serial {}", sp.policy.serial) });
372
                }
373
                Err(e) => {
374
                    results.push(RefResult { name: "policy".into(), ok: false, reason: e });
375
                    for u in updates {
376
                        results.push(RefResult { name: u.name, ok: false, reason: "policy update was rejected".into() });
377
                    }
378
                    return Ok(results);
379
                }
380
            }
381
        }
382
        let policy = self.current_policy()?.map(|(_, sp)| sp.policy);
383
        if let Err(e) = self.import_bundle(&bundle, policy.is_some()) {
384
            for u in updates {
385
                results.push(RefResult { name: u.name, ok: false, reason: e.to_string() });
386
            }
387
            return Ok(results);
388
        }
389
        let (op_ref, op) = self.head_op()?;
390
        let mut view = op.view.clone();
391
        let mut names = Vec::new();
392
        for u in updates {
393
            let outcome = self.apply_update(&mut view, &u, policy.as_ref());
394
            let (ok, reason) = match outcome {
395
                Ok(()) => (true, String::new()),
396
                Err(e) => (false, e),
397
            };
398
            if ok {
399
                names.push(u.name.clone());
400
            }
401
            results.push(RefResult { name: u.name, ok, reason });
402
        }
403
        for t in &tags {
404
            let name = format!("tags/{}", t.name);
405
            match self.adopt_tag(&mut view, t, policy.as_ref()) {
406
                Ok(()) => {
407
                    names.push(name.clone());
408
                    results.push(RefResult { name, ok: true, reason: String::new() });
409
                }
410
                Err(e) => results.push(RefResult { name, ok: false, reason: e }),
411
            }
412
        }
413
        if !names.is_empty() {
414
            self.record_op(&[op_ref], &format!("push received: {}", names.join(" ")), view)?;
415
        }
416
        Ok(results)
417
    }
418
419
    fn apply_update(&self, view: &mut View, u: &RefUpdate, policy: Option<&crate::policy::Policy>) -> std::result::Result<(), String> {
420
        if u.name == "policy" {
421
            return Err("the policy reference is updated by sending a policy, not a bookmark update".into());
422
        }
423
        let empty = self.store.put_state(&StateObject::new(vec![])).map_err(|e| e.to_string())?;
424
        let current = view.bookmarks.get(&u.name).copied().unwrap_or(empty);
425
        if current != u.old_state {
426
            return Err("stale: the remote bookmark has moved; pull first".into());
427
        }
428
        let new_state = self.store.get_state(&u.new_state).map_err(|_| "new state object is missing".to_string())?;
429
        for h in &new_state.heads {
430
            if !self.store.has(h) {
431
                return Err(format!("head {} is missing", h.short(12)));
432
            }
433
        }
434
        if !u.force {
435
            let old_heads = self.store.get_state(&current).map_err(|e| e.to_string())?.heads;
436
            let closure = self.closure_in_store(&new_state.heads).map_err(|e| e.to_string())?;
437
            if let Some(h) = old_heads.iter().find(|h| !closure.contains(h)) {
438
                return Err(format!("not a fast-forward: current head {} would be dropped (use --force)", h.short(12)));
439
            }
440
        }
441
        if let Some(p) = policy {
442
            let Some(sa) = &u.attestation else {
443
                return Err(format!("{:?} requires a signed reference attestation under this repository's policy", u.name));
444
            };
445
            let signer = sa.sigs.first().map(|s| s.key).ok_or("attestation carries no signature")?;
446
            if !p.authorizes(&signer, &u.name) {
447
                return Err(format!("key {} may not update {:?} under this repository's policy", signer.short(12), u.name));
448
            }
449
            let old_heads = self.store.get_state(&current).map_err(|e| e.to_string())?.heads;
450
            self.state_allowed(&old_heads, &new_state.heads, p, &u.name)?;
451
        }
452
        if let Some(sa) = &u.attestation {
453
            let last = self.latest_ref_attest(&u.name).map_err(|e| e.to_string())?.and_then(|(_, a)| match a.attest.body {
454
                AttestBody::Ref { serial, .. } => Some(serial),
455
                _ => None,
456
            });
457
            self.check_ref_attest(sa, &u.name, &u.old_state, &u.new_state, last, false)?;
458
            let r = self.store.put_attest(&sa.attest).map_err(|e| e.to_string())?;
459
            for sig in &sa.sigs {
460
                self.store.put_signature(&r, sig).map_err(|e| e.to_string())?;
461
            }
462
            self.set_ref_attest(&u.name, &r).map_err(|e| e.to_string())?;
463
        }
464
        view.bookmarks.insert(u.name.clone(), u.new_state);
465
        Ok(())
466
    }
467
468
    // ------------------------------------------------------------ client side
469
470
    fn remote_url(&self, name: &str) -> Result<String> {
471
        self.config.remotes.get(name).cloned().ok_or_else(|| RepoError::Msg(format!("no remote named {name:?}; add one with `oxy remote add`")))
472
    }
473
474
    /// Credentials and trust for a remote: `remote.<name>.token` (else
475
    /// `$OXY_TOKEN`) and `remote.<name>.ca` (else `$OXY_CA`).
476
    fn remote_options(&self, name: &str) -> RemoteOptions {
477
        RemoteOptions {
478
            token: self.config.remote_tokens.get(name).cloned().or_else(|| std::env::var("OXY_TOKEN").ok()),
479
            ca: self.config.remote_cas.get(name).map(PathBuf::from).or_else(|| std::env::var("OXY_CA").ok().map(PathBuf::from)),
480
        }
481
    }
482
483
    fn open_named_remote(&self, name: &str) -> Result<Box<dyn crate::protocol::Remote>> {
484
        open_remote(&self.remote_url(name)?, &self.remote_options(name))
485
    }
486
487
    fn serial_path(&self, remote: &str, bookmark: &str) -> PathBuf {
488
        self.oxy.join("remotes").join(remote).join(format!("{bookmark}.serial"))
489
    }
490
491
    fn last_seen_serial(&self, remote: &str, bookmark: &str) -> Option<u64> {
492
        fs::read_to_string(self.serial_path(remote, bookmark)).ok().and_then(|s| s.trim().parse().ok())
493
    }
494
495
    fn record_serial(&self, remote: &str, bookmark: &str, serial: u64) -> Result<()> {
496
        let p = self.serial_path(remote, bookmark);
497
        fs::create_dir_all(p.parent().unwrap())?;
498
        write_atomic(&p, serial.to_string().as_bytes())
499
    }
500
501
    fn policy_serial_path(&self, remote: &str) -> PathBuf {
502
        self.oxy.join("remotes").join(remote).join("policy.serial")
503
    }
504
505
    /// The highest policy serial `remote` has ever presented.
506
    fn remote_policy_serial(&self, remote: &str) -> Option<u64> {
507
        fs::read_to_string(self.policy_serial_path(remote)).ok().and_then(|s| s.trim().parse().ok())
508
    }
509
510
    fn record_policy_serial(&self, remote: &str, serial: u64) -> Result<()> {
511
        let p = self.policy_serial_path(remote);
512
        fs::create_dir_all(p.parent().unwrap())?;
513
        write_atomic(&p, serial.to_string().as_bytes())
514
    }
515
516
    fn lazy_dir(&self) -> PathBuf {
517
        self.oxy.join("lazy")
518
    }
519
520
    /// Remember that `blob` was withheld by `remote` (spec §13.5).
521
    fn register_lazy(&self, blob: &ObjectRef, remote: &str) -> Result<()> {
522
        fs::create_dir_all(self.lazy_dir())?;
523
        write_atomic(&self.lazy_dir().join(blob.to_hex()), remote.as_bytes())
524
    }
525
526
    /// Which remote promised an absent blob, if any.
527
    pub fn lazy_remote(&self, blob: &ObjectRef) -> Option<String> {
528
        fs::read_to_string(self.lazy_dir().join(blob.to_hex())).ok().map(|s| s.trim().to_string())
529
    }
530
531
    /// Fetch an absent blob (and its chunks) from the remote that promised it.
532
    pub(crate) fn fetch_lazy_blob(&self, blob: &ObjectRef) -> Result<()> {
533
        let Some(remote_name) = self.lazy_remote(blob) else {
534
            return Err(RepoError::Store(StoreError::NotFound(*blob)));
535
        };
536
        let remote = self.open_named_remote(&remote_name)?;
537
        let objects = remote.objects(&[*blob])?;
538
        for (ty, body) in objects {
539
            if matches!(ty, ObjectType::Blob | ObjectType::Chunk) {
540
                self.store.put(ty, &body)?;
541
            }
542
        }
543
        if !self.store.has(blob) {
544
            return fail(format!("{remote_name} did not send blob {}", blob.short(12)));
545
        }
546
        let _ = fs::remove_file(self.lazy_dir().join(blob.to_hex()));
547
        Ok(())
548
    }
549
550
    /// After importing from `remote`, record every blob the changes reference
551
    /// that the store lacks as lazily available from that remote.
552
    fn register_missing_blobs(&self, changes: &[ChangeHash], remote: &str) -> Result<usize> {
553
        let mut n = 0;
554
        for h in changes {
555
            let c = self.store.get_change(h)?;
556
            for e in &c.edits {
557
                if let Edit::Replace { old, new, .. } = e {
558
                    for b in [old, new].into_iter().flatten() {
559
                        if !self.store.has(b) {
560
                            self.register_lazy(b, remote)?;
561
                            n += 1;
562
                        }
563
                    }
564
                }
565
            }
566
        }
567
        Ok(n)
568
    }
569
570
    /// Every head this repository knows: bookmarks, remote tracking, working copy.
571
    fn known_heads(&self, view: &View) -> Result<Vec<ChangeHash>> {
572
        let mut have: BTreeSet<ChangeHash> = BTreeSet::new();
573
        for st in view.bookmarks.values().chain(view.remotes.values().flat_map(|b| b.values())) {
574
            if let Ok(s) = self.store.get_state(st) {
575
                have.extend(s.heads);
576
            }
577
        }
578
        have.extend(view.workspaces.values().copied());
579
        Ok(have.into_iter().collect())
580
    }
581
582
    /// Fetch every bookmark of a remote into `remotes/<name>/bookmarks/*` (spec §10.2).
583
    pub fn fetch(&self, remote_name: &str) -> Result<FetchReport> {
584
        let remote = self.open_named_remote(remote_name)?;
585
        if remote.repo_id()? != self.id()? {
586
            return fail(format!("remote {remote_name:?} is a different repository (repository IDs differ)"));
587
        }
588
        let resp = remote.refs()?;
589
        // Trust policy (spec §14.5). Adopt the remote's policy when it is newer
590
        // and signed by roots we trust, or on first contact. A remote that
591
        // once presented a policy may never present none or an older one:
592
        // that is a downgrade. A remote that never presented one is
593
        // unverified, and this fetch is checked only as far as possible.
594
        let local_policy = self.current_policy()?.map(|(_, sp)| sp.policy);
595
        let seen = self.remote_policy_serial(remote_name);
596
        let mut unverified = false;
597
        match &resp.policy {
598
            Some(sp) => {
599
                if seen.is_some_and(|s| sp.policy.serial < s) {
600
                    return fail(format!("{remote_name} presents policy serial {} but presented {} before: refusing the downgrade", sp.policy.serial, seen.unwrap()));
601
                }
602
                match &local_policy {
603
                    Some(l) if sp.policy.serial < l.serial => {}
604
                    Some(l) if sp.policy.serial == l.serial => {
605
                        if sp.policy.hash() != l.hash() {
606
                            return fail(format!("{remote_name} presents a different policy with the same serial {}", l.serial));
607
                        }
608
                    }
609
                    _ => {
610
                        self.adopt_policy_chain(&resp.history, sp, &format!("oxy fetch {remote_name}")).map_err(|e| RepoError::Msg(format!("policy from {remote_name}: {e}")))?;
611
                    }
612
                }
613
                self.record_policy_serial(remote_name, sp.policy.serial)?;
614
            }
615
            None => {
616
                if seen.is_some() {
617
                    return fail(format!("{remote_name} presented a policy before but has none now: refusing the downgrade"));
618
                }
619
                unverified = true;
620
            }
621
        }
622
        let policy = if unverified { None } else { self.current_policy()?.map(|(_, sp)| sp.policy) };
623
        let (op_ref, op) = self.head_op()?;
624
        let mut view = op.view.clone();
625
        let have = self.known_heads(&view)?;
626
        let mut report = Vec::new();
627
        let max_blob = self.config.max_blob;
628
        for r in resp.refs {
629
            let needed = remote.negotiate(&have, &r.heads)?;
630
            let bundle = remote.fetch(&needed, max_blob)?;
631
            // Changes are not individually re-verified here: the bookmark's
632
            // attestation (checked below) binds its whole state, and the
633
            // receiver of the push verified the added changes at that time.
634
            let imported = self.import_bundle(&bundle, false)?;
635
            if max_blob.is_some() {
636
                self.register_missing_blobs(&imported, remote_name)?;
637
            }
638
            for h in &r.heads {
639
                if !self.store.has(h) {
640
                    return fail(format!("remote did not send head {}", h.short(12)));
641
                }
642
            }
643
            let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
644
            if state != r.state {
645
                return fail(format!("remote state {} does not match its heads", r.state.short(12)));
646
            }
647
            if let Some(p) = &policy {
648
                let Some(sa) = &r.attestation else {
649
                    return fail(format!("{remote_name}/{} has no attestation, which its policy requires", r.name));
650
                };
651
                let signer = sa.sigs.first().map(|s| s.key).ok_or_else(|| RepoError::Msg("attestation carries no signature".into()))?;
652
                if !p.authorizes(&signer, &r.name) {
653
                    return fail(format!("{remote_name}/{}: attested by {}, which the policy does not authorize", r.name, signer.short(12)));
654
                }
655
            }
656
            if let Some(sa) = &r.attestation {
657
                let AttestBody::Ref { old_state, new_state, .. } = &sa.attest.body else {
658
                    return fail("remote sent a non-ref attestation for a bookmark");
659
                };
660
                let (old_state, new_state) = (*old_state, *new_state);
661
                let serial = self
662
                    .check_ref_attest(sa, &r.name, &old_state, &new_state, self.last_seen_serial(remote_name, &r.name), true)
663
                    .map_err(|e| RepoError::Msg(format!("attestation for {}/{}: {e}", remote_name, r.name)))?;
664
                if new_state != r.state {
665
                    return fail(format!("attestation for {}/{} does not cover its current state", remote_name, r.name));
666
                }
667
                let ar = self.store.put_attest(&sa.attest)?;
668
                for sig in &sa.sigs {
669
                    self.store.put_signature(&ar, sig)?;
670
                }
671
                self.record_serial(remote_name, &r.name, serial)?;
672
            }
673
            view.remotes.entry(remote_name.to_string()).or_default().insert(r.name.clone(), state);
674
            report.push((r.name, r.heads, imported.len()));
675
        }
676
        let mut new_tags = 0;
677
        for t in &resp.tags {
678
            if view.tags.contains_key(&t.name) {
679
                continue;
680
            }
681
            // The tag's changes must be here: fetch them if the bookmarks did not bring them.
682
            let missing: Vec<ChangeHash> = t.heads.iter().filter(|h| !self.store.has(h)).copied().collect();
683
            if !missing.is_empty() {
684
                let needed = remote.negotiate(&self.known_heads(&view)?, &missing)?;
685
                let bundle = remote.fetch(&needed, max_blob)?;
686
                self.import_bundle(&bundle, false)?;
687
            }
688
            self.store.put_state(&StateObject::new(t.heads.clone()))?;
689
            self.adopt_tag(&mut view, t, policy.as_ref()).map_err(|e| RepoError::Msg(format!("{remote_name} {}: {e}", t.name)))?;
690
            new_tags += 1;
691
        }
692
        if view != op.view {
693
            self.record_op(&[op_ref], &format!("oxy fetch {remote_name}"), view)?;
694
        }
695
        let _ = self.run_hook(
696
            "post-fetch",
697
            &format!(
698
                "{{\"event\":\"post-fetch\",\"remote\":{},\"bookmarks\":{}}}",
699
                json_str(remote_name),
700
                json_array(report.iter().map(|(n, heads, _)| format!("{{\"name\":{},\"heads\":{}}}", json_str(n), json_array(heads.iter().map(|h| json_str(&h.to_hex()))))))
701
            ),
702
        );
703
        Ok(FetchReport { refs: report, unverified, new_tags })
704
    }
705
706
    /// Shallow fetch of the remote's bookmark: its checkpoint as our base,
707
    /// the changes within `depth` steps of its heads, and the boundary below.
708
    pub fn fetch_shallow(&self, remote_name: &str, depth: u32) -> Result<usize> {
709
        let remote = self.open_named_remote(remote_name)?;
710
        if remote.repo_id()? != self.id()? {
711
            return fail(format!("remote {remote_name:?} is a different repository (repository IDs differ)"));
712
        }
713
        let resp = remote.refs()?;
714
        let bookmark = self.config.bookmark.clone();
715
        if let Some(sp) = &resp.policy {
716
            self.adopt_policy_chain(&resp.history, sp, "oxy clone --depth").map_err(|e| RepoError::Msg(format!("policy from {remote_name}: {e}")))?;
717
            self.record_policy_serial(remote_name, sp.policy.serial)?;
718
        }
719
        let Some(r) = resp.refs.into_iter().find(|r| r.name == bookmark) else {
720
            return fail(format!("{remote_name} has no bookmark {bookmark:?}"));
721
        };
722
        if r.heads.is_empty() {
723
            return Ok(0);
724
        }
725
        // Base: the remote's checkpoint of exactly these heads.
726
        let cp_bundle = remote.checkpoint(&r.heads)?;
727
        let mut cp_ref = None;
728
        for (ty, body) in &cp_bundle.objects {
729
            let put = self.store.put(*ty, body)?;
730
            if *ty == ObjectType::Checkpoint {
731
                cp_ref = Some(put);
732
            }
733
        }
734
        let Some(cp_ref) = cp_ref else { return fail(format!("{remote_name} sent no checkpoint")) };
735
        self.install_checkpoint(&bookmark, &cp_ref)?;
736
        // History window.
737
        let wanted = remote.shallow(&r.heads, depth)?;
738
        let bundle = remote.fetch(&wanted, self.config.max_blob)?;
739
        let imported = self.import_bundle_shallow(&bundle, false)?;
740
        if self.config.max_blob.is_some() {
741
            self.register_missing_blobs(&imported, remote_name)?;
742
        }
743
        self.refresh_boundary(&imported)?;
744
        if let Some(sa) = &r.attestation {
745
            let ar = self.store.put_attest(&sa.attest)?;
746
            for sig in &sa.sigs {
747
                self.store.put_signature(&ar, sig)?;
748
            }
749
            if let AttestBody::Ref { serial, .. } = sa.attest.body {
750
                self.record_serial(remote_name, &r.name, serial)?;
751
            }
752
        }
753
        let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
754
        let (op_ref, op) = self.head_op()?;
755
        let mut view = op.view.clone();
756
        view.remotes.entry(remote_name.to_string()).or_default().insert(r.name.clone(), state);
757
        self.record_op(&[op_ref], &format!("oxy fetch {remote_name} --depth {depth}"), view)?;
758
        Ok(imported.len())
759
    }
760
761
    /// Fetch, then take the union of the local bookmark's state with the
762
    /// remote's: no merge change, conflicts become reified (spec §9.4).
763
    pub fn pull(&self, remote_name: &str) -> Result<PullReport> {
764
        self.fetch(remote_name)?;
765
        let (op_ref, op) = self.head_op()?;
766
        let bookmark = self.config.bookmark.clone();
767
        let local = self.heads_of(&op.view, &bookmark)?;
768
        let Some(remote_state) = op.view.remotes.get(remote_name).and_then(|b| b.get(&bookmark)).copied() else {
769
            return Ok(PullReport { heads: local, changed: false, checked_out: false });
770
        };
771
        let remote_heads = self.store.get_state(&remote_state)?.heads;
772
        let mut all = local.clone();
773
        all.extend(remote_heads);
774
        let st = self.load_state(&all)?;
775
        let mut heads = st.maximal(&all);
776
        heads.sort();
777
        let mut sorted_local = local.clone();
778
        sorted_local.sort();
779
        if heads == sorted_local {
780
            return Ok(PullReport { heads, changed: false, checked_out: false });
781
        }
782
        let mut view = op.view.clone();
783
        view.bookmarks.insert(bookmark.clone(), self.store.put_state(&StateObject::new(heads.clone()))?);
784
        self.record_op(&[op_ref], &format!("oxy pull {remote_name}"), view)?;
785
        // Follow along if the working copy was clean and on the old heads.
786
        let snap = self.snapshot("oxy pull")?;
787
        let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
788
        let mut base = snap.base.maximal(&snap.wc.deps);
789
        base.sort();
790
        let mut checked_out = false;
791
        if clean && base == sorted_local {
792
            self.checkout(&heads, false, "oxy pull (checkout)")?;
793
            checked_out = true;
794
        }
795
        Ok(PullReport { heads, changed: true, checked_out })
796
    }
797
798
    /// Push the current bookmark (spec §16.3), sealing this author's unsigned
799
    /// changes and attesting the reference when a key is available.
800
    pub fn push(&self, remote_name: &str, force: bool) -> Result<Vec<RefResult>> {
801
        let bookmark = self.config.bookmark.clone();
802
        self.push_bookmark(remote_name, &bookmark, force)
803
    }
804
805
    /// `push` for a named bookmark (`oxy push -b NAME`).
806
    pub fn push_bookmark(&self, remote_name: &str, bookmark: &str, force: bool) -> Result<Vec<RefResult>> {
807
        let (op_ref, op) = self.head_op()?;
808
        if !op.view.bookmarks.contains_key(bookmark) {
809
            return fail(format!("no bookmark {bookmark}"));
810
        }
811
        let bookmark = bookmark.to_string();
812
        let local_heads = self.heads_of(&op.view, &bookmark)?;
813
        // pre-push hook: may veto before the remote is contacted.
814
        self.run_hook(
815
            "pre-push",
816
            &format!("{{\"event\":\"pre-push\",\"remote\":{},\"bookmark\":{},\"heads\":{}}}", json_str(remote_name), json_str(&bookmark), json_array(local_heads.iter().map(|h| json_str(&h.to_hex())))),
817
        )?;
818
        let remote = self.open_named_remote(remote_name)?;
819
        if remote.repo_id()? != self.id()? {
820
            return fail(format!("remote {remote_name:?} is a different repository (repository IDs differ)"));
821
        }
822
        let resp = remote.refs()?;
823
        let local_policy = self.current_policy()?.map(|(_, sp)| sp);
824
        let resp_policy_serial = resp.policy.as_ref().map(|p| p.policy.serial);
825
        let policy_to_send = match (&local_policy, &resp.policy) {
826
            (Some(l), Some(r)) if l.policy.serial > r.policy.serial => Some(l.clone()),
827
            (Some(l), None) => Some(l.clone()),
828
            (Some(l), Some(r)) if l.policy.serial < r.policy.serial => {
829
                return fail(format!("{remote_name} has a newer policy (serial {}); fetch first", r.policy.serial));
830
            }
831
            _ => None,
832
        };
833
        let remote_has_policy = resp.policy.is_some() || policy_to_send.is_some();
834
        if remote_has_policy && self.secret_key()?.is_none() {
835
            return fail(format!("{remote_name} enforces a trust policy; pushing needs an identity (`oxy identity create`)"));
836
        }
837
        let remote_tags: BTreeSet<String> = resp.tags.iter().map(|t| t.name.clone()).collect();
838
        let tags_to_send: Vec<crate::protocol::TagRef> = self.tag_refs()?.into_iter().filter(|t| !remote_tags.contains(&t.name)).collect();
839
        let refs = resp.refs;
840
        let empty = self.store.put_state(&StateObject::new(vec![]))?;
841
        let (remote_state, remote_heads, last_serial) = match refs.iter().find(|r| r.name == bookmark) {
842
            Some(r) => (r.state, r.heads.clone(), r.last_serial),
843
            None => (empty, vec![], None),
844
        };
845
        let new_state = self.store.put_state(&StateObject::new(local_heads.clone()))?;
846
        // A push that carries a policy re-attests the bookmark even when its
847
        // state is unchanged, so the current attestation is always by a key
848
        // the current policy authorizes (a revoked key's attestation would
849
        // otherwise be refused by every later fetch).
850
        if new_state == remote_state && policy_to_send.is_none() && tags_to_send.is_empty() {
851
            return Ok(vec![RefResult { name: bookmark, ok: true, reason: "already up to date".into() }]);
852
        }
853
        for h in &remote_heads {
854
            if !self.store.has(h) {
855
                return fail(format!("the remote has change {} which you do not; pull first", h.short(12)));
856
            }
857
        }
858
        if !force {
859
            let closure = self.closure_in_store(&local_heads)?;
860
            if let Some(h) = remote_heads.iter().find(|h| !closure.contains(h)) {
861
                return fail(format!("not a fast-forward: remote head {} is not in your state; pull first or use --force", h.short(12)));
862
            }
863
        }
864
        let to_send = self.negotiate(&remote_heads, &local_heads)?;
865
        let mut bundle = self.bundle_for(&to_send)?;
866
        let key = self.secret_key()?.map(|(k, _)| k);
867
        if let Some(key) = key {
868
            // Seal this author's changes that carry no valid authenticator.
869
            let mut unsigned = Vec::new();
870
            for h in &to_send {
871
                let c = self.store.get_change(h)?;
872
                if c.author == key && !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. }) {
873
                    unsigned.push(*h);
874
                }
875
            }
876
            for chunk in unsigned.chunks(crate::attest::MAX_SEAL_HASHES) {
877
                let sa = self.sign_attest(&Attest::seal(self.id()?, chunk.to_vec()))?;
878
                bundle.attests.push(sa);
879
            }
880
            bundle = self.bundle_for(&to_send)?;
881
        }
882
        bundle.objects.push((ObjectType::State, StateObject::new(local_heads.clone()).encode()));
883
        let attestation = match key {
884
            Some(_) => Some(self.sign_attest(&Attest {
885
                repo: self.id()?,
886
                hlc: Hlc::now(),
887
                body: AttestBody::Ref { name: bookmark.clone(), old_state: remote_state, new_state, serial: last_serial.unwrap_or(0) + 1 },
888
            })?),
889
            None => None,
890
        };
891
        let serial = attestation.as_ref().and_then(|a| match a.attest.body {
892
            AttestBody::Ref { serial, .. } => Some(serial),
893
            _ => None,
894
        });
895
        let sent_serial = policy_to_send.as_ref().map(|p| p.policy.serial);
896
        // Tag states must be present on the remote: bundle their changes too.
897
        for t in &tags_to_send {
898
            let extra = self.negotiate(&remote_heads, &t.heads)?;
899
            let extra: Vec<ChangeHash> = extra.into_iter().filter(|h| !to_send.contains(h)).collect();
900
            if !extra.is_empty() {
901
                let more = self.bundle_for(&extra)?;
902
                bundle.changes.extend(more.changes);
903
                bundle.attests.extend(more.attests);
904
                bundle.objects.extend(more.objects);
905
            }
906
            bundle.objects.push((ObjectType::State, StateObject::new(t.heads.clone()).encode()));
907
        }
908
        if let Some(l) = &policy_to_send {
909
            let remote_serial = resp_policy_serial;
910
            bundle.policies = self.policy_history()?.into_iter().filter(|sp| remote_serial.is_none_or(|r| sp.policy.serial > r) && sp.policy.serial < l.policy.serial).collect();
911
        }
912
        let results = remote.push(bundle, vec![RefUpdate { name: bookmark.clone(), old_state: remote_state, new_state, force, attestation }], policy_to_send, tags_to_send)?;
913
        if let (Some(serial), true) = (sent_serial, results.iter().any(|r| r.ok && r.name == "policy")) {
914
            self.record_policy_serial(remote_name, serial)?;
915
        }
916
        if results.iter().any(|r| r.ok && r.name == bookmark) {
917
            let mut view = op.view.clone();
918
            view.remotes.entry(remote_name.to_string()).or_default().insert(bookmark.clone(), new_state);
919
            self.record_op(&[op_ref], &format!("oxy push {remote_name}"), view)?;
920
            if let Some(s) = serial {
921
                self.record_serial(remote_name, &bookmark, s)?;
922
            }
923
        }
924
        Ok(results)
925
    }
926
927
    /// Clone a remote into `dest`: same repository ID, `origin` remote,
928
    /// bookmark set to the remote's, working copy checked out.
929
    pub fn clone(url: &str, dest: &Path) -> Result<Repo> {
930
        Self::clone_with(url, dest, None, &[])
931
    }
932
933
    /// Clone, leaving blobs above `max_blob` bytes to be fetched on first
934
    /// read, and materializing only `sparse` paths (which implies lazy blobs
935
    /// unless a size is given, so out-of-scope binaries are never fetched).
936
    pub fn clone_with(url: &str, dest: &Path, max_blob: Option<u64>, sparse: &[String]) -> Result<Repo> {
937
        Self::clone_full(url, dest, max_blob, sparse, None)
938
    }
939
940
    /// Shallow clone (spec §17): a checkpoint of the bookmark's heads plus the
941
    /// changes within `depth` steps of them; the rest is the boundary.
942
    pub fn clone_shallow(url: &str, dest: &Path, depth: u32, max_blob: Option<u64>, sparse: &[String]) -> Result<Repo> {
943
        Self::clone_full(url, dest, max_blob, sparse, Some(depth))
944
    }
945
946
    fn clone_full(url: &str, dest: &Path, max_blob: Option<u64>, sparse: &[String], depth: Option<u32>) -> Result<Repo> {
947
        let options = RemoteOptions {
948
            token: std::env::var("OXY_TOKEN").ok(),
949
            ca: std::env::var("OXY_CA").ok().map(PathBuf::from),
950
        };
951
        let remote = open_remote(url, &options)?;
952
        let id = remote.repo_id()?;
953
        let mut repo = Repo::init_with_id(dest, id)?;
954
        repo.config.set("remote.origin.url", url)?;
955
        if let Some(m) = max_blob {
956
            repo.config.set("filter.max-blob", &m.to_string())?;
957
        } else if !sparse.is_empty() {
958
            repo.config.set("filter.max-blob", "0")?;
959
        }
960
        if !sparse.is_empty() {
961
            repo.config.set("filter.paths", &sparse.join(","))?;
962
        }
963
        repo.save_config()?;
964
        if let Some(depth) = depth {
965
            repo.fetch_shallow("origin", depth)?;
966
        } else {
967
            repo.fetch("origin")?;
968
        }
969
        let (op_ref, op) = repo.head_op()?;
970
        let bookmark = repo.config.bookmark.clone();
971
        if let Some(state) = op.view.remotes.get("origin").and_then(|b| b.get(&bookmark)).copied() {
972
            let heads = repo.store.get_state(&state)?.heads;
973
            let mut view = op.view.clone();
974
            view.bookmarks.insert(bookmark, state);
975
            repo.record_op(&[op_ref], &format!("oxy clone {url}"), view)?;
976
            repo.checkout(&heads, false, "oxy clone (checkout)")?;
977
        }
978
        Ok(repo)
979
    }
980
}