OXYGENHUB
# 3faccd90ddfa oxygen / crates / oxy-core / src / repo / sync.rs
55.6 KiB · Rust · last changed Roadmap: phase D done 1 month ago source blame history raw
98286406 Andy McCall ↶ 1
//! Serving and talking to remotes (spec §10.2, §16).
2
3
use super::*;
4
7aa2dc93 Andy McCall ↶ 5
/// What a push did, for hosts (hub spec §6.12).
6
#[derive(Clone, Debug, Default)]
7
pub struct PushReport {
8
    pub results: Vec<RefResult>,
9
    /// Accepted reference updates, in request order.
10
    pub updated: Vec<RefUpdated>,
11
    /// Changes this push stored that were not already present.
12
    pub imported: Vec<ChangeHash>,
13
}
14
15
#[derive(Clone, Debug)]
16
pub struct RefUpdated {
17
    pub name: String,
18
    pub old_state: ObjectRef,
19
    pub new_state: ObjectRef,
20
    pub heads: Vec<ChangeHash>,
21
}
22
83e0677b Andy McCall ↶ 23
/// What a fetch brought in.
24
#[derive(Debug, Clone, PartialEq, Eq, Default)]
25
pub struct FetchReport {
26
    /// Bookmark name, its heads, and how many changes were new.
27
    pub refs: Vec<(String, Vec<ChangeHash>, usize)>,
132b884b Andy McCall ↶ 28
    /// Tags adopted from the remote.
29
    pub new_tags: usize,
83e0677b Andy McCall ↶ 30
    /// The remote has never presented a policy, so nothing was verified
31
    /// against one (spec §14.5: warn loudly).
32
    pub unverified: bool,
33
}
34
98286406 Andy McCall ↶ 35
#[derive(Debug, Clone, PartialEq, Eq)]
36
pub struct PullReport {
37
    pub heads: Vec<ChangeHash>,
38
    pub changed: bool,
39
    pub checked_out: bool,
40
}
41
42
impl Repo {
43
    // ------------------------------------------------------------ serving (spec §16)
44
45
    fn ref_attest_path(&self, bookmark: &str) -> PathBuf {
46
        self.oxy.join("attest").join(bookmark)
47
    }
48
49
    /// The latest reference attestation stored for a bookmark.
50
    pub fn latest_ref_attest(&self, bookmark: &str) -> Result<Option<(ObjectRef, SignedAttest)>> {
51
        match fs::read_to_string(self.ref_attest_path(bookmark)) {
52
            Ok(h) => {
75167672 Andy McCall 53
                let r = ObjectRef::from_hex(h.trim())
54
                    .ok_or_else(|| RepoError::Msg("corrupt attestation pointer".into()))?;
55
                Ok(Some((
56
                    r,
57
                    SignedAttest {
58
                        attest: self.store.get_attest(&r)?,
59
                        sigs: self.store.signatures(&r)?,
60
                    },
61
                )))
98286406 Andy McCall ↶ 62
            }
63
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
64
            Err(e) => Err(e.into()),
65
        }
66
    }
67
68
    fn set_ref_attest(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
69
        fs::create_dir_all(self.oxy.join("attest"))?;
70
        write_atomic(&self.ref_attest_path(bookmark), r.to_hex().as_bytes())
71
    }
72
83e0677b Andy McCall ↶ 73
    pub fn remote_refs(&self) -> Result<RefsResponse> {
98286406 Andy McCall ↶ 74
        let view = self.view()?;
75
        let mut out = Vec::new();
76
        for (name, state) in &view.bookmarks {
77
            let latest = self.latest_ref_attest(name)?.map(|(_, a)| a);
78
            let last_serial = latest.as_ref().and_then(|a| match a.attest.body {
79
                AttestBody::Ref { serial, .. } => Some(serial),
80
                _ => None,
81
            });
82
            let attestation = latest.filter(|a| matches!(&a.attest.body, AttestBody::Ref { new_state, .. } if new_state == state));
75167672 Andy McCall 83
            out.push(RemoteRef {
84
                name: name.clone(),
85
                state: *state,
86
                heads: self.store.get_state(state)?.heads,
87
                attestation,
88
                last_serial,
89
            });
98286406 Andy McCall ↶ 90
        }
28fc23d7 Andy McCall ↶ 91
        let policy = self.current_policy()?.map(|(_, sp)| sp);
75167672 Andy McCall 92
        let history = self
93
            .policy_history()?
94
            .into_iter()
95
            .filter(|sp| {
96
                policy
97
                    .as_ref()
98
                    .is_some_and(|p| sp.policy.serial < p.policy.serial)
99
            })
100
            .collect();
101
        Ok(RefsResponse {
102
            refs: out,
103
            policy,
104
            tags: self.tag_refs()?,
105
            history,
106
        })
98286406 Andy McCall ↶ 107
    }
108
109
    /// Every change reachable from `heads` that is in the store.
110
    fn closure_in_store(&self, heads: &[ChangeHash]) -> Result<BTreeSet<ChangeHash>> {
111
        let mut seen = BTreeSet::new();
112
        let mut stack: Vec<ChangeHash> = heads.to_vec();
113
        while let Some(h) = stack.pop() {
114
            if seen.contains(&h) || !self.store.has(&h) {
115
                continue;
116
            }
117
            seen.insert(h);
118
            stack.extend(self.store.get_change(&h)?.deps);
119
        }
120
        Ok(seen)
121
    }
122
2863456a Andy McCall ↶ 123
    /// The changes within `depth` dependency steps of `heads` (spec §17),
124
    /// nearest first; depth 1 is the heads themselves.
125
    pub fn shallow_changes(&self, heads: &[ChangeHash], depth: u32) -> Result<Vec<ChangeHash>> {
126
        let mut out = Vec::new();
127
        let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
128
        let mut layer: Vec<ChangeHash> = heads.to_vec();
129
        for _ in 0..depth.max(1) {
130
            let mut next = Vec::new();
131
            for h in layer {
132
                if !seen.insert(h) || !self.store.has(&h) {
133
                    continue;
134
                }
135
                let c = self.store.get_change(&h)?;
136
                next.extend(c.deps.iter().copied());
137
                out.push(h);
138
            }
139
            if next.is_empty() {
140
                break;
141
            }
142
            layer = next;
143
        }
144
        out.reverse();
145
        Ok(out)
146
    }
147
148
    fn shallow_path(&self) -> PathBuf {
149
        self.oxy.join("shallow")
150
    }
151
152
    /// The shallow boundary: dependencies this repository does not hold (spec §17).
153
    pub fn boundary(&self) -> BTreeSet<ChangeHash> {
75167672 Andy McCall 154
        fs::read_to_string(self.shallow_path())
155
            .map(|t| t.lines().filter_map(ObjectRef::from_hex).collect())
156
            .unwrap_or_default()
2863456a Andy McCall ↶ 157
    }
158
159
    pub fn is_shallow(&self) -> bool {
160
        !self.boundary().is_empty()
161
    }
162
163
    fn write_boundary(&self, boundary: &BTreeSet<ChangeHash>) -> Result<()> {
164
        if boundary.is_empty() {
165
            let _ = fs::remove_file(self.shallow_path());
166
            return Ok(());
167
        }
75167672 Andy McCall 168
        let text: String = boundary
169
            .iter()
170
            .map(|h| format!("{}\n", h.to_hex()))
171
            .collect();
2863456a Andy McCall ↶ 172
        write_atomic(&self.shallow_path(), text.as_bytes())
173
    }
174
175
    /// Recompute the boundary from what is stored: every dependency of a
176
    /// stored change that is not itself stored.
177
    fn refresh_boundary(&self, imported: &[ChangeHash]) -> Result<()> {
178
        let mut boundary = self.boundary();
179
        boundary.retain(|h| !self.store.has(h));
180
        for h in imported {
181
            for d in self.store.get_change(h)?.deps {
182
                if !self.store.has(&d) {
183
                    boundary.insert(d);
184
                }
185
            }
186
        }
187
        self.write_boundary(&boundary)
188
    }
189
190
    /// Fetch `depth` more levels of history below the shallow boundary.
191
    pub fn deepen(&self, remote_name: &str, depth: u32) -> Result<usize> {
192
        let boundary = self.boundary();
193
        if boundary.is_empty() {
194
            return Ok(0);
195
        }
196
        let remote = self.open_named_remote(remote_name)?;
197
        let wanted = remote.shallow(&boundary.iter().copied().collect::<Vec<_>>(), depth)?;
198
        let bundle = remote.fetch(&wanted, self.config.max_blob)?;
199
        let imported = self.import_bundle_shallow(&bundle, false)?;
200
        if self.config.max_blob.is_some() {
201
            self.register_missing_blobs(&imported, remote_name)?;
202
        }
203
        self.refresh_boundary(&imported)?;
204
        Ok(imported.len())
205
    }
206
98286406 Andy McCall ↶ 207
    /// `closure(want) \ closure(have)`, dependencies first (spec §16.2).
208
    pub fn negotiate(&self, have: &[ChangeHash], want: &[ChangeHash]) -> Result<Vec<ChangeHash>> {
209
        let have = self.closure_in_store(have)?;
2863456a Andy McCall ↶ 210
        // In a shallow repository the walk stops at the boundary, whose
211
        // changes the remote is assumed to hold.
212
        let boundary = self.boundary();
98286406 Andy McCall ↶ 213
        let mut out = Vec::new();
214
        let mut done: BTreeSet<ChangeHash> = have.clone();
2863456a Andy McCall ↶ 215
        done.extend(boundary.iter().copied());
98286406 Andy McCall ↶ 216
        // Iterative post-order DFS.
217
        let mut stack: Vec<(ChangeHash, bool)> = want.iter().rev().map(|h| (*h, false)).collect();
218
        while let Some((h, expanded)) = stack.pop() {
219
            if done.contains(&h) {
220
                continue;
221
            }
222
            if !self.store.has(&h) {
223
                return fail(format!("change {} is not in this repository", h.short(12)));
224
            }
225
            if expanded {
226
                done.insert(h);
227
                out.push(h);
228
                continue;
229
            }
230
            stack.push((h, true));
231
            for d in self.store.get_change(&h)?.deps {
232
                if !done.contains(&d) {
233
                    stack.push((d, false));
234
                }
235
            }
236
        }
237
        Ok(out)
238
    }
239
240
    /// Bundle changes (in the given dependency order) with their signatures,
241
    /// seals, identities, blobs, chunks.
242
    pub fn bundle_for(&self, changes: &[ChangeHash]) -> Result<Bundle> {
b3c5a59a Andy McCall ↶ 243
        self.bundle_for_filtered(changes, None)
244
    }
245
246
    /// Objects by reference for the `objects` endpoint; a blob brings its chunks.
247
    pub fn objects_for(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>> {
248
        let mut out = Vec::new();
249
        for r in refs {
250
            let (ty, body) = match self.store.get(r) {
251
                Ok(x) => x,
252
                Err(StoreError::NotFound(_)) => continue,
253
                Err(e) => return Err(e.into()),
254
            };
255
            if ty == ObjectType::Blob {
256
                for c in Blob::decode(&body)?.chunks {
257
                    if let Ok((cty, cb)) = self.store.get(&c) {
258
                        out.push((cty, cb));
259
                    }
260
                }
261
            }
262
            out.push((ty, body));
263
        }
264
        Ok(out)
265
    }
266
267
    /// Like `bundle_for`, leaving out blobs larger than `max_blob` bytes
268
    /// (and their chunks) so the receiver fetches them lazily (spec §13.5).
75167672 Andy McCall 269
    pub fn bundle_for_filtered(
270
        &self,
271
        changes: &[ChangeHash],
272
        max_blob: Option<u64>,
273
    ) -> Result<Bundle> {
98286406 Andy McCall ↶ 274
        let mut bundle = Bundle::default();
275
        let mut objects: BTreeSet<ObjectRef> = BTreeSet::new();
276
        let mut seals: BTreeSet<ObjectRef> = BTreeSet::new();
277
        let add_object = |r: &ObjectRef, objects: &mut BTreeSet<ObjectRef>| {
278
            if !r.is_zero() && self.store.has(r) {
279
                objects.insert(*r);
280
            }
281
        };
282
        for h in changes {
283
            let c = self.store.get_change(h)?;
284
            let sigs = self.store.signatures(h)?;
285
            add_object(&c.author, &mut objects);
286
            for s in &sigs {
287
                add_object(&s.key, &mut objects);
288
            }
289
            for e in &c.edits {
290
                if let Edit::Replace { old, new, .. } = e {
291
                    for b in [old, new].into_iter().flatten() {
292
                        if self.store.has(b) {
75167672 Andy McCall 293
                            if let Ok(blob) =
294
                                Blob::decode(&self.store.get_typed(b, ObjectType::Blob)?)
295
                            {
b3c5a59a Andy McCall ↶ 296
                                if max_blob.is_some_and(|m| blob.length > m) {
297
                                    continue;
298
                                }
299
                                objects.insert(*b);
98286406 Andy McCall ↶ 300
                                objects.extend(blob.chunks.iter().copied());
301
                            }
302
                        }
303
                    }
304
                }
305
            }
306
            for s in self.store.seals_for(h)? {
307
                seals.insert(s);
308
            }
309
            bundle.changes.push((c, sigs));
310
        }
311
        for r in seals {
312
            let attest = self.store.get_attest(&r)?;
313
            let sigs = self.store.signatures(&r)?;
314
            for s in &sigs {
315
                add_object(&s.key, &mut objects);
316
            }
317
            bundle.attests.push(SignedAttest { attest, sigs });
318
        }
319
        for r in objects {
320
            let (ty, body) = self.store.get(&r)?;
321
            bundle.objects.push((ty, body));
322
        }
323
        Ok(bundle)
324
    }
325
326
    /// Store a bundle's contents. With `strict`, every change must
327
    /// authenticate (signature or seal by its author or a delegate).
328
    pub fn import_bundle(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
2863456a Andy McCall ↶ 329
        self.import_bundle_inner(bundle, strict, false)
330
    }
331
332
    /// Like `import_bundle`, accepting changes whose dependencies are absent
333
    /// (they join the shallow boundary).
334
    pub fn import_bundle_shallow(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
335
        self.import_bundle_inner(bundle, strict, true)
336
    }
337
75167672 Andy McCall 338
    fn import_bundle_inner(
339
        &self,
340
        bundle: &Bundle,
341
        strict: bool,
342
        shallow: bool,
343
    ) -> Result<Vec<ChangeHash>> {
8f10a07a Andy McCall ↶ 344
        let batch = self.store.begin_batch()?;
345
        let result = self.import_bundle_batched(bundle, strict, shallow);
346
        batch.finish()?;
347
        result
348
    }
349
75167672 Andy McCall 350
    fn import_bundle_batched(
351
        &self,
352
        bundle: &Bundle,
353
        strict: bool,
354
        shallow: bool,
355
    ) -> Result<Vec<ChangeHash>> {
98286406 Andy McCall ↶ 356
        for (ty, body) in &bundle.objects {
357
            match ty {
75167672 Andy McCall 358
                ObjectType::Chunk
359
                | ObjectType::Blob
360
                | ObjectType::Identity
361
                | ObjectType::State
362
                | ObjectType::Checkpoint => {
98286406 Andy McCall ↶ 363
                    self.store.put(*ty, body)?;
364
                }
365
                other => return fail(format!("bundle may not carry {} objects", other.name())),
366
            }
367
        }
368
        for sa in &bundle.attests {
369
            let r = self.store.put_attest(&sa.attest)?;
370
            for sig in &sa.sigs {
371
                self.store.put_signature(&r, sig)?;
372
            }
373
            if let AttestBody::Seal { hashes } = &sa.attest.body {
374
                for h in hashes {
375
                    self.store.index_seal(h, &r)?;
376
                }
377
            }
378
        }
379
        let incoming: BTreeSet<ChangeHash> = bundle.changes.iter().map(|(c, _)| c.hash()).collect();
2863456a Andy McCall ↶ 380
        let boundary = self.boundary();
98286406 Andy McCall ↶ 381
        let mut imported = Vec::new();
382
        for (c, sigs) in &bundle.changes {
383
            let h = c.hash();
384
            for d in &c.deps {
75167672 Andy McCall 385
                if !self.store.has(d) && !incoming.contains(d) && !shallow && !boundary.contains(d)
386
                {
387
                    return fail(format!(
388
                        "change {} depends on {}, which is neither stored nor in the bundle",
389
                        h.short(12),
390
                        d.short(12)
391
                    ));
98286406 Andy McCall ↶ 392
                }
393
            }
394
            self.store.put_change(c)?;
395
            for sig in sigs {
396
                self.store.put_signature(&h, sig)?;
397
            }
398
            imported.push(h);
399
        }
400
        if strict {
401
            for h in &imported {
402
                if !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. }) {
75167672 Andy McCall 403
                    return fail(format!(
404
                        "change {} is not authenticated by its author",
405
                        h.short(12)
406
                    ));
98286406 Andy McCall ↶ 407
                }
408
            }
409
        }
410
        Ok(imported)
411
    }
412
413
    /// Verify a reference attestation against an update it claims to describe.
414
    /// `allow_equal` lets a client re-read the attestation it already has; a
415
    /// server receiving a push requires a strictly greater serial.
75167672 Andy McCall 416
    fn check_ref_attest(
417
        &self,
418
        sa: &SignedAttest,
419
        name: &str,
420
        old_state: &ObjectRef,
421
        new_state: &ObjectRef,
422
        last_serial: Option<u64>,
423
        allow_equal: bool,
424
    ) -> std::result::Result<u64, String> {
425
        let AttestBody::Ref {
426
            name: n,
427
            old_state: o,
428
            new_state: nw,
429
            serial,
430
        } = &sa.attest.body
431
        else {
98286406 Andy McCall ↶ 432
            return Err("attestation is not a ref attestation".into());
433
        };
434
        if n != name || o != old_state || nw != new_state {
435
            return Err("attestation does not describe this update".into());
436
        }
437
        if sa.attest.repo != self.id().map_err(|e| e.to_string())? {
438
            return Err("attestation is for a different repository".into());
439
        }
440
        if let Some(last) = last_serial {
441
            if *serial < last || (*serial == last && !allow_equal) {
75167672 Andy McCall 442
                return Err(format!(
443
                    "attestation serial {serial} is not greater than the last seen {last}"
444
                ));
98286406 Andy McCall ↶ 445
            }
446
        }
447
        let r = sa.attest.hash();
448
        let sig = sa.sigs.first().ok_or("attestation carries no signature")?;
75167672 Andy McCall 449
        let id = self
450
            .store
451
            .get_identity(&sig.key)
452
            .map_err(|_| format!("unknown key {}", sig.key.short(12)))?;
98286406 Andy McCall ↶ 453
        verify_signature(&id, sig, Domain::Attest, &r).map_err(|e| e.to_string())?;
454
        Ok(*serial)
455
    }
456
83e0677b Andy McCall ↶ 457
    /// Accept a push (spec §16.4). Without a trust policy every repository
98286406 Andy McCall ↶ 458
    /// is unverified, so unsigned changes and unattested updates are accepted;
83e0677b Andy McCall ↶ 459
    /// attestations that are present must still be valid. With a policy,
460
    /// every change and update is checked against it. A newer policy in the
461
    /// push is applied first when it is signed by the current roots.
75167672 Andy McCall 462
    pub fn receive_push(
463
        &self,
464
        bundle: Bundle,
465
        updates: Vec<RefUpdate>,
466
        policy: Option<SignedPolicy>,
467
        tags: Vec<crate::protocol::TagRef>,
468
    ) -> Result<Vec<RefResult>> {
119bbf50 Andy McCall ↶ 469
        Ok(self
470
            .receive_push_report(bundle, updates, policy, tags)?
471
            .results)
7aa2dc93 Andy McCall ↶ 472
    }
473
474
    /// `receive_push` with what a host wants to log: each accepted
475
    /// reference's old and new state and heads, and the changes this push
476
    /// stored (hub spec §6.12).
477
    pub fn receive_push_report(
478
        &self,
479
        bundle: Bundle,
480
        updates: Vec<RefUpdate>,
481
        policy: Option<SignedPolicy>,
482
        tags: Vec<crate::protocol::TagRef>,
483
    ) -> Result<PushReport> {
484
        let mut report = PushReport::default();
485
        let results = &mut report.results;
83e0677b Andy McCall ↶ 486
        if let Some(sp) = policy {
28fc23d7 Andy McCall ↶ 487
            match self.adopt_policy_chain(&bundle.policies, &sp, "push received") {
83e0677b Andy McCall ↶ 488
                Ok(()) => {
75167672 Andy McCall 489
                    results.push(RefResult {
490
                        name: "policy".into(),
491
                        ok: true,
492
                        reason: format!("serial {}", sp.policy.serial),
493
                    });
83e0677b Andy McCall ↶ 494
                }
495
                Err(e) => {
75167672 Andy McCall 496
                    results.push(RefResult {
497
                        name: "policy".into(),
498
                        ok: false,
499
                        reason: e,
500
                    });
83e0677b Andy McCall ↶ 501
                    for u in updates {
75167672 Andy McCall 502
                        results.push(RefResult {
503
                            name: u.name,
504
                            ok: false,
505
                            reason: "policy update was rejected".into(),
506
                        });
83e0677b Andy McCall ↶ 507
                    }
7aa2dc93 Andy McCall ↶ 508
                    return Ok(report);
83e0677b Andy McCall ↶ 509
                }
510
            }
511
        }
512
        let policy = self.current_policy()?.map(|(_, sp)| sp.policy);
7aa2dc93 Andy McCall ↶ 513
        match self.import_bundle(&bundle, policy.is_some()) {
514
            Ok(imported) => report.imported = imported,
515
            Err(e) => {
516
                for u in updates {
517
                    results.push(RefResult {
518
                        name: u.name,
519
                        ok: false,
520
                        reason: e.to_string(),
521
                    });
522
                }
523
                return Ok(report);
83e0677b Andy McCall ↶ 524
            }
525
        }
98286406 Andy McCall ↶ 526
        let (op_ref, op) = self.head_op()?;
527
        let mut view = op.view.clone();
528
        let mut names = Vec::new();
529
        for u in updates {
83e0677b Andy McCall ↶ 530
            let outcome = self.apply_update(&mut view, &u, policy.as_ref());
98286406 Andy McCall ↶ 531
            let (ok, reason) = match outcome {
532
                Ok(()) => (true, String::new()),
533
                Err(e) => (false, e),
534
            };
535
            if ok {
536
                names.push(u.name.clone());
119bbf50 Andy McCall ↶ 537
                let heads = self
538
                    .store
539
                    .get_state(&u.new_state)
540
                    .map(|s| s.heads)
541
                    .unwrap_or_default();
542
                report.updated.push(RefUpdated {
543
                    name: u.name.clone(),
544
                    old_state: u.old_state,
545
                    new_state: u.new_state,
546
                    heads,
547
                });
98286406 Andy McCall ↶ 548
            }
75167672 Andy McCall 549
            results.push(RefResult {
550
                name: u.name,
551
                ok,
552
                reason,
553
            });
98286406 Andy McCall ↶ 554
        }
132b884b Andy McCall ↶ 555
        for t in &tags {
556
            let name = format!("tags/{}", t.name);
557
            match self.adopt_tag(&mut view, t, policy.as_ref()) {
558
                Ok(()) => {
559
                    names.push(name.clone());
75167672 Andy McCall 560
                    results.push(RefResult {
561
                        name,
562
                        ok: true,
563
                        reason: String::new(),
564
                    });
132b884b Andy McCall ↶ 565
                }
75167672 Andy McCall 566
                Err(e) => results.push(RefResult {
567
                    name,
568
                    ok: false,
569
                    reason: e,
570
                }),
132b884b Andy McCall ↶ 571
            }
572
        }
98286406 Andy McCall ↶ 573
        if !names.is_empty() {
75167672 Andy McCall 574
            self.record_op(
575
                &[op_ref],
576
                &format!("push received: {}", names.join(" ")),
577
                view,
578
            )?;
98286406 Andy McCall ↶ 579
        }
7aa2dc93 Andy McCall ↶ 580
        Ok(report)
98286406 Andy McCall ↶ 581
    }
582
75167672 Andy McCall 583
    fn apply_update(
584
        &self,
585
        view: &mut View,
586
        u: &RefUpdate,
587
        policy: Option<&crate::policy::Policy>,
588
    ) -> std::result::Result<(), String> {
83e0677b Andy McCall ↶ 589
        if u.name == "policy" {
75167672 Andy McCall 590
            return Err(
591
                "the policy reference is updated by sending a policy, not a bookmark update".into(),
592
            );
83e0677b Andy McCall ↶ 593
        }
75167672 Andy McCall 594
        let empty = self
595
            .store
596
            .put_state(&StateObject::new(vec![]))
597
            .map_err(|e| e.to_string())?;
98286406 Andy McCall ↶ 598
        let current = view.bookmarks.get(&u.name).copied().unwrap_or(empty);
599
        if current != u.old_state {
600
            return Err("stale: the remote bookmark has moved; pull first".into());
601
        }
75167672 Andy McCall 602
        let new_state = self
603
            .store
604
            .get_state(&u.new_state)
605
            .map_err(|_| "new state object is missing".to_string())?;
98286406 Andy McCall ↶ 606
        for h in &new_state.heads {
607
            if !self.store.has(h) {
608
                return Err(format!("head {} is missing", h.short(12)));
609
            }
610
        }
611
        if !u.force {
75167672 Andy McCall 612
            let old_heads = self
613
                .store
614
                .get_state(&current)
615
                .map_err(|e| e.to_string())?
616
                .heads;
617
            let closure = self
618
                .closure_in_store(&new_state.heads)
619
                .map_err(|e| e.to_string())?;
98286406 Andy McCall ↶ 620
            if let Some(h) = old_heads.iter().find(|h| !closure.contains(h)) {
75167672 Andy McCall 621
                return Err(format!(
622
                    "not a fast-forward: current head {} would be dropped (use --force)",
623
                    h.short(12)
624
                ));
98286406 Andy McCall ↶ 625
            }
626
        }
83e0677b Andy McCall ↶ 627
        if let Some(p) = policy {
628
            let Some(sa) = &u.attestation else {
75167672 Andy McCall 629
                return Err(format!(
630
                    "{:?} requires a signed reference attestation under this repository's policy",
631
                    u.name
632
                ));
83e0677b Andy McCall ↶ 633
            };
75167672 Andy McCall 634
            let signer = sa
635
                .sigs
636
                .first()
637
                .map(|s| s.key)
638
                .ok_or("attestation carries no signature")?;
83e0677b Andy McCall ↶ 639
            if !p.authorizes(&signer, &u.name) {
75167672 Andy McCall 640
                return Err(format!(
641
                    "key {} may not update {:?} under this repository's policy",
642
                    signer.short(12),
643
                    u.name
644
                ));
83e0677b Andy McCall ↶ 645
            }
75167672 Andy McCall 646
            let old_heads = self
647
                .store
648
                .get_state(&current)
649
                .map_err(|e| e.to_string())?
650
                .heads;
83e0677b Andy McCall ↶ 651
            self.state_allowed(&old_heads, &new_state.heads, p, &u.name)?;
652
        }
98286406 Andy McCall ↶ 653
        if let Some(sa) = &u.attestation {
75167672 Andy McCall 654
            let last = self
655
                .latest_ref_attest(&u.name)
656
                .map_err(|e| e.to_string())?
657
                .and_then(|(_, a)| match a.attest.body {
658
                    AttestBody::Ref { serial, .. } => Some(serial),
659
                    _ => None,
660
                });
98286406 Andy McCall ↶ 661
            self.check_ref_attest(sa, &u.name, &u.old_state, &u.new_state, last, false)?;
75167672 Andy McCall 662
            let r = self
663
                .store
664
                .put_attest(&sa.attest)
665
                .map_err(|e| e.to_string())?;
98286406 Andy McCall ↶ 666
            for sig in &sa.sigs {
75167672 Andy McCall 667
                self.store
668
                    .put_signature(&r, sig)
669
                    .map_err(|e| e.to_string())?;
98286406 Andy McCall ↶ 670
            }
75167672 Andy McCall 671
            self.set_ref_attest(&u.name, &r)
672
                .map_err(|e| e.to_string())?;
98286406 Andy McCall ↶ 673
        }
674
        view.bookmarks.insert(u.name.clone(), u.new_state);
675
        Ok(())
676
    }
677
678
    // ------------------------------------------------------------ client side
679
680
    fn remote_url(&self, name: &str) -> Result<String> {
75167672 Andy McCall 681
        self.config.remotes.get(name).cloned().ok_or_else(|| {
682
            RepoError::Msg(format!(
683
                "no remote named {name:?}; add one with `oxy remote add`"
684
            ))
685
        })
98286406 Andy McCall ↶ 686
    }
687
2b5ee371 Andy McCall ↶ 688
    /// Credentials and trust for a remote: `remote.<name>.token` (else
689
    /// `$OXY_TOKEN`) and `remote.<name>.ca` (else `$OXY_CA`).
75167672 Andy McCall 690
    /// Refuse a remote that is another repository. An **unbound** remote
691
    /// (id all zero: a hosted repository nobody has pushed to) is accepted
692
    /// for pushes, which bind it to this repository's id (spec §16.1).
05e71ed6 Andy McCall ↶ 693
    fn check_remote_id(
694
        &self,
695
        remote: &dyn crate::protocol::Remote,
696
        remote_name: &str,
697
        allow_unbound: bool,
698
    ) -> Result<()> {
75167672 Andy McCall 699
        let theirs = remote.repo_id()?;
700
        if theirs == self.id()? {
701
            return Ok(());
702
        }
703
        if theirs == crate::protocol::UNBOUND_ID {
704
            if allow_unbound {
705
                return Ok(());
706
            }
05e71ed6 Andy McCall ↶ 707
            return fail(format!(
708
                "remote {remote_name:?} is empty and unbound: push to it first"
709
            ));
75167672 Andy McCall 710
        }
05e71ed6 Andy McCall ↶ 711
        fail(format!(
712
            "remote {remote_name:?} is a different repository (repository IDs differ)"
713
        ))
75167672 Andy McCall 714
    }
715
2b5ee371 Andy McCall ↶ 716
    fn remote_options(&self, name: &str) -> RemoteOptions {
717
        RemoteOptions {
75167672 Andy McCall 718
            repo_id: self.id().ok(),
719
            token: self
720
                .config
721
                .remote_tokens
722
                .get(name)
723
                .cloned()
724
                .or_else(|| std::env::var("OXY_TOKEN").ok()),
725
            ca: self
726
                .config
727
                .remote_cas
728
                .get(name)
729
                .map(PathBuf::from)
730
                .or_else(|| std::env::var("OXY_CA").ok().map(PathBuf::from)),
2b5ee371 Andy McCall ↶ 731
        }
98286406 Andy McCall ↶ 732
    }
733
d82e4409 Andy McCall ↶ 734
    /// Push seal attestations on their own (hub spec §7.4): a bundle with
735
    /// no changes and no reference updates, which the receiver stores.
736
    pub fn push_seals(&self, remote_name: &str, seals: &[ObjectRef]) -> Result<()> {
737
        let remote = self.open_named_remote(remote_name)?;
738
        self.check_remote_id(remote.as_ref(), remote_name, false)?;
739
        let mut bundle = Bundle::default();
740
        for r in seals {
741
            let attest = self.store.get_attest(r)?;
742
            let sigs = self.store.signatures(r)?;
4213c41d Andy McCall ↶ 743
            bundle
744
                .attests
745
                .push(crate::protocol::SignedAttest { attest, sigs });
d82e4409 Andy McCall ↶ 746
        }
747
        let results = remote.push(bundle, vec![], None, vec![])?;
748
        if let Some(r) = results.iter().find(|r| !r.ok) {
749
            return fail(format!("{}: {}", r.name, r.reason));
750
        }
751
        Ok(())
752
    }
753
754
    /// The remote whose URL starts with `base` (a hub), if configured.
755
    pub fn remote_for_url(&self, base: &str) -> Option<String> {
756
        let base = base.trim_end_matches('/');
4213c41d Andy McCall ↶ 757
        self.config
758
            .remotes
759
            .iter()
760
            .find(|(_, url)| url.trim_end_matches('/') == base)
761
            .map(|(name, _)| name.clone())
d82e4409 Andy McCall ↶ 762
    }
763
764
    /// Open a remote by name or, failing that, by URL with the default
765
    /// credentials (`OXY_TOKEN`, `OXY_CA`).
766
    pub fn open_remote_at(&self, url: &str) -> Result<(String, Box<dyn crate::protocol::Remote>)> {
767
        if let Some(name) = self.remote_for_url(url) {
768
            return Ok((name.clone(), self.open_named_remote(&name)?));
769
        }
770
        let options = RemoteOptions {
771
            token: std::env::var("OXY_TOKEN").ok(),
772
            ca: std::env::var("OXY_CA").ok().map(PathBuf::from),
773
            repo_id: self.id().ok(),
774
        };
775
        Ok((url.to_string(), open_remote(url, &options)?))
776
    }
777
778
    /// Credentials for a URL: those of the configured remote with that
779
    /// URL, else the environment's.
780
    pub fn options_for_url(&self, url: &str) -> RemoteOptions {
781
        match self.remote_for_url(url) {
782
            Some(name) => self.remote_options(&name),
783
            None => RemoteOptions {
784
                token: std::env::var("OXY_TOKEN").ok(),
785
                ca: std::env::var("OXY_CA").ok().map(PathBuf::from),
786
                repo_id: self.id().ok(),
787
            },
788
        }
789
    }
790
98286406 Andy McCall ↶ 791
    fn open_named_remote(&self, name: &str) -> Result<Box<dyn crate::protocol::Remote>> {
2b5ee371 Andy McCall ↶ 792
        open_remote(&self.remote_url(name)?, &self.remote_options(name))
98286406 Andy McCall ↶ 793
    }
794
795
    fn serial_path(&self, remote: &str, bookmark: &str) -> PathBuf {
75167672 Andy McCall 796
        self.oxy
797
            .join("remotes")
798
            .join(remote)
799
            .join(format!("{bookmark}.serial"))
98286406 Andy McCall ↶ 800
    }
801
802
    fn last_seen_serial(&self, remote: &str, bookmark: &str) -> Option<u64> {
75167672 Andy McCall 803
        fs::read_to_string(self.serial_path(remote, bookmark))
804
            .ok()
805
            .and_then(|s| s.trim().parse().ok())
98286406 Andy McCall ↶ 806
    }
807
808
    fn record_serial(&self, remote: &str, bookmark: &str, serial: u64) -> Result<()> {
809
        let p = self.serial_path(remote, bookmark);
810
        fs::create_dir_all(p.parent().unwrap())?;
811
        write_atomic(&p, serial.to_string().as_bytes())
812
    }
813
83e0677b Andy McCall ↶ 814
    fn policy_serial_path(&self, remote: &str) -> PathBuf {
815
        self.oxy.join("remotes").join(remote).join("policy.serial")
816
    }
817
818
    /// The highest policy serial `remote` has ever presented.
819
    fn remote_policy_serial(&self, remote: &str) -> Option<u64> {
75167672 Andy McCall 820
        fs::read_to_string(self.policy_serial_path(remote))
821
            .ok()
822
            .and_then(|s| s.trim().parse().ok())
83e0677b Andy McCall ↶ 823
    }
824
825
    fn record_policy_serial(&self, remote: &str, serial: u64) -> Result<()> {
826
        let p = self.policy_serial_path(remote);
827
        fs::create_dir_all(p.parent().unwrap())?;
828
        write_atomic(&p, serial.to_string().as_bytes())
829
    }
830
b3c5a59a Andy McCall ↶ 831
    fn lazy_dir(&self) -> PathBuf {
832
        self.oxy.join("lazy")
833
    }
834
835
    /// Remember that `blob` was withheld by `remote` (spec §13.5).
836
    fn register_lazy(&self, blob: &ObjectRef, remote: &str) -> Result<()> {
837
        fs::create_dir_all(self.lazy_dir())?;
838
        write_atomic(&self.lazy_dir().join(blob.to_hex()), remote.as_bytes())
839
    }
840
841
    /// Which remote promised an absent blob, if any.
842
    pub fn lazy_remote(&self, blob: &ObjectRef) -> Option<String> {
75167672 Andy McCall 843
        fs::read_to_string(self.lazy_dir().join(blob.to_hex()))
844
            .ok()
845
            .map(|s| s.trim().to_string())
b3c5a59a Andy McCall ↶ 846
    }
847
848
    /// Fetch an absent blob (and its chunks) from the remote that promised it.
849
    pub(crate) fn fetch_lazy_blob(&self, blob: &ObjectRef) -> Result<()> {
850
        let Some(remote_name) = self.lazy_remote(blob) else {
851
            return Err(RepoError::Store(StoreError::NotFound(*blob)));
852
        };
853
        let remote = self.open_named_remote(&remote_name)?;
854
        let objects = remote.objects(&[*blob])?;
855
        for (ty, body) in objects {
856
            if matches!(ty, ObjectType::Blob | ObjectType::Chunk) {
857
                self.store.put(ty, &body)?;
858
            }
859
        }
860
        if !self.store.has(blob) {
75167672 Andy McCall 861
            return fail(format!(
862
                "{remote_name} did not send blob {}",
863
                blob.short(12)
864
            ));
b3c5a59a Andy McCall ↶ 865
        }
866
        let _ = fs::remove_file(self.lazy_dir().join(blob.to_hex()));
867
        Ok(())
868
    }
869
870
    /// After importing from `remote`, record every blob the changes reference
871
    /// that the store lacks as lazily available from that remote.
872
    fn register_missing_blobs(&self, changes: &[ChangeHash], remote: &str) -> Result<usize> {
873
        let mut n = 0;
874
        for h in changes {
875
            let c = self.store.get_change(h)?;
876
            for e in &c.edits {
877
                if let Edit::Replace { old, new, .. } = e {
878
                    for b in [old, new].into_iter().flatten() {
879
                        if !self.store.has(b) {
880
                            self.register_lazy(b, remote)?;
881
                            n += 1;
882
                        }
883
                    }
884
                }
885
            }
886
        }
887
        Ok(n)
888
    }
889
98286406 Andy McCall ↶ 890
    /// Every head this repository knows: bookmarks, remote tracking, working copy.
d82e4409 Andy McCall ↶ 891
    pub fn known_heads(&self, view: &View) -> Result<Vec<ChangeHash>> {
98286406 Andy McCall ↶ 892
        let mut have: BTreeSet<ChangeHash> = BTreeSet::new();
75167672 Andy McCall 893
        for st in view
894
            .bookmarks
895
            .values()
896
            .chain(view.remotes.values().flat_map(|b| b.values()))
897
        {
98286406 Andy McCall ↶ 898
            if let Ok(s) = self.store.get_state(st) {
899
                have.extend(s.heads);
900
            }
901
        }
902
        have.extend(view.workspaces.values().copied());
903
        Ok(have.into_iter().collect())
904
    }
905
906
    /// Fetch every bookmark of a remote into `remotes/<name>/bookmarks/*` (spec §10.2).
83e0677b Andy McCall ↶ 907
    pub fn fetch(&self, remote_name: &str) -> Result<FetchReport> {
98286406 Andy McCall ↶ 908
        let remote = self.open_named_remote(remote_name)?;
75167672 Andy McCall 909
        self.check_remote_id(remote.as_ref(), remote_name, false)?;
83e0677b Andy McCall ↶ 910
        let resp = remote.refs()?;
911
        // Trust policy (spec §14.5). Adopt the remote's policy when it is newer
912
        // and signed by roots we trust, or on first contact. A remote that
913
        // once presented a policy may never present none or an older one:
914
        // that is a downgrade. A remote that never presented one is
915
        // unverified, and this fetch is checked only as far as possible.
916
        let local_policy = self.current_policy()?.map(|(_, sp)| sp.policy);
917
        let seen = self.remote_policy_serial(remote_name);
918
        let mut unverified = false;
919
        match &resp.policy {
920
            Some(sp) => {
921
                if seen.is_some_and(|s| sp.policy.serial < s) {
922
                    return fail(format!("{remote_name} presents policy serial {} but presented {} before: refusing the downgrade", sp.policy.serial, seen.unwrap()));
923
                }
924
                match &local_policy {
925
                    Some(l) if sp.policy.serial < l.serial => {}
926
                    Some(l) if sp.policy.serial == l.serial => {
927
                        if sp.policy.hash() != l.hash() {
75167672 Andy McCall 928
                            return fail(format!(
929
                                "{remote_name} presents a different policy with the same serial {}",
930
                                l.serial
931
                            ));
83e0677b Andy McCall ↶ 932
                        }
933
                    }
28fc23d7 Andy McCall ↶ 934
                    _ => {
75167672 Andy McCall 935
                        self.adopt_policy_chain(
936
                            &resp.history,
937
                            sp,
938
                            &format!("oxy fetch {remote_name}"),
939
                        )
940
                        .map_err(|e| RepoError::Msg(format!("policy from {remote_name}: {e}")))?;
83e0677b Andy McCall ↶ 941
                    }
942
                }
943
                self.record_policy_serial(remote_name, sp.policy.serial)?;
944
            }
945
            None => {
946
                if seen.is_some() {
947
                    return fail(format!("{remote_name} presented a policy before but has none now: refusing the downgrade"));
948
                }
949
                unverified = true;
950
            }
951
        }
75167672 Andy McCall 952
        let policy = if unverified {
953
            None
954
        } else {
955
            self.current_policy()?.map(|(_, sp)| sp.policy)
956
        };
98286406 Andy McCall ↶ 957
        let (op_ref, op) = self.head_op()?;
958
        let mut view = op.view.clone();
959
        let have = self.known_heads(&view)?;
960
        let mut report = Vec::new();
b3c5a59a Andy McCall ↶ 961
        let max_blob = self.config.max_blob;
83e0677b Andy McCall ↶ 962
        for r in resp.refs {
98286406 Andy McCall ↶ 963
            let needed = remote.negotiate(&have, &r.heads)?;
b3c5a59a Andy McCall ↶ 964
            let bundle = remote.fetch(&needed, max_blob)?;
83e0677b Andy McCall ↶ 965
            // Changes are not individually re-verified here: the bookmark's
966
            // attestation (checked below) binds its whole state, and the
967
            // receiver of the push verified the added changes at that time.
98286406 Andy McCall ↶ 968
            let imported = self.import_bundle(&bundle, false)?;
b3c5a59a Andy McCall ↶ 969
            if max_blob.is_some() {
970
                self.register_missing_blobs(&imported, remote_name)?;
971
            }
98286406 Andy McCall ↶ 972
            for h in &r.heads {
973
                if !self.store.has(h) {
974
                    return fail(format!("remote did not send head {}", h.short(12)));
975
                }
976
            }
977
            let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
978
            if state != r.state {
75167672 Andy McCall 979
                return fail(format!(
980
                    "remote state {} does not match its heads",
981
                    r.state.short(12)
982
                ));
98286406 Andy McCall ↶ 983
            }
83e0677b Andy McCall ↶ 984
            if let Some(p) = &policy {
985
                let Some(sa) = &r.attestation else {
75167672 Andy McCall 986
                    return fail(format!(
987
                        "{remote_name}/{} has no attestation, which its policy requires",
988
                        r.name
989
                    ));
83e0677b Andy McCall ↶ 990
                };
75167672 Andy McCall 991
                let signer = sa
992
                    .sigs
993
                    .first()
994
                    .map(|s| s.key)
995
                    .ok_or_else(|| RepoError::Msg("attestation carries no signature".into()))?;
83e0677b Andy McCall ↶ 996
                if !p.authorizes(&signer, &r.name) {
75167672 Andy McCall 997
                    return fail(format!(
998
                        "{remote_name}/{}: attested by {}, which the policy does not authorize",
999
                        r.name,
1000
                        signer.short(12)
1001
                    ));
83e0677b Andy McCall ↶ 1002
                }
1003
            }
98286406 Andy McCall ↶ 1004
            if let Some(sa) = &r.attestation {
75167672 Andy McCall 1005
                let AttestBody::Ref {
1006
                    old_state,
1007
                    new_state,
1008
                    ..
1009
                } = &sa.attest.body
1010
                else {
98286406 Andy McCall ↶ 1011
                    return fail("remote sent a non-ref attestation for a bookmark");
1012
                };
1013
                let (old_state, new_state) = (*old_state, *new_state);
1014
                let serial = self
75167672 Andy McCall 1015
                    .check_ref_attest(
1016
                        sa,
1017
                        &r.name,
1018
                        &old_state,
1019
                        &new_state,
1020
                        self.last_seen_serial(remote_name, &r.name),
1021
                        true,
1022
                    )
1023
                    .map_err(|e| {
1024
                        RepoError::Msg(format!("attestation for {}/{}: {e}", remote_name, r.name))
1025
                    })?;
98286406 Andy McCall ↶ 1026
                if new_state != r.state {
75167672 Andy McCall 1027
                    return fail(format!(
1028
                        "attestation for {}/{} does not cover its current state",
1029
                        remote_name, r.name
1030
                    ));
98286406 Andy McCall ↶ 1031
                }
1032
                let ar = self.store.put_attest(&sa.attest)?;
1033
                for sig in &sa.sigs {
1034
                    self.store.put_signature(&ar, sig)?;
1035
                }
1036
                self.record_serial(remote_name, &r.name, serial)?;
1037
            }
75167672 Andy McCall 1038
            view.remotes
1039
                .entry(remote_name.to_string())
1040
                .or_default()
1041
                .insert(r.name.clone(), state);
98286406 Andy McCall ↶ 1042
            report.push((r.name, r.heads, imported.len()));
1043
        }
132b884b Andy McCall ↶ 1044
        let mut new_tags = 0;
1045
        for t in &resp.tags {
1046
            if view.tags.contains_key(&t.name) {
1047
                continue;
1048
            }
1049
            // The tag's changes must be here: fetch them if the bookmarks did not bring them.
75167672 Andy McCall 1050
            let missing: Vec<ChangeHash> = t
1051
                .heads
1052
                .iter()
1053
                .filter(|h| !self.store.has(h))
1054
                .copied()
1055
                .collect();
132b884b Andy McCall ↶ 1056
            if !missing.is_empty() {
1057
                let needed = remote.negotiate(&self.known_heads(&view)?, &missing)?;
1058
                let bundle = remote.fetch(&needed, max_blob)?;
1059
                self.import_bundle(&bundle, false)?;
1060
            }
1061
            self.store.put_state(&StateObject::new(t.heads.clone()))?;
75167672 Andy McCall 1062
            self.adopt_tag(&mut view, t, policy.as_ref())
1063
                .map_err(|e| RepoError::Msg(format!("{remote_name} {}: {e}", t.name)))?;
132b884b Andy McCall ↶ 1064
            new_tags += 1;
1065
        }
98286406 Andy McCall ↶ 1066
        if view != op.view {
1067
            self.record_op(&[op_ref], &format!("oxy fetch {remote_name}"), view)?;
1068
        }
6d3eefbc Andy McCall ↶ 1069
        let _ = self.run_hook(
1070
            "post-fetch",
1071
            &format!(
1072
                "{{\"event\":\"post-fetch\",\"remote\":{},\"bookmarks\":{}}}",
1073
                json_str(remote_name),
75167672 Andy McCall 1074
                json_array(report.iter().map(|(n, heads, _)| format!(
1075
                    "{{\"name\":{},\"heads\":{}}}",
1076
                    json_str(n),
1077
                    json_array(heads.iter().map(|h| json_str(&h.to_hex())))
1078
                )))
6d3eefbc Andy McCall ↶ 1079
            ),
1080
        );
75167672 Andy McCall 1081
        Ok(FetchReport {
1082
            refs: report,
1083
            unverified,
1084
            new_tags,
1085
        })
98286406 Andy McCall ↶ 1086
    }
1087
2863456a Andy McCall ↶ 1088
    /// Shallow fetch of the remote's bookmark: its checkpoint as our base,
1089
    /// the changes within `depth` steps of its heads, and the boundary below.
1090
    pub fn fetch_shallow(&self, remote_name: &str, depth: u32) -> Result<usize> {
1091
        let remote = self.open_named_remote(remote_name)?;
75167672 Andy McCall 1092
        self.check_remote_id(remote.as_ref(), remote_name, false)?;
2863456a Andy McCall ↶ 1093
        let resp = remote.refs()?;
1094
        let bookmark = self.config.bookmark.clone();
1095
        if let Some(sp) = &resp.policy {
75167672 Andy McCall 1096
            self.adopt_policy_chain(&resp.history, sp, "oxy clone --depth")
1097
                .map_err(|e| RepoError::Msg(format!("policy from {remote_name}: {e}")))?;
2863456a Andy McCall ↶ 1098
            self.record_policy_serial(remote_name, sp.policy.serial)?;
1099
        }
1100
        let Some(r) = resp.refs.into_iter().find(|r| r.name == bookmark) else {
1101
            return fail(format!("{remote_name} has no bookmark {bookmark:?}"));
1102
        };
1103
        if r.heads.is_empty() {
1104
            return Ok(0);
1105
        }
1106
        // Base: the remote's checkpoint of exactly these heads.
1107
        let cp_bundle = remote.checkpoint(&r.heads)?;
1108
        let mut cp_ref = None;
1109
        for (ty, body) in &cp_bundle.objects {
1110
            let put = self.store.put(*ty, body)?;
1111
            if *ty == ObjectType::Checkpoint {
1112
                cp_ref = Some(put);
1113
            }
1114
        }
75167672 Andy McCall 1115
        let Some(cp_ref) = cp_ref else {
1116
            return fail(format!("{remote_name} sent no checkpoint"));
1117
        };
2863456a Andy McCall ↶ 1118
        self.install_checkpoint(&bookmark, &cp_ref)?;
1119
        // History window.
1120
        let wanted = remote.shallow(&r.heads, depth)?;
1121
        let bundle = remote.fetch(&wanted, self.config.max_blob)?;
1122
        let imported = self.import_bundle_shallow(&bundle, false)?;
1123
        if self.config.max_blob.is_some() {
1124
            self.register_missing_blobs(&imported, remote_name)?;
1125
        }
1126
        self.refresh_boundary(&imported)?;
1127
        if let Some(sa) = &r.attestation {
1128
            let ar = self.store.put_attest(&sa.attest)?;
1129
            for sig in &sa.sigs {
1130
                self.store.put_signature(&ar, sig)?;
1131
            }
1132
            if let AttestBody::Ref { serial, .. } = sa.attest.body {
1133
                self.record_serial(remote_name, &r.name, serial)?;
1134
            }
1135
        }
1136
        let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
1137
        let (op_ref, op) = self.head_op()?;
1138
        let mut view = op.view.clone();
75167672 Andy McCall 1139
        view.remotes
1140
            .entry(remote_name.to_string())
1141
            .or_default()
1142
            .insert(r.name.clone(), state);
1143
        self.record_op(
1144
            &[op_ref],
1145
            &format!("oxy fetch {remote_name} --depth {depth}"),
1146
            view,
1147
        )?;
2863456a Andy McCall ↶ 1148
        Ok(imported.len())
1149
    }
1150
98286406 Andy McCall ↶ 1151
    /// Fetch, then take the union of the local bookmark's state with the
1152
    /// remote's: no merge change, conflicts become reified (spec §9.4).
1153
    pub fn pull(&self, remote_name: &str) -> Result<PullReport> {
1154
        self.fetch(remote_name)?;
1155
        let (op_ref, op) = self.head_op()?;
1156
        let bookmark = self.config.bookmark.clone();
1157
        let local = self.heads_of(&op.view, &bookmark)?;
75167672 Andy McCall 1158
        let Some(remote_state) = op
1159
            .view
1160
            .remotes
1161
            .get(remote_name)
1162
            .and_then(|b| b.get(&bookmark))
1163
            .copied()
1164
        else {
1165
            return Ok(PullReport {
1166
                heads: local,
1167
                changed: false,
1168
                checked_out: false,
1169
            });
98286406 Andy McCall ↶ 1170
        };
1171
        let remote_heads = self.store.get_state(&remote_state)?.heads;
1172
        let mut all = local.clone();
1173
        all.extend(remote_heads);
1174
        let st = self.load_state(&all)?;
1175
        let mut heads = st.maximal(&all);
1176
        heads.sort();
1177
        let mut sorted_local = local.clone();
1178
        sorted_local.sort();
1179
        if heads == sorted_local {
75167672 Andy McCall 1180
            return Ok(PullReport {
1181
                heads,
1182
                changed: false,
1183
                checked_out: false,
1184
            });
98286406 Andy McCall ↶ 1185
        }
1186
        let mut view = op.view.clone();
75167672 Andy McCall 1187
        view.bookmarks.insert(
1188
            bookmark.clone(),
1189
            self.store.put_state(&StateObject::new(heads.clone()))?,
1190
        );
98286406 Andy McCall ↶ 1191
        self.record_op(&[op_ref], &format!("oxy pull {remote_name}"), view)?;
1192
        // Follow along if the working copy was clean and on the old heads.
1193
        let snap = self.snapshot("oxy pull")?;
1194
        let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
1195
        let mut base = snap.base.maximal(&snap.wc.deps);
1196
        base.sort();
1197
        let mut checked_out = false;
1198
        if clean && base == sorted_local {
1199
            self.checkout(&heads, false, "oxy pull (checkout)")?;
1200
            checked_out = true;
1201
        }
75167672 Andy McCall 1202
        Ok(PullReport {
1203
            heads,
1204
            changed: true,
1205
            checked_out,
1206
        })
98286406 Andy McCall ↶ 1207
    }
1208
1209
    /// Push the current bookmark (spec §16.3), sealing this author's unsigned
1210
    /// changes and attesting the reference when a key is available.
1211
    pub fn push(&self, remote_name: &str, force: bool) -> Result<Vec<RefResult>> {
6d3eefbc Andy McCall ↶ 1212
        let bookmark = self.config.bookmark.clone();
7b9809e6 Andy McCall ↶ 1213
        self.push_bookmark(remote_name, &bookmark, force)
1214
    }
1215
1216
    /// `push` for a named bookmark (`oxy push -b NAME`).
75167672 Andy McCall 1217
    pub fn push_bookmark(
1218
        &self,
1219
        remote_name: &str,
1220
        bookmark: &str,
1221
        force: bool,
1222
    ) -> Result<Vec<RefResult>> {
7b9809e6 Andy McCall ↶ 1223
        let (op_ref, op) = self.head_op()?;
1224
        if !op.view.bookmarks.contains_key(bookmark) {
1225
            return fail(format!("no bookmark {bookmark}"));
1226
        }
1227
        let bookmark = bookmark.to_string();
6d3eefbc Andy McCall ↶ 1228
        let local_heads = self.heads_of(&op.view, &bookmark)?;
1229
        // pre-push hook: may veto before the remote is contacted.
1230
        self.run_hook(
1231
            "pre-push",
75167672 Andy McCall 1232
            &format!(
1233
                "{{\"event\":\"pre-push\",\"remote\":{},\"bookmark\":{},\"heads\":{}}}",
1234
                json_str(remote_name),
1235
                json_str(&bookmark),
1236
                json_array(local_heads.iter().map(|h| json_str(&h.to_hex())))
1237
            ),
6d3eefbc Andy McCall ↶ 1238
        )?;
98286406 Andy McCall ↶ 1239
        let remote = self.open_named_remote(remote_name)?;
75167672 Andy McCall 1240
        self.check_remote_id(remote.as_ref(), remote_name, true)?;
83e0677b Andy McCall ↶ 1241
        let resp = remote.refs()?;
1242
        let local_policy = self.current_policy()?.map(|(_, sp)| sp);
28fc23d7 Andy McCall ↶ 1243
        let resp_policy_serial = resp.policy.as_ref().map(|p| p.policy.serial);
83e0677b Andy McCall ↶ 1244
        let policy_to_send = match (&local_policy, &resp.policy) {
1245
            (Some(l), Some(r)) if l.policy.serial > r.policy.serial => Some(l.clone()),
1246
            (Some(l), None) => Some(l.clone()),
1247
            (Some(l), Some(r)) if l.policy.serial < r.policy.serial => {
75167672 Andy McCall 1248
                return fail(format!(
1249
                    "{remote_name} has a newer policy (serial {}); fetch first",
1250
                    r.policy.serial
1251
                ));
83e0677b Andy McCall ↶ 1252
            }
1253
            _ => None,
1254
        };
1255
        let remote_has_policy = resp.policy.is_some() || policy_to_send.is_some();
1256
        if remote_has_policy && self.secret_key()?.is_none() {
1257
            return fail(format!("{remote_name} enforces a trust policy; pushing needs an identity (`oxy identity create`)"));
1258
        }
132b884b Andy McCall ↶ 1259
        let remote_tags: BTreeSet<String> = resp.tags.iter().map(|t| t.name.clone()).collect();
75167672 Andy McCall 1260
        let tags_to_send: Vec<crate::protocol::TagRef> = self
1261
            .tag_refs()?
1262
            .into_iter()
1263
            .filter(|t| !remote_tags.contains(&t.name))
1264
            .collect();
83e0677b Andy McCall ↶ 1265
        let refs = resp.refs;
98286406 Andy McCall ↶ 1266
        let empty = self.store.put_state(&StateObject::new(vec![]))?;
75167672 Andy McCall 1267
        let (remote_state, remote_heads, last_serial) =
1268
            match refs.iter().find(|r| r.name == bookmark) {
1269
                Some(r) => (r.state, r.heads.clone(), r.last_serial),
1270
                None => (empty, vec![], None),
1271
            };
1272
        let new_state = self
1273
            .store
1274
            .put_state(&StateObject::new(local_heads.clone()))?;
83e0677b Andy McCall ↶ 1275
        // A push that carries a policy re-attests the bookmark even when its
1276
        // state is unchanged, so the current attestation is always by a key
1277
        // the current policy authorizes (a revoked key's attestation would
1278
        // otherwise be refused by every later fetch).
132b884b Andy McCall ↶ 1279
        if new_state == remote_state && policy_to_send.is_none() && tags_to_send.is_empty() {
75167672 Andy McCall 1280
            return Ok(vec![RefResult {
1281
                name: bookmark,
1282
                ok: true,
1283
                reason: "already up to date".into(),
1284
            }]);
98286406 Andy McCall ↶ 1285
        }
1286
        for h in &remote_heads {
1287
            if !self.store.has(h) {
75167672 Andy McCall 1288
                return fail(format!(
1289
                    "the remote has change {} which you do not; pull first",
1290
                    h.short(12)
1291
                ));
98286406 Andy McCall ↶ 1292
            }
1293
        }
1294
        if !force {
1295
            let closure = self.closure_in_store(&local_heads)?;
1296
            if let Some(h) = remote_heads.iter().find(|h| !closure.contains(h)) {
1297
                return fail(format!("not a fast-forward: remote head {} is not in your state; pull first or use --force", h.short(12)));
1298
            }
1299
        }
1300
        let to_send = self.negotiate(&remote_heads, &local_heads)?;
1301
        let mut bundle = self.bundle_for(&to_send)?;
1302
        let key = self.secret_key()?.map(|(k, _)| k);
1303
        if let Some(key) = key {
05e71ed6 Andy McCall ↶ 1304
            // Seal this author's changes that carry no valid authenticator,
1305
            // and adopt anonymous ones (zero author, spec §14.4).
98286406 Andy McCall ↶ 1306
            let mut unsigned = Vec::new();
1307
            for h in &to_send {
1308
                let c = self.store.get_change(h)?;
05e71ed6 Andy McCall ↶ 1309
                if (c.author == key || c.author.is_zero())
75167672 Andy McCall 1310
                    && !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. })
1311
                {
98286406 Andy McCall ↶ 1312
                    unsigned.push(*h);
1313
                }
1314
            }
1315
            for chunk in unsigned.chunks(crate::attest::MAX_SEAL_HASHES) {
1316
                let sa = self.sign_attest(&Attest::seal(self.id()?, chunk.to_vec()))?;
1317
                bundle.attests.push(sa);
1318
            }
1319
            bundle = self.bundle_for(&to_send)?;
1320
        }
75167672 Andy McCall 1321
        bundle.objects.push((
1322
            ObjectType::State,
1323
            StateObject::new(local_heads.clone()).encode(),
1324
        ));
98286406 Andy McCall ↶ 1325
        let attestation = match key {
1326
            Some(_) => Some(self.sign_attest(&Attest {
1327
                repo: self.id()?,
1328
                hlc: Hlc::now(),
75167672 Andy McCall 1329
                body: AttestBody::Ref {
1330
                    name: bookmark.clone(),
1331
                    old_state: remote_state,
1332
                    new_state,
1333
                    serial: last_serial.unwrap_or(0) + 1,
1334
                },
98286406 Andy McCall ↶ 1335
            })?),
1336
            None => None,
1337
        };
1338
        let serial = attestation.as_ref().and_then(|a| match a.attest.body {
1339
            AttestBody::Ref { serial, .. } => Some(serial),
1340
            _ => None,
1341
        });
83e0677b Andy McCall ↶ 1342
        let sent_serial = policy_to_send.as_ref().map(|p| p.policy.serial);
132b884b Andy McCall ↶ 1343
        // Tag states must be present on the remote: bundle their changes too.
1344
        for t in &tags_to_send {
1345
            let extra = self.negotiate(&remote_heads, &t.heads)?;
75167672 Andy McCall 1346
            let extra: Vec<ChangeHash> =
1347
                extra.into_iter().filter(|h| !to_send.contains(h)).collect();
132b884b Andy McCall ↶ 1348
            if !extra.is_empty() {
1349
                let more = self.bundle_for(&extra)?;
1350
                bundle.changes.extend(more.changes);
1351
                bundle.attests.extend(more.attests);
1352
                bundle.objects.extend(more.objects);
1353
            }
75167672 Andy McCall 1354
            bundle.objects.push((
1355
                ObjectType::State,
1356
                StateObject::new(t.heads.clone()).encode(),
1357
            ));
132b884b Andy McCall ↶ 1358
        }
28fc23d7 Andy McCall ↶ 1359
        if let Some(l) = &policy_to_send {
1360
            let remote_serial = resp_policy_serial;
75167672 Andy McCall 1361
            bundle.policies = self
1362
                .policy_history()?
1363
                .into_iter()
1364
                .filter(|sp| {
1365
                    remote_serial.is_none_or(|r| sp.policy.serial > r)
1366
                        && sp.policy.serial < l.policy.serial
1367
                })
1368
                .collect();
28fc23d7 Andy McCall ↶ 1369
        }
75167672 Andy McCall 1370
        let results = remote.push(
1371
            bundle,
1372
            vec![RefUpdate {
1373
                name: bookmark.clone(),
1374
                old_state: remote_state,
1375
                new_state,
1376
                force,
1377
                attestation,
1378
            }],
1379
            policy_to_send,
1380
            tags_to_send,
1381
        )?;
1382
        if let (Some(serial), true) = (
1383
            sent_serial,
1384
            results.iter().any(|r| r.ok && r.name == "policy"),
1385
        ) {
83e0677b Andy McCall ↶ 1386
            self.record_policy_serial(remote_name, serial)?;
1387
        }
1388
        if results.iter().any(|r| r.ok && r.name == bookmark) {
98286406 Andy McCall ↶ 1389
            let mut view = op.view.clone();
75167672 Andy McCall 1390
            view.remotes
1391
                .entry(remote_name.to_string())
1392
                .or_default()
1393
                .insert(bookmark.clone(), new_state);
98286406 Andy McCall ↶ 1394
            self.record_op(&[op_ref], &format!("oxy push {remote_name}"), view)?;
1395
            if let Some(s) = serial {
1396
                self.record_serial(remote_name, &bookmark, s)?;
1397
            }
1398
        }
1399
        Ok(results)
1400
    }
1401
1402
    /// Clone a remote into `dest`: same repository ID, `origin` remote,
1403
    /// bookmark set to the remote's, working copy checked out.
1404
    pub fn clone(url: &str, dest: &Path) -> Result<Repo> {
7421309f Andy McCall ↶ 1405
        Self::clone_with(url, dest, None, &[])
b3c5a59a Andy McCall ↶ 1406
    }
1407
7421309f Andy McCall ↶ 1408
    /// Clone, leaving blobs above `max_blob` bytes to be fetched on first
1409
    /// read, and materializing only `sparse` paths (which implies lazy blobs
1410
    /// unless a size is given, so out-of-scope binaries are never fetched).
75167672 Andy McCall 1411
    pub fn clone_with(
1412
        url: &str,
1413
        dest: &Path,
1414
        max_blob: Option<u64>,
1415
        sparse: &[String],
1416
    ) -> Result<Repo> {
2863456a Andy McCall ↶ 1417
        Self::clone_full(url, dest, max_blob, sparse, None)
1418
    }
1419
1420
    /// Shallow clone (spec §17): a checkpoint of the bookmark's heads plus the
1421
    /// changes within `depth` steps of them; the rest is the boundary.
75167672 Andy McCall 1422
    pub fn clone_shallow(
1423
        url: &str,
1424
        dest: &Path,
1425
        depth: u32,
1426
        max_blob: Option<u64>,
1427
        sparse: &[String],
1428
    ) -> Result<Repo> {
2863456a Andy McCall ↶ 1429
        Self::clone_full(url, dest, max_blob, sparse, Some(depth))
1430
    }
1431
75167672 Andy McCall 1432
    fn clone_full(
1433
        url: &str,
1434
        dest: &Path,
1435
        max_blob: Option<u64>,
1436
        sparse: &[String],
1437
        depth: Option<u32>,
1438
    ) -> Result<Repo> {
2b5ee371 Andy McCall ↶ 1439
        let options = RemoteOptions {
1440
            token: std::env::var("OXY_TOKEN").ok(),
1441
            ca: std::env::var("OXY_CA").ok().map(PathBuf::from),
75167672 Andy McCall 1442
            repo_id: None,
2b5ee371 Andy McCall ↶ 1443
        };
1444
        let remote = open_remote(url, &options)?;
98286406 Andy McCall ↶ 1445
        let id = remote.repo_id()?;
75167672 Andy McCall 1446
        if id == crate::protocol::UNBOUND_ID {
05e71ed6 Andy McCall ↶ 1447
            return fail(format!(
1448
                "{url} is empty and unbound: push a repository to it first"
1449
            ));
75167672 Andy McCall 1450
        }
98286406 Andy McCall ↶ 1451
        let mut repo = Repo::init_with_id(dest, id)?;
1452
        repo.config.set("remote.origin.url", url)?;
b3c5a59a Andy McCall ↶ 1453
        if let Some(m) = max_blob {
1454
            repo.config.set("filter.max-blob", &m.to_string())?;
7421309f Andy McCall ↶ 1455
        } else if !sparse.is_empty() {
1456
            repo.config.set("filter.max-blob", "0")?;
1457
        }
1458
        if !sparse.is_empty() {
1459
            repo.config.set("filter.paths", &sparse.join(","))?;
b3c5a59a Andy McCall ↶ 1460
        }
98286406 Andy McCall ↶ 1461
        repo.save_config()?;
2863456a Andy McCall ↶ 1462
        if let Some(depth) = depth {
1463
            repo.fetch_shallow("origin", depth)?;
1464
        } else {
1465
            repo.fetch("origin")?;
1466
        }
98286406 Andy McCall ↶ 1467
        let (op_ref, op) = repo.head_op()?;
1468
        let bookmark = repo.config.bookmark.clone();
75167672 Andy McCall 1469
        if let Some(state) = op
1470
            .view
1471
            .remotes
1472
            .get("origin")
1473
            .and_then(|b| b.get(&bookmark))
1474
            .copied()
1475
        {
98286406 Andy McCall ↶ 1476
            let heads = repo.store.get_state(&state)?.heads;
1477
            let mut view = op.view.clone();
1478
            view.bookmarks.insert(bookmark, state);
1479
            repo.record_op(&[op_ref], &format!("oxy clone {url}"), view)?;
1480
            repo.checkout(&heads, false, "oxy clone (checkout)")?;
1481
        }
1482
        Ok(repo)
1483
    }
1484
}