OXYGENHUB
# cea2e2f860c5 oxygen / crates / oxy-core / src / repo / sync.rs
114.5 KiB · Rust · last changed Fall back from a checkout's own token only when the hub does not know it (401), never when it refuses it for permission (refs #155) 7 days ago source blame history raw
1
//! Serving and talking to remotes (spec §10.2, §16).
2
3
use super::tags::TagCheck;
4
use super::*;
5
6
/// What a push did, for hosts (hub spec §6.12).
7
#[derive(Clone, Debug, Default)]
8
pub struct PushReport {
9
    pub results: Vec<RefResult>,
10
    /// Accepted reference updates, in request order.
11
    pub updated: Vec<RefUpdated>,
12
    /// Changes this push stored that were not already present.
13
    pub imported: Vec<ChangeHash>,
14
}
15
16
#[derive(Clone, Debug)]
17
pub struct RefUpdated {
18
    pub name: String,
19
    pub old_state: ObjectRef,
20
    pub new_state: ObjectRef,
21
    pub heads: Vec<ChangeHash>,
22
}
23
24
/// What a fetch brought in.
25
#[derive(Debug, Clone, PartialEq, Eq, Default)]
26
pub struct FetchReport {
27
    /// Bookmark name, its heads, and how many changes were new.
28
    pub refs: Vec<(String, Vec<ChangeHash>, usize)>,
29
    /// Tags adopted from the remote.
30
    pub new_tags: usize,
31
    /// The remote has never presented a policy, so nothing was verified
32
    /// against one (spec §14.5: warn loudly).
33
    pub unverified: bool,
34
    /// What checking the host's transparency log found short of a fork
35
    /// (spec §14.8): attestations not logged yet, a log that could not be
36
    /// read. A fork fails the fetch instead.
37
    pub log_notes: Vec<String>,
38
    /// Bookmarks left as they were because their attestation could not be
39
    /// verified (its signer unknown here, or its signature bad), each with
40
    /// why (oxygenhub#266). The current bookmark is never skipped: its
41
    /// fetch fails instead.
42
    pub skipped: Vec<(String, String)>,
43
}
44
45
#[derive(Debug, Clone, PartialEq, Eq)]
46
pub struct PullReport {
47
    /// The current bookmark's heads after the pull.
48
    pub heads: Vec<ChangeHash>,
49
    /// Whether the current bookmark changed.
50
    pub changed: bool,
51
    pub checked_out: bool,
52
    /// Other bookmarks the pull moved: each local bookmark tracks the
53
    /// remote's of the same name.
54
    pub others: Vec<String>,
55
    /// What the fetch left as it was ([`FetchReport::skipped`]).
56
    pub skipped: Vec<(String, String)>,
57
}
58
59
/// The result a push of objects alone reports (a batch of a long history):
60
/// not a reference.
61
pub const OBJECTS_RESULT: &str = "objects";
62
63
/// Graph blobs asked for in one `objects` request when a shallow clone
64
/// fetches its checkpoint in parts (#137): their chunk lists and chunks,
65
/// a few MiB a request rather than one answer of gigabytes.
66
const CHECKPOINT_PART: usize = 2048;
67
68
impl Repo {
69
    // ------------------------------------------------------------ serving (spec §16)
70
71
    fn ref_attest_path(&self, bookmark: &str) -> PathBuf {
72
        self.oxy.join("attest").join(name_file(bookmark))
73
    }
74
75
    /// The latest reference attestation stored for a bookmark.
76
    pub fn latest_ref_attest(&self, bookmark: &str) -> Result<Option<(ObjectRef, SignedAttest)>> {
77
        match fs::read_to_string(self.ref_attest_path(bookmark)) {
78
            Ok(h) => {
79
                let r = ObjectRef::from_hex(h.trim())
80
                    .ok_or_else(|| RepoError::Msg("corrupt attestation pointer".into()))?;
81
                Ok(Some((
82
                    r,
83
                    SignedAttest {
84
                        attest: self.store.get_attest(&r)?,
85
                        sigs: self.store.signatures(&r)?,
86
                    },
87
                )))
88
            }
89
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
90
            Err(e) => Err(e.into()),
91
        }
92
    }
93
94
    pub(crate) fn set_ref_attest(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
95
        fs::create_dir_all(self.oxy.join("attest"))?;
96
        write_atomic(&self.ref_attest_path(bookmark), r.to_hex().as_bytes())
97
    }
98
99
    pub fn remote_refs(&self) -> Result<RefsResponse> {
100
        let view = self.view()?;
101
        let mut out = Vec::new();
102
        for (name, state) in &view.bookmarks {
103
            let latest = self.latest_ref_attest(name)?.map(|(_, a)| a);
104
            let last_serial = latest.as_ref().and_then(|a| match a.attest.body {
105
                AttestBody::Ref { serial, .. } => Some(serial),
106
                _ => None,
107
            });
108
            let attestation = latest.filter(|a| matches!(&a.attest.body, AttestBody::Ref { new_state, .. } if new_state == state));
109
            let accepted = match &attestation {
110
                Some(a) => self.acceptance_of(&a.attest.hash())?,
111
                None => None,
112
            };
113
            out.push(RemoteRef {
114
                name: name.clone(),
115
                state: *state,
116
                heads: self.store.get_state(state)?.heads,
117
                attestation,
118
                last_serial,
119
                accepted,
120
                logged: None,
121
                integration: vec![],
122
            });
123
        }
124
        let policy = self.current_policy()?.map(|(_, sp)| sp);
125
        let history = self
126
            .policy_history()?
127
            .into_iter()
128
            .filter(|sp| {
129
                policy
130
                    .as_ref()
131
                    .is_some_and(|p| sp.policy.serial < p.policy.serial)
132
            })
133
            .collect();
134
        Ok(RefsResponse {
135
            refs: out,
136
            policy,
137
            tags: self.tag_refs()?,
138
            history,
139
            default_bookmark: Some(self.config.bookmark.clone()),
140
            log: None,
141
            witnesses: vec![],
142
        })
143
    }
144
145
    /// Every change reachable from `heads` that is in the store.
146
    fn closure_in_store(&self, heads: &[ChangeHash]) -> Result<BTreeSet<ChangeHash>> {
147
        let mut seen = BTreeSet::new();
148
        let mut stack: Vec<ChangeHash> = heads.to_vec();
149
        while let Some(h) = stack.pop() {
150
            if seen.contains(&h) || !self.store.has(&h) {
151
                continue;
152
            }
153
            seen.insert(h);
154
            stack.extend(self.store.get_change_meta(&h)?.deps);
155
        }
156
        Ok(seen)
157
    }
158
159
    /// The changes within `depth` dependency steps of `heads` (spec §17),
160
    /// nearest first; depth 1 is the heads themselves.
161
    pub fn shallow_changes(&self, heads: &[ChangeHash], depth: u32) -> Result<Vec<ChangeHash>> {
162
        let mut out = Vec::new();
163
        let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
164
        let mut layer: Vec<ChangeHash> = heads.to_vec();
165
        for _ in 0..depth.max(1) {
166
            let mut next = Vec::new();
167
            for h in layer {
168
                if !seen.insert(h) || !self.store.has(&h) {
169
                    continue;
170
                }
171
                let c = self.store.get_change(&h)?;
172
                next.extend(c.deps.iter().copied());
173
                out.push(h);
174
            }
175
            if next.is_empty() {
176
                break;
177
            }
178
            layer = next;
179
        }
180
        out.reverse();
181
        Ok(out)
182
    }
183
184
    fn shallow_path(&self) -> PathBuf {
185
        self.oxy.join("shallow")
186
    }
187
188
    /// The shallow boundary: dependencies this repository does not hold (spec §17).
189
    pub fn boundary(&self) -> BTreeSet<ChangeHash> {
190
        fs::read_to_string(self.shallow_path())
191
            .map(|t| t.lines().filter_map(ObjectRef::from_hex).collect())
192
            .unwrap_or_default()
193
    }
194
195
    pub fn is_shallow(&self) -> bool {
196
        !self.boundary().is_empty()
197
    }
198
199
    fn write_boundary(&self, boundary: &BTreeSet<ChangeHash>) -> Result<()> {
200
        if boundary.is_empty() {
201
            let _ = fs::remove_file(self.shallow_path());
202
            return Ok(());
203
        }
204
        let text: String = boundary
205
            .iter()
206
            .map(|h| format!("{}\n", h.to_hex()))
207
            .collect();
208
        write_atomic(&self.shallow_path(), text.as_bytes())
209
    }
210
211
    /// Recompute the boundary from what is stored: every dependency of a
212
    /// stored change that is not itself stored.
213
    fn refresh_boundary(&self, imported: &[ChangeHash]) -> Result<()> {
214
        let mut boundary = self.boundary();
215
        boundary.retain(|h| !self.store.has(h));
216
        for h in imported {
217
            for d in self.store.get_change_meta(h)?.deps {
218
                if !self.store.has(&d) {
219
                    boundary.insert(d);
220
                }
221
            }
222
        }
223
        self.write_boundary(&boundary)
224
    }
225
226
    /// Fetch `depth` more levels of history below the shallow boundary.
227
    pub fn deepen(&self, remote_name: &str, depth: u32) -> Result<usize> {
228
        let boundary = self.boundary();
229
        if boundary.is_empty() {
230
            return Ok(0);
231
        }
232
        let remote = self.open_named_remote(remote_name)?;
233
        let wanted = remote.shallow(&boundary.iter().copied().collect::<Vec<_>>(), depth)?;
234
        let bundle = remote.fetch(&wanted, self.config.max_blob)?;
235
        let imported = self.import_bundle_shallow(&bundle, false)?;
236
        if self.config.max_blob.is_some() {
237
            self.register_missing_blobs(&imported, remote_name)?;
238
        }
239
        self.refresh_boundary(&imported)?;
240
        Ok(imported.len())
241
    }
242
243
    /// `closure(want) \ closure(have)`, dependencies first (spec §16.2).
244
    pub fn negotiate(&self, have: &[ChangeHash], want: &[ChangeHash]) -> Result<Vec<ChangeHash>> {
245
        let have = self.closure_in_store(have)?;
246
        // In a shallow repository the walk stops at the boundary, whose
247
        // changes the remote is assumed to hold.
248
        let boundary = self.boundary();
249
        let mut out = Vec::new();
250
        let mut done: BTreeSet<ChangeHash> = have.clone();
251
        done.extend(boundary.iter().copied());
252
        // Iterative post-order DFS.
253
        let shallow = self.is_shallow();
254
        let mut stack: Vec<(ChangeHash, bool)> = want.iter().rev().map(|h| (*h, false)).collect();
255
        while let Some((h, expanded)) = stack.pop() {
256
            if done.contains(&h) {
257
                continue;
258
            }
259
            if !self.store.has(&h) {
260
                // A shallow repository holds only part of its history; a
261
                // change it lacks is below what it cloned (in a depth-0
262
                // clone, any change of the checkpoint), which the remote
263
                // is assumed to hold, as it is for the boundary (#137).
264
                if shallow {
265
                    done.insert(h);
266
                    continue;
267
                }
268
                return fail(format!("change {} is not in this repository", h.short(12)));
269
            }
270
            if expanded {
271
                done.insert(h);
272
                out.push(h);
273
                continue;
274
            }
275
            stack.push((h, true));
276
            for d in self.store.get_change_meta(&h)?.deps {
277
                if !done.contains(&d) {
278
                    stack.push((d, false));
279
                }
280
            }
281
        }
282
        Ok(out)
283
    }
284
285
    /// Bundle changes (in the given dependency order) with their signatures,
286
    /// seals, identities, blobs, chunks.
287
    pub fn bundle_for(&self, changes: &[ChangeHash]) -> Result<Bundle> {
288
        self.bundle_of(changes, None, false, false)
289
    }
290
291
    /// Objects by reference for the `objects` endpoint; a blob brings its chunks.
292
    pub fn objects_for(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>> {
293
        // In two reads in pack order (#137): a checkpoint's graph blobs come
294
        // this way in parts, tens of thousands for a big state.
295
        let mut found = Vec::with_capacity(refs.len());
296
        for got in self.store.get_many(refs) {
297
            match got {
298
                Ok(x) => found.push(x),
299
                Err(StoreError::NotFound(_)) => {}
300
                Err(e) => return Err(e.into()),
301
            }
302
        }
303
        let mut chunks = Vec::new();
304
        for (ty, body) in &found {
305
            if *ty == ObjectType::Blob {
306
                chunks.extend(Blob::decode(body)?.chunks);
307
            }
308
        }
309
        let mut out = Vec::with_capacity(found.len() + chunks.len());
310
        for got in self.store.get_many(&chunks) {
311
            match got {
312
                Ok(x) => out.push(x),
313
                Err(StoreError::NotFound(_)) => {}
314
                Err(e) => return Err(e.into()),
315
            }
316
        }
317
        out.extend(found);
318
        Ok(out)
319
    }
320
321
    /// Like `bundle_for`, leaving out blobs larger than `max_blob` bytes
322
    /// (and their chunks) so the receiver fetches them lazily (spec §13.5).
323
    ///
324
    /// This is what a server answers a fetch with, so it also carries what
325
    /// verifies the attestations it serves beside it (see
326
    /// [`Repo::served_attest_signers`]), also when `changes` is empty.
327
    pub fn bundle_for_filtered(
328
        &self,
329
        changes: &[ChangeHash],
330
        max_blob: Option<u64>,
331
    ) -> Result<Bundle> {
332
        self.bundle_of(changes, max_blob, false, true)
333
    }
334
335
    /// Like [`Repo::bundle_for_filtered`] for a fetch between repositories
336
    /// on one machine: changes as they are stored ([`Bundle::raw_changes`]),
337
    /// with their text blobs and chunks among the objects, so neither side
338
    /// decodes or encodes their text.
339
    pub fn bundle_raw_for(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle> {
340
        self.bundle_of(changes, max_blob, true, true)
341
    }
342
343
    /// The keys that signed the attestations this repository serves with
344
    /// its references: each bookmark's latest and each tag's. A fetch
345
    /// bundle carries their identities and certificates (spec §16.2), so a
346
    /// client can check an attestation whose signer signed none of the
347
    /// changes it receives, a delegate or a short-lived key that only
348
    /// moved a bookmark (oxygenhub#266). One pointer and one signature
349
    /// list per reference; nothing per change.
350
    pub fn served_attest_signers(&self) -> Result<BTreeSet<KeyId>> {
351
        let view = self.view()?;
352
        let pointers = view
353
            .bookmarks
354
            .keys()
355
            .map(|b| self.ref_attest_path(b))
356
            .chain(view.tags.keys().map(|t| self.tag_attest_path(t)));
357
        let mut keys = BTreeSet::new();
358
        for p in pointers {
359
            let Ok(text) = fs::read_to_string(&p) else {
360
                continue;
361
            };
362
            let Some(r) = ObjectRef::from_hex(text.trim()) else {
363
                continue;
364
            };
365
            keys.extend(self.store.signatures(&r)?.into_iter().map(|s| s.key));
366
        }
367
        Ok(keys)
368
    }
369
370
    /// `serving`: a fetch's answer, which carries the identities and
371
    /// certificates of [`Repo::served_attest_signers`] too.
372
    fn bundle_of(
373
        &self,
374
        changes: &[ChangeHash],
375
        max_blob: Option<u64>,
376
        raw: bool,
377
        serving: bool,
378
    ) -> Result<Bundle> {
379
        let mut bundle = Bundle::default();
380
        let mut objects: BTreeSet<ObjectRef> = BTreeSet::new();
381
        let mut seals: BTreeSet<ObjectRef> = BTreeSet::new();
382
        let add_object = |r: &ObjectRef, objects: &mut BTreeSet<ObjectRef>| {
383
            if !r.is_zero() && self.store.has(r) {
384
                objects.insert(*r);
385
            }
386
        };
387
        for h in changes {
388
            // Raw: the stored bytes, and a skeleton for the rest.
389
            let (c, encoded) = if raw {
390
                let bytes = self.store.get_typed(h, ObjectType::Change)?;
391
                let (skel, texts) = Change::decode_skeleton(&bytes, *h)?;
392
                for t in texts {
393
                    if let crate::objects::TextRef::Blob(r, _, _) = t {
394
                        let blob = Blob::decode(&self.store.get_typed(&r, ObjectType::Blob)?)?;
395
                        objects.insert(r);
396
                        objects.extend(blob.chunks);
397
                    }
398
                }
399
                (skel, Some(bytes))
400
            } else {
401
                (self.store.get_change(h)?, None)
402
            };
403
            let sigs = self.store.signatures(h)?;
404
            add_object(&c.author, &mut objects);
405
            for s in &sigs {
406
                add_object(&s.key, &mut objects);
407
            }
408
            // A replacement's new content only: its old content is the new
409
            // content of a change it depends on, which the receiver holds or
410
            // which this bundle carries (oxygen#136: removing ~1,100 binary
411
            // files sent them all again).
412
            for e in &c.edits {
413
                if let Edit::Replace { new: Some(b), .. } = e {
414
                    if self.store.has(b) {
415
                        if let Ok(blob) = Blob::decode(&self.store.get_typed(b, ObjectType::Blob)?)
416
                        {
417
                            if max_blob.is_some_and(|m| blob.length > m) {
418
                                continue;
419
                            }
420
                            objects.insert(*b);
421
                            objects.extend(blob.chunks.iter().copied());
422
                        }
423
                    }
424
                }
425
            }
426
            for s in self.store.seals_for(h)? {
427
                seals.insert(s);
428
            }
429
            match encoded {
430
                Some(bytes) => bundle.raw_changes.push((bytes, sigs)),
431
                None => {
432
                    bundle.changes.push((c, sigs));
433
                    bundle.hashes.push(*h);
434
                }
435
            }
436
        }
437
        for r in seals {
438
            let attest = self.store.get_attest(&r)?;
439
            let sigs = self.store.signatures(&r)?;
440
            for s in &sigs {
441
                add_object(&s.key, &mut objects);
442
            }
443
            bundle.attests.push(SignedAttest { attest, sigs });
444
        }
445
        let signers: Vec<KeyId> = if serving {
446
            let signers = self.served_attest_signers()?;
447
            for k in &signers {
448
                add_object(k, &mut objects);
449
            }
450
            signers.into_iter().collect()
451
        } else {
452
            Vec::new()
453
        };
454
        for r in objects {
455
            let (ty, body) = self.store.get(&r)?;
456
            bundle.objects.push((ty, body));
457
        }
458
        self.attach_certificates_for(&mut bundle, &signers)?;
459
        Ok(bundle)
460
    }
461
462
    /// Add the certificates of short-lived keys that signed what `bundle`
463
    /// carries, and their issuers' identities, so a receiver can tell who
464
    /// they stand for (spec §14.7). `also` names more signers (a reference
465
    /// attestation sent beside the bundle).
466
    pub(crate) fn attach_certificates_for(
467
        &self,
468
        bundle: &mut Bundle,
469
        also: &[KeyId],
470
    ) -> Result<()> {
471
        let mut keys: BTreeSet<KeyId> = also.iter().copied().collect();
472
        keys.extend(
473
            bundle
474
                .changes
475
                .iter()
476
                .flat_map(|(_, s)| s.iter().map(|s| s.key)),
477
        );
478
        keys.extend(
479
            bundle
480
                .raw_changes
481
                .iter()
482
                .flat_map(|(_, s)| s.iter().map(|s| s.key)),
483
        );
484
        keys.extend(
485
            bundle
486
                .attests
487
                .iter()
488
                .flat_map(|a| a.sigs.iter().map(|s| s.key)),
489
        );
490
        for k in keys {
491
            let Some(r) = self.store.certificate_of(&k)? else {
492
                continue;
493
            };
494
            let cert = self.store.get_certificate(&r)?;
495
            if bundle
496
                .certificates
497
                .iter()
498
                .any(|c| c.cert.hash() == cert.cert.hash())
499
            {
500
                continue;
501
            }
502
            // An agent's certificate brings its grant's signer too, so the
503
            // receiver can check the grant (oxygen#134).
504
            let grantor = cert.cert.grant.as_ref().map(|g| g.grant.grantor);
505
            for id in [Some(k), Some(cert.cert.issuer), grantor].iter().flatten() {
506
                if let Ok(body) = self.store.get_typed(id, ObjectType::Identity) {
507
                    if !bundle
508
                        .objects
509
                        .iter()
510
                        .any(|(t, b)| *t == ObjectType::Identity && *b == body)
511
                    {
512
                        bundle.objects.push((ObjectType::Identity, body));
513
                    }
514
                }
515
            }
516
            bundle.certificates.push(cert);
517
        }
518
        Ok(())
519
    }
520
521
    /// Keep the certificates a bundle brings once each verifies with the
522
    /// identity of the issuer it names; one that does not refuses the bundle.
523
    fn adopt_certificates(&self, certs: &[crate::cert::SignedCertificate]) -> Result<()> {
524
        for c in certs {
525
            let issuer = self.store.get_identity(&c.cert.issuer).map_err(|_| {
526
                RepoError::Msg(format!(
527
                    "a certificate names issuer {}, whose identity did not come with it",
528
                    c.cert.issuer.short(12)
529
                ))
530
            })?;
531
            c.verify(&issuer)
532
                .map_err(|e| RepoError::Msg(format!("a certificate does not verify: {e}")))?;
533
            self.store.put_identity(&c.cert.key)?;
534
            self.store.put_certificate(c)?;
535
        }
536
        Ok(())
537
    }
538
539
    /// A signer authorized for `reference` by what an issuer certified it
540
    /// as (spec §14.7): a short-lived key with a kept certificate that
541
    /// verifies with its issuer, gives `scope`, covers `at_ms` (a time the
542
    /// signer did not choose) and names a subject the policy grants.
543
    pub(crate) fn certified_authorized(
544
        &self,
545
        p: &crate::policy::Policy,
546
        signer: &KeyId,
547
        reference: &str,
548
        scope: Scope,
549
        at_ms: u64,
550
    ) -> bool {
551
        let Ok(Some(r)) = self.store.certificate_of(signer) else {
552
            return false;
553
        };
554
        let Ok(c) = self.store.get_certificate(&r) else {
555
            return false;
556
        };
557
        let Some(issuer) = p
558
            .identity(&c.cert.issuer)
559
            .cloned()
560
            .or_else(|| self.store.get_identity(&c.cert.issuer).ok())
561
        else {
562
            return false;
563
        };
564
        c.verify(&issuer).is_ok()
565
            && c.cert.key_id() == *signer
566
            && c.cert.covers(scope, at_ms)
567
            && p.subject_grant(reference, &c.cert.issuer, &c.cert.subject, scope)
568
    }
569
570
    /// Store a bundle's contents. With `strict`, every change must
571
    /// authenticate (signature or seal by its author or a delegate).
572
    pub fn import_bundle(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
573
        self.import_bundle_inner(bundle, strict, false)
574
    }
575
576
    /// Like `import_bundle`, accepting changes whose dependencies are absent
577
    /// (they join the shallow boundary).
578
    pub fn import_bundle_shallow(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
579
        self.import_bundle_inner(bundle, strict, true)
580
    }
581
582
    fn import_bundle_inner(
583
        &self,
584
        bundle: &Bundle,
585
        strict: bool,
586
        shallow: bool,
587
    ) -> Result<Vec<ChangeHash>> {
588
        let batch = self.store.begin_batch()?;
589
        let result = self.import_bundle_batched(bundle, strict, shallow);
590
        batch.finish()?;
591
        result
592
    }
593
594
    fn import_bundle_batched(
595
        &self,
596
        bundle: &Bundle,
597
        strict: bool,
598
        shallow: bool,
599
    ) -> Result<Vec<ChangeHash>> {
600
        for (ty, _) in &bundle.objects {
601
            if !matches!(
602
                ty,
603
                ObjectType::Chunk
604
                    | ObjectType::Blob
605
                    | ObjectType::Identity
606
                    | ObjectType::State
607
                    | ObjectType::Checkpoint
608
            ) {
609
                return fail(format!("bundle may not carry {} objects", ty.name()));
610
            }
611
        }
612
        // All at once: in a large batch they are compressed on every core.
613
        let objects: Vec<(ObjectType, &[u8])> = bundle
614
            .objects
615
            .iter()
616
            .map(|(ty, body)| (*ty, body.as_slice()))
617
            .collect();
618
        self.store.put_many(&objects)?;
619
        self.adopt_certificates(&bundle.certificates)?;
620
        for sa in &bundle.attests {
621
            let r = self.store.put_attest(&sa.attest)?;
622
            for sig in &sa.sigs {
623
                self.store.put_signature(&r, sig)?;
624
            }
625
            if let AttestBody::Seal { hashes } = &sa.attest.body {
626
                for h in hashes {
627
                    self.store.index_seal(h, &r)?;
628
                }
629
            }
630
        }
631
        // Each change's hash: from the bundle when its maker knew it, else
632
        // worked out once (for a change adding a whole tree, hashing is
633
        // chunking all its text).
634
        let hashes: Vec<ChangeHash> = if bundle.hashes.len() == bundle.changes.len() {
635
            bundle.hashes.clone()
636
        } else {
637
            bundle.changes.iter().map(|(c, _)| c.hash()).collect()
638
        };
639
        let raw_hashes: Vec<ChangeHash> = bundle
640
            .raw_changes
641
            .iter()
642
            .map(|(b, _)| hash_object(ObjectType::Change, b))
643
            .collect();
644
        let incoming: BTreeSet<ChangeHash> = hashes.iter().chain(&raw_hashes).copied().collect();
645
        let boundary = self.boundary();
646
        let mut imported = Vec::new();
647
        // Changes as stored: checked, then stored as they are, their text
648
        // blobs having come with the objects.
649
        for ((bytes, sigs), h) in bundle.raw_changes.iter().zip(raw_hashes) {
650
            let (skel, texts) = Change::decode_skeleton(bytes, h)?;
651
            for d in &skel.deps {
652
                if !self.store.has(d) && !incoming.contains(d) && !shallow && !boundary.contains(d)
653
                {
654
                    return fail(format!(
655
                        "change {} depends on {}, which is neither stored nor in the bundle",
656
                        h.short(12),
657
                        d.short(12)
658
                    ));
659
                }
660
            }
661
            for t in texts {
662
                if let crate::objects::TextRef::Blob(r, _, _) = t {
663
                    if !self.store.has(&r) {
664
                        return fail(format!(
665
                            "change {} came without its text blob {}",
666
                            h.short(12),
667
                            r.short(12)
668
                        ));
669
                    }
670
                }
671
            }
672
            self.store.put(ObjectType::Change, bytes)?;
673
            for sig in sigs {
674
                self.store.put_signature(&h, sig)?;
675
            }
676
            imported.push(h);
677
        }
678
        for ((c, sigs), h) in bundle.changes.iter().zip(hashes) {
679
            for d in &c.deps {
680
                if !self.store.has(d) && !incoming.contains(d) && !shallow && !boundary.contains(d)
681
                {
682
                    return fail(format!(
683
                        "change {} depends on {}, which is neither stored nor in the bundle",
684
                        h.short(12),
685
                        d.short(12)
686
                    ));
687
                }
688
            }
689
            let stored = self.store.put_change(c)?;
690
            if stored != h {
691
                return fail(format!(
692
                    "change {} is not the change the bundle names",
693
                    h.short(12)
694
                ));
695
            }
696
            for sig in sigs {
697
                self.store.put_signature(&h, sig)?;
698
            }
699
            imported.push(h);
700
        }
701
        if strict {
702
            for h in &imported {
703
                if !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. }) {
704
                    return fail(format!(
705
                        "change {} is not authenticated by its author",
706
                        h.short(12)
707
                    ));
708
                }
709
            }
710
        }
711
        Ok(imported)
712
    }
713
714
    /// Verify a reference attestation against an update it claims to describe.
715
    /// `allow_equal` lets a client re-read the attestation it already has; a
716
    /// server receiving a push requires a strictly greater serial.
717
    /// A reference attestation signed by a delegate of an identity the
718
    /// rule lists is authorized as that identity (spec §14.5): the identity
719
    /// named the delegate with the `refs` scope, still in force at `at_ms`
720
    /// (the attestation's wall time), and the policy names the identity.
721
    /// Revoked keys authorize nothing either way.
722
    pub(crate) fn delegate_authorized(
723
        &self,
724
        p: &crate::policy::Policy,
725
        signer: &KeyId,
726
        reference: &str,
727
        at_ms: u64,
728
    ) -> bool {
729
        if p.is_revoked(signer) {
730
            return false;
731
        }
732
        let Some(rule) = p.rule_for(reference) else {
733
            return false;
734
        };
735
        rule.keys.iter().any(|k| {
736
            !p.is_revoked(k)
737
                && p.identity(k)
738
                    .cloned()
739
                    .or_else(|| self.store.get_identity(k).ok())
740
                    .is_some_and(|id| id.delegates_to(signer, Scope::REFS, at_ms))
741
        })
742
    }
743
744
    /// A lasting attestation (a tag) signed by a key the policy has since
745
    /// rotated away from (spec §14.2) is authorized as the key that
746
    /// succeeded it: a key the rule lists reaches the signer through
747
    /// `previous` links, and the attestation is dated (`at_ms`) before the
748
    /// signer's successor was made, so a superseded key makes nothing new.
749
    /// Revoked keys authorize nothing.
750
    pub(crate) fn predecessor_authorized(
751
        &self,
752
        p: &crate::policy::Policy,
753
        signer: &KeyId,
754
        reference: &str,
755
        at_ms: u64,
756
    ) -> bool {
757
        if p.is_revoked(signer) {
758
            return false;
759
        }
760
        let Some(rule) = p.rule_for(reference) else {
761
            return false;
762
        };
763
        let identity = |k: &KeyId| {
764
            p.identity(k)
765
                .cloned()
766
                .or_else(|| self.store.get_identity(k).ok())
767
        };
768
        rule.keys.iter().any(|k| {
769
            if p.is_revoked(k) {
770
                return false;
771
            }
772
            let mut cur = identity(k);
773
            for _ in 0..32 {
774
                let Some(id) = cur else {
775
                    return false;
776
                };
777
                match id.previous {
778
                    Some(prev) if prev == *signer => return at_ms < id.created,
779
                    Some(prev) if !p.is_revoked(&prev) => cur = identity(&prev),
780
                    _ => return false,
781
                }
782
            }
783
            false
784
        })
785
    }
786
787
    fn check_ref_attest(
788
        &self,
789
        sa: &SignedAttest,
790
        name: &str,
791
        old_state: &ObjectRef,
792
        new_state: &ObjectRef,
793
        last_serial: Option<u64>,
794
        allow_equal: bool,
795
    ) -> std::result::Result<u64, String> {
796
        let serial =
797
            self.check_ref_attest_body(sa, name, old_state, new_state, last_serial, allow_equal)?;
798
        self.verify_attest_signature(sa)?;
799
        Ok(serial)
800
    }
801
802
    /// What [`Repo::check_ref_attest`] checks short of the signature: the
803
    /// attestation describes this update of this repository with a serial
804
    /// past the last seen.
805
    fn check_ref_attest_body(
806
        &self,
807
        sa: &SignedAttest,
808
        name: &str,
809
        old_state: &ObjectRef,
810
        new_state: &ObjectRef,
811
        last_serial: Option<u64>,
812
        allow_equal: bool,
813
    ) -> std::result::Result<u64, String> {
814
        let AttestBody::Ref {
815
            name: n,
816
            old_state: o,
817
            new_state: nw,
818
            serial,
819
        } = &sa.attest.body
820
        else {
821
            return Err("attestation is not a ref attestation".into());
822
        };
823
        if n != name || o != old_state || nw != new_state {
824
            return Err("attestation does not describe this update".into());
825
        }
826
        if sa.attest.repo != self.id().map_err(|e| e.to_string())? {
827
            return Err("attestation is for a different repository".into());
828
        }
829
        if let Some(last) = last_serial {
830
            if *serial < last || (*serial == last && !allow_equal) {
831
                return Err(format!(
832
                    "attestation serial {serial} is not greater than the last seen {last}"
833
                ));
834
            }
835
        }
836
        Ok(*serial)
837
    }
838
839
    /// An attestation's first signature verifies with its signer's
840
    /// identity, which must be stored here.
841
    fn verify_attest_signature(&self, sa: &SignedAttest) -> std::result::Result<(), String> {
842
        let r = sa.attest.hash();
843
        let sig = sa.sigs.first().ok_or("attestation carries no signature")?;
844
        let id = self
845
            .store
846
            .get_identity(&sig.key)
847
            .map_err(|_| format!("unknown key {}", sig.key.short(12)))?;
848
        verify_signature(&id, sig, Domain::Attest, &r).map_err(|e| e.to_string())
849
    }
850
851
    /// Accept a push (spec §16.4). Without a trust policy every repository
852
    /// is unverified, so unsigned changes and unattested updates are accepted;
853
    /// attestations that are present must still be valid. With a policy,
854
    /// every change and update is checked against it. A newer policy in the
855
    /// push is applied first when it is signed by the current roots.
856
    pub fn receive_push(
857
        &self,
858
        bundle: Bundle,
859
        updates: Vec<RefUpdate>,
860
        policy: Option<SignedPolicy>,
861
        tags: Vec<crate::protocol::TagRef>,
862
    ) -> Result<Vec<RefResult>> {
863
        Ok(self
864
            .receive_push_report(bundle, updates, policy, tags)?
865
            .results)
866
    }
867
868
    /// `receive_push` with what a host wants to log: each accepted
869
    /// reference's old and new state and heads, and the changes this push
870
    /// stored (hub spec §6.12).
871
    pub fn receive_push_report(
872
        &self,
873
        bundle: Bundle,
874
        updates: Vec<RefUpdate>,
875
        policy: Option<SignedPolicy>,
876
        tags: Vec<crate::protocol::TagRef>,
877
    ) -> Result<PushReport> {
878
        // Updates are checked against the head as it is now, not as this
879
        // handle first read it (oxygenhub#249): another push may have
880
        // landed since.
881
        self.refresh()?;
882
        let mut report = PushReport::default();
883
        let results = &mut report.results;
884
        if let Some(sp) = policy {
885
            match self.adopt_policy_chain(&bundle.policies, &sp, "push received") {
886
                Ok(()) => {
887
                    results.push(RefResult {
888
                        name: "policy".into(),
889
                        ok: true,
890
                        reason: format!("serial {}", sp.policy.serial),
891
                    });
892
                }
893
                Err(e) => {
894
                    results.push(RefResult {
895
                        name: "policy".into(),
896
                        ok: false,
897
                        reason: e,
898
                    });
899
                    for u in updates {
900
                        results.push(RefResult {
901
                            name: u.name,
902
                            ok: false,
903
                            reason: "policy update was rejected".into(),
904
                        });
905
                    }
906
                    return Ok(report);
907
                }
908
            }
909
        }
910
        let policy = self.current_policy()?.map(|(_, sp)| sp.policy);
911
        // A push of objects alone (one batch of a long history, oxygen#95)
912
        // names no reference: its outcome is reported as `objects`.
913
        let objects_only = updates.is_empty() && tags.is_empty();
914
        match self.import_bundle(&bundle, policy.is_some()) {
915
            Ok(imported) => {
916
                if objects_only {
917
                    results.push(RefResult {
918
                        name: OBJECTS_RESULT.into(),
919
                        ok: true,
920
                        reason: format!("{} changes stored", imported.len()),
921
                    });
922
                }
923
                report.imported = imported;
924
            }
925
            Err(e) => {
926
                if objects_only {
927
                    results.push(RefResult {
928
                        name: OBJECTS_RESULT.into(),
929
                        ok: false,
930
                        reason: e.to_string(),
931
                    });
932
                }
933
                for u in updates {
934
                    results.push(RefResult {
935
                        name: u.name,
936
                        ok: false,
937
                        reason: e.to_string(),
938
                    });
939
                }
940
                return Ok(report);
941
            }
942
        }
943
        let (op_ref, op) = self.head_op()?;
944
        let mut view = op.view.clone();
945
        let mut names = Vec::new();
946
        for u in updates {
947
            let outcome = self.apply_update(&mut view, &u, policy.as_ref());
948
            let (ok, reason) = match outcome {
949
                Ok(()) => (true, String::new()),
950
                Err(e) => (false, e),
951
            };
952
            if ok {
953
                names.push(u.name.clone());
954
                let heads = self
955
                    .store
956
                    .get_state(&u.new_state)
957
                    .map(|s| s.heads)
958
                    .unwrap_or_default();
959
                report.updated.push(RefUpdated {
960
                    name: u.name.clone(),
961
                    old_state: u.old_state,
962
                    new_state: u.new_state,
963
                    heads,
964
                });
965
            }
966
            results.push(RefResult {
967
                name: u.name,
968
                ok,
969
                reason,
970
            });
971
        }
972
        for t in &tags {
973
            let name = format!("tags/{}", t.name);
974
            let new = !view.tags.contains_key(&t.name);
975
            match self.adopt_tag(&mut view, t, policy.as_ref(), &mut TagCheck::Received) {
976
                Ok(()) => {
977
                    // A tag new here is an update too, so a hub can start
978
                    // its pipeline's tag jobs (oxygenflow#24); one it
979
                    // already had, sent again, is not.
980
                    if new {
981
                        report.updated.push(RefUpdated {
982
                            name: name.clone(),
983
                            old_state: ObjectRef::ZERO,
984
                            new_state: t.state,
985
                            heads: t.heads.clone(),
986
                        });
987
                    }
988
                    names.push(name.clone());
989
                    results.push(RefResult {
990
                        name,
991
                        ok: true,
992
                        reason: String::new(),
993
                    });
994
                }
995
                Err(e) => results.push(RefResult {
996
                    name,
997
                    ok: false,
998
                    reason: e.reason,
999
                }),
1000
            }
1001
        }
1002
        if !names.is_empty() {
1003
            self.record_op(
1004
                &[op_ref],
1005
                &format!("push received: {}", names.join(" ")),
1006
                view,
1007
            )?;
1008
        }
1009
        Ok(report)
1010
    }
1011
1012
    /// Check an update's attestation against the latest one for its
1013
    /// reference and store it as the new latest. Failures say what was
1014
    /// being done, for which reference, so a refused push can be diagnosed
1015
    /// from its reason alone.
1016
    fn record_update_attest(
1017
        &self,
1018
        u: &RefUpdate,
1019
        sa: &SignedAttest,
1020
    ) -> std::result::Result<(), String> {
1021
        let name = &u.name;
1022
        let last = self
1023
            .latest_ref_attest(name)
1024
            .map_err(|e| format!("reading the last attestation of {name:?}: {e}"))?
1025
            .and_then(|(_, a)| match a.attest.body {
1026
                AttestBody::Ref { serial, .. } => Some(serial),
1027
                _ => None,
1028
            });
1029
        self.check_ref_attest(sa, name, &u.old_state, &u.new_state, last, false)?;
1030
        let store = |e: &dyn std::fmt::Display| format!("storing the attestation of {name:?}: {e}");
1031
        let r = self.store.put_attest(&sa.attest).map_err(|e| store(&e))?;
1032
        for sig in &sa.sigs {
1033
            self.store.put_signature(&r, sig).map_err(|e| store(&e))?;
1034
        }
1035
        self.set_ref_attest(name, &r).map_err(|e| store(&e))?;
1036
        // Signed by a short-lived key: a host records when it accepted it.
1037
        let certified = sa
1038
            .sigs
1039
            .first()
1040
            .is_some_and(|s| self.store.certificate_of(&s.key).ok().flatten().is_some());
1041
        if certified {
1042
            self.record_acceptance(&r).map_err(|e| store(&e))?;
1043
        }
1044
        Ok(())
1045
    }
1046
1047
    /// Record that this host accepted attestation `r` now, signed with its
1048
    /// acceptor key (spec §14.7); nothing without one.
1049
    pub(crate) fn record_acceptance(&self, r: &ObjectRef) -> Result<()> {
1050
        let Some(acceptor) = &self.acceptor else {
1051
            return Ok(());
1052
        };
1053
        let (id, sk) = (&acceptor.0, &acceptor.1);
1054
        let attest = Attest {
1055
            repo: self.id()?,
1056
            hlc: Hlc::now(),
1057
            body: AttestBody::Accepted { attest: *r },
1058
        };
1059
        let sig = sk.sign(Domain::Attest, &id.hash(), &attest.hash());
1060
        self.store.put_identity(id)?;
1061
        let a = self.store.put_attest(&attest)?;
1062
        self.store.put_signature(&a, &sig)?;
1063
        self.store.index_add_pub("accepted", r, &a)?;
1064
        Ok(())
1065
    }
1066
1067
    /// The host's acceptance of attestation `r`, if one was recorded.
1068
    pub fn acceptance_of(&self, r: &ObjectRef) -> Result<Option<SignedAttest>> {
1069
        let Some(a) = self.store.index_get_pub("accepted", r)?.into_iter().next() else {
1070
            return Ok(None);
1071
        };
1072
        Ok(Some(SignedAttest {
1073
            attest: self.store.get_attest(&a)?,
1074
            sigs: self.store.signatures(&a)?,
1075
        }))
1076
    }
1077
1078
    /// When a certified signer's attestation `sa` is checked on a fetch: the
1079
    /// host's acceptance time, if `accepted` is signed by the issuer of the
1080
    /// signer's certificate and accepts `sa`; else the attestation's own
1081
    /// time, which the signer chose.
1082
    pub(crate) fn certified_time(
1083
        &self,
1084
        signer: &KeyId,
1085
        sa: &SignedAttest,
1086
        accepted: Option<&SignedAttest>,
1087
    ) -> u64 {
1088
        self.issuer_acceptance(signer, sa, accepted)
1089
            .unwrap_or(sa.attest.hlc.wall_ms)
1090
    }
1091
1092
    /// The time of `accepted` when it accepts exactly `sa` and is signed by
1093
    /// the issuer of `signer`'s kept certificate (spec §14.7); else none.
1094
    pub(crate) fn issuer_acceptance(
1095
        &self,
1096
        signer: &KeyId,
1097
        sa: &SignedAttest,
1098
        accepted: Option<&SignedAttest>,
1099
    ) -> Option<u64> {
1100
        let acc = accepted?;
1101
        let c = self.store.certificate_of(signer).ok().flatten()?;
1102
        let cert = self.store.get_certificate(&c).ok()?;
1103
        let AttestBody::Accepted { attest } = &acc.attest.body else {
1104
            return None;
1105
        };
1106
        let sig = acc.sigs.iter().find(|s| s.key == cert.cert.issuer)?;
1107
        let issuer = self.store.get_identity(&cert.cert.issuer).ok()?;
1108
        (*attest == sa.attest.hash()
1109
            && verify_signature(&issuer, sig, Domain::Attest, &acc.attest.hash()).is_ok())
1110
        .then_some(acc.attest.hlc.wall_ms)
1111
    }
1112
1113
    /// Keep an acceptance this repository was given for attestation `r`
1114
    /// (one [`Repo::issuer_acceptance`] took), so it is served again with
1115
    /// what it accepts.
1116
    pub(crate) fn keep_acceptance(&self, r: &ObjectRef, acc: &SignedAttest) -> Result<()> {
1117
        let a = self.store.put_attest(&acc.attest)?;
1118
        for sig in &acc.sigs {
1119
            self.store.put_signature(&a, sig)?;
1120
        }
1121
        self.store.index_add_pub("accepted", r, &a)?;
1122
        Ok(())
1123
    }
1124
1125
    /// A fetched bookmark's attestation by a signer the policy does not
1126
    /// list itself: a delegate or a short-lived key, whose authority
1127
    /// depends on when it signed (spec §14.5, §14.8). Checked at the
1128
    /// attestation's integrated time when the host's log gives one; else
1129
    /// as before, at the host's acceptance for a certified signer, when it
1130
    /// countersigned one, or at the attestation's own time, and said so
1131
    /// when the host keeps a log. Failing at the integrated time but not
1132
    /// as before is a note while logs are new; a large gap between the
1133
    /// claimed and the integrated time is a note.
1134
    #[allow(clippy::too_many_arguments)]
1135
    pub(crate) fn authorize_in_time(
1136
        &self,
1137
        p: &crate::policy::Policy,
1138
        remote_name: &str,
1139
        r: &crate::protocol::RemoteRef,
1140
        sa: &SignedAttest,
1141
        signer: &KeyId,
1142
        log: &super::transparency::LogCheck,
1143
        notes: &mut Vec<String>,
1144
    ) -> Result<()> {
1145
        use super::transparency::{gap_note, judge, Timed};
1146
        let own = sa.attest.hlc.wall_ms;
1147
        let at = self.certified_time(signer, sa, r.accepted.as_ref());
1148
        let time = log.time_of(&r.name);
1149
        let check = |t_delegate: u64, t_cert: u64| {
1150
            self.delegate_authorized(p, signer, &r.name, t_delegate)
1151
                || self.certified_authorized(p, signer, &r.name, Scope::REFS, t_cert)
1152
        };
1153
        let before = if at != own {
1154
            "the host's acceptance"
1155
        } else {
1156
            "the time it claims"
1157
        };
1158
        let what = format!("{remote_name}/{}", r.name);
1159
        match judge(time.as_ref().ok().map(|i| i.ms()), |t| check(t, t), || check(own, at)) {
1160
            Timed::Refused => {
1161
                return fail(format!(
1162
                    "{what}: attested by {}, which the policy does not authorize for this bookmark; a policy owner can grant it with `oxy policy grant`",
1163
                    signer.short(12)
1164
                ))
1165
            }
1166
            Timed::Integrated => {}
1167
            Timed::Disagrees => notes.push(format!(
1168
                "{what}: {}'s authority had lapsed when witness {} first saw its attestation, though not at {before}; accepted while transparency logs are new",
1169
                signer.short(12),
1170
                time.as_ref().map(|i| i.witness.as_str()).unwrap_or("?")
1171
            )),
1172
            Timed::Fallback if log.kept => notes.push(format!(
1173
                "{what}: {} checked at {before}, not at a witnessed time: {}",
1174
                signer.short(12),
1175
                time.as_ref().err().map(String::as_str).unwrap_or("none")
1176
            )),
1177
            Timed::Fallback => {}
1178
        }
1179
        if let Some(n) = time.ok().and_then(|i| gap_note(own, i)) {
1180
            notes.push(format!("{what}'s attestation {n}"));
1181
        }
1182
        Ok(())
1183
    }
1184
1185
    /// A deletion (spec §16.4): `new_state` is zero. The bookmark must exist
1186
    /// at `old_state`, is never the bookmark a non-bare repository has
1187
    /// checked out (a bare one's default is its host's to protect, see
1188
    /// `oxy_server::Backend::default_bookmark`), and under a policy needs
1189
    /// an attestation by a key that may update it.
1190
    /// The deletion's attestation stays the bookmark's latest, so an older
1191
    /// one cannot be replayed to bring it back.
1192
    fn apply_deletion(
1193
        &self,
1194
        view: &mut View,
1195
        u: &RefUpdate,
1196
        policy: Option<&crate::policy::Policy>,
1197
    ) -> std::result::Result<(), String> {
1198
        let Some(current) = view.bookmarks.get(&u.name).copied() else {
1199
            return Err(format!("no bookmark {:?} to delete", u.name));
1200
        };
1201
        if current != u.old_state {
1202
            return Err("stale: the remote bookmark has moved; pull first".into());
1203
        }
1204
        if !self.is_bare() && u.name == self.config.bookmark {
1205
            return Err(format!(
1206
                "{:?} is this repository's default bookmark (checked out there) and cannot be deleted",
1207
                u.name
1208
            ));
1209
        }
1210
        if let Some(p) = policy {
1211
            let Some(sa) = &u.attestation else {
1212
                return Err(format!(
1213
                    "deleting {:?} requires a signed reference attestation under this repository's policy",
1214
                    u.name
1215
                ));
1216
            };
1217
            let signer = sa
1218
                .sigs
1219
                .first()
1220
                .map(|s| s.key)
1221
                .ok_or("attestation carries no signature")?;
1222
            // Received now: the receiver's own clock is the trusted time.
1223
            let now = Hlc::now().wall_ms;
1224
            if !p.authorizes(&signer, &u.name)
1225
                && !self.delegate_authorized(p, &signer, &u.name, sa.attest.hlc.wall_ms)
1226
                && !self.certified_authorized(p, &signer, &u.name, Scope::REFS, now)
1227
            {
1228
                return Err(format!(
1229
                    "key {} may not delete {:?} under this repository's policy",
1230
                    signer.short(12),
1231
                    u.name
1232
                ));
1233
            }
1234
        }
1235
        if let Some(sa) = &u.attestation {
1236
            self.record_update_attest(u, sa)?;
1237
        }
1238
        view.bookmarks.remove(&u.name);
1239
        Ok(())
1240
    }
1241
1242
    fn apply_update(
1243
        &self,
1244
        view: &mut View,
1245
        u: &RefUpdate,
1246
        policy: Option<&crate::policy::Policy>,
1247
    ) -> std::result::Result<(), String> {
1248
        if u.name == "policy" {
1249
            return Err(
1250
                "the policy reference is updated by sending a policy, not a bookmark update".into(),
1251
            );
1252
        }
1253
        let empty = self
1254
            .store
1255
            .put_state(&StateObject::new(vec![]))
1256
            .map_err(|e| e.to_string())?;
1257
        if u.new_state.is_zero() {
1258
            return self.apply_deletion(view, u, policy);
1259
        }
1260
        let current = view.bookmarks.get(&u.name).copied().unwrap_or(empty);
1261
        if current != u.old_state {
1262
            return Err("stale: the remote bookmark has moved; pull first".into());
1263
        }
1264
        let new_state = self
1265
            .store
1266
            .get_state(&u.new_state)
1267
            .map_err(|_| "new state object is missing".to_string())?;
1268
        for h in &new_state.heads {
1269
            if !self.store.has(h) {
1270
                return Err(format!("head {} is missing", h.short(12)));
1271
            }
1272
        }
1273
        if !u.force {
1274
            let old_heads = self
1275
                .store
1276
                .get_state(&current)
1277
                .map_err(|e| e.to_string())?
1278
                .heads;
1279
            let closure = self
1280
                .closure_in_store(&new_state.heads)
1281
                .map_err(|e| e.to_string())?;
1282
            if let Some(h) = old_heads.iter().find(|h| !closure.contains(h)) {
1283
                return Err(format!(
1284
                    "not a fast-forward: current head {} would be dropped (use --force)",
1285
                    h.short(12)
1286
                ));
1287
            }
1288
        }
1289
        if let Some(p) = policy {
1290
            let Some(sa) = &u.attestation else {
1291
                return Err(format!(
1292
                    "{:?} requires a signed reference attestation under this repository's policy",
1293
                    u.name
1294
                ));
1295
            };
1296
            let signer = sa
1297
                .sigs
1298
                .first()
1299
                .map(|s| s.key)
1300
                .ok_or("attestation carries no signature")?;
1301
            // Received now: the receiver's own clock is the trusted time.
1302
            let now = Hlc::now().wall_ms;
1303
            if !p.authorizes(&signer, &u.name)
1304
                && !self.delegate_authorized(p, &signer, &u.name, sa.attest.hlc.wall_ms)
1305
                && !self.certified_authorized(p, &signer, &u.name, Scope::REFS, now)
1306
            {
1307
                return Err(format!(
1308
                    "key {} may not update {:?} under this repository's policy",
1309
                    signer.short(12),
1310
                    u.name
1311
                ));
1312
            }
1313
            let old_heads = self
1314
                .store
1315
                .get_state(&current)
1316
                .map_err(|e| e.to_string())?
1317
                .heads;
1318
            self.state_allowed(&old_heads, &new_state.heads, p, &u.name)?;
1319
        }
1320
        if let Some(sa) = &u.attestation {
1321
            self.record_update_attest(u, sa)?;
1322
        }
1323
        view.bookmarks.insert(u.name.clone(), u.new_state);
1324
        Ok(())
1325
    }
1326
1327
    // ------------------------------------------------------------ client side
1328
1329
    fn remote_url(&self, name: &str) -> Result<String> {
1330
        self.config.remotes.get(name).cloned().ok_or_else(|| {
1331
            let names: Vec<&str> = self.config.remotes.keys().map(String::as_str).collect();
1332
            RepoError::Msg(match names.as_slice() {
1333
                [] => format!("no remote named {name:?}; add one with `oxy remote add`"),
1334
                _ if name == "origin" && names.len() > 1 && self.config.default_remote.is_none() => format!(
1335
                    "this repository has no remote named origin, and several others ({}): name one (`oxy pull {}`), or make one the default with `oxy config core.remote {}`",
1336
                    names.join(", "),
1337
                    names[0],
1338
                    names[0]
1339
                ),
1340
                _ => format!(
1341
                    "no remote named {name:?}; this repository has {}",
1342
                    names.join(", ")
1343
                ),
1344
            })
1345
        })
1346
    }
1347
1348
    /// Refuse a remote that is another repository. An **unbound** remote
1349
    /// (id all zero: a hosted repository nobody has pushed to) is accepted
1350
    /// for pushes, which bind it to this repository's id (spec §16.1).
1351
    fn check_remote_id(
1352
        &self,
1353
        remote: &dyn crate::protocol::Remote,
1354
        remote_name: &str,
1355
        allow_unbound: bool,
1356
    ) -> Result<()> {
1357
        let theirs = remote.repo_id()?;
1358
        if theirs == self.id()? {
1359
            return Ok(());
1360
        }
1361
        if theirs == crate::protocol::UNBOUND_ID {
1362
            if allow_unbound {
1363
                return Ok(());
1364
            }
1365
            return fail(format!(
1366
                "remote {remote_name:?} is empty and unbound: push to it first"
1367
            ));
1368
        }
1369
        fail(format!(
1370
            "remote {remote_name:?} is a different repository (repository IDs differ); check its URL with `oxy remote list`"
1371
        ))
1372
    }
1373
1374
    /// Credentials and trust for a remote: `remote.<name>.token` and
1375
    /// `remote.<name>.ca`, else the defaults for its URL
1376
    /// ([`Repo::fill_credentials`]).
1377
    pub(crate) fn remote_options(&self, name: &str) -> RemoteOptions {
1378
        let url = self.config.remotes.get(name).cloned().unwrap_or_default();
1379
        let mut options = RemoteOptions {
1380
            repo_id: self.id().ok(),
1381
            token: self.config.remote_tokens.get(name).cloned(),
1382
            ca: self.config.remote_cas.get(name).map(PathBuf::from),
1383
            cache_dir: Some(self.oxy.join("remotes").join(name).join("strata")),
1384
            ..RemoteOptions::default()
1385
        };
1386
        self.fill_credentials(&url, &mut options);
1387
        options
1388
    }
1389
1390
    /// The user's credential store (`credentials.toml` beside the user
1391
    /// configuration file).
1392
    pub fn credentials(&self) -> crate::credentials::Credentials {
1393
        crate::credentials::Credentials::at(
1394
            self.user_config
1395
                .parent()
1396
                .map(Path::to_path_buf)
1397
                .unwrap_or_else(default_home),
1398
        )
1399
    }
1400
1401
    /// Fill in what `options` lacks for `url`: `$OXY_TOKEN` and `$OXY_CA`,
1402
    /// then the user's credential store for its hub, then a remote on the
1403
    /// same hub in an enclosing checkout (a subrepo borrows its parent's).
1404
    /// When the token is not this remote's own or the environment's, the
1405
    /// other tokens enclosing checkouts hold for the hub are kept to try
1406
    /// if it is refused: a stale stored login must not hide a parent's
1407
    /// good token (oxygen#112).
1408
    fn fill_credentials(&self, url: &str, options: &mut RemoteOptions) {
1409
        if options.token.is_some() && options.ca.is_some() {
1410
            return;
1411
        }
1412
        let own = options.token.is_some();
1413
        let env = std::env::var("OXY_TOKEN").is_ok_and(|t| !t.is_empty());
1414
        let explicit = own || env;
1415
        let stored = self.credentials().options_for(url);
1416
        // A checkout's own token refused as expired or unknown (401) falls
1417
        // back to this computer's sign-in for the hub, never to another
1418
        // checkout's, and never when it is refused for permission (#155).
1419
        if own && !env {
1420
            if let Some(t) = stored
1421
                .token
1422
                .clone()
1423
                .filter(|t| options.token.as_ref() != Some(t))
1424
            {
1425
                options.fallback_tokens.push(t);
1426
                options.fallback_on_401_only = true;
1427
            }
1428
        }
1429
        options.token = options.token.take().or(stored.token);
1430
        options.ca = options.ca.take().or(stored.ca);
1431
        let enclosing = self.enclosing_credentials(url);
1432
        if options.token.is_none() {
1433
            options.token = enclosing.iter().find_map(|(t, _)| t.clone());
1434
        }
1435
        if options.ca.is_none() {
1436
            options.ca = enclosing.iter().find_map(|(_, c)| c.clone());
1437
        }
1438
        if !explicit {
1439
            for t in enclosing.into_iter().filter_map(|(t, _)| t) {
1440
                if options.token.as_ref() != Some(&t) && !options.fallback_tokens.contains(&t) {
1441
                    options.fallback_tokens.push(t);
1442
                }
1443
            }
1444
        }
1445
    }
1446
1447
    /// What to do when `remote`'s own token was refused (`message` says
1448
    /// HTTP 401) and so was this computer's sign-in for its hub, which it
1449
    /// falls back to (oxygen#155).
1450
    pub fn stale_token_note(&self, remote: &str, message: &str) -> Option<String> {
1451
        if !message.contains("HTTP 401") {
1452
            return None;
1453
        }
1454
        let own = self.config.remote_tokens.get(remote)?;
1455
        let url = self.config.remotes.get(remote)?;
1456
        let stored = self.credentials().options_for(url).token?;
1457
        (stored != *own).then(|| {
1458
            format!(
1459
                "the hub refused both this checkout's own token for {remote} (remote.{remote}.token) and your sign-in: \
1460
                 `oxy auth login HUB`, then `oxy config --unset remote.{remote}.token` so the sign-in is used"
1461
            )
1462
        })
1463
    }
1464
1465
    /// The tokens and CAs of remotes on `url`'s hub in enclosing
1466
    /// checkouts, nearest first.
1467
    fn enclosing_credentials(&self, url: &str) -> Vec<(Option<String>, Option<PathBuf>)> {
1468
        let mut out = Vec::new();
1469
        let Some(hub) = crate::credentials::hub_of(url) else {
1470
            return out;
1471
        };
1472
        let mut at = self.root.parent();
1473
        while let Some(dir) = at {
1474
            let Ok(cfg) = read_config_file(&dir.join(OXY_DIR).join("config")) else {
1475
                break;
1476
            };
1477
            for (key, remote_url) in &cfg {
1478
                let Some(name) = key
1479
                    .strip_prefix("remote.")
1480
                    .and_then(|k| k.strip_suffix(".url"))
1481
                else {
1482
                    continue;
1483
                };
1484
                if crate::credentials::hub_of(remote_url).as_deref() != Some(hub.as_str()) {
1485
                    continue;
1486
                }
1487
                out.push((
1488
                    cfg.get(&format!("remote.{name}.token")).cloned(),
1489
                    cfg.get(&format!("remote.{name}.ca")).map(PathBuf::from),
1490
                ));
1491
            }
1492
            at = dir.parent();
1493
        }
1494
        out
1495
    }
1496
1497
    /// Push seal attestations on their own (hub spec §7.4): a bundle with
1498
    /// no changes and no reference updates, which the receiver stores.
1499
    pub fn push_seals(&self, remote_name: &str, seals: &[ObjectRef]) -> Result<()> {
1500
        let remote = self.open_named_remote(remote_name)?;
1501
        self.check_remote_id(remote.as_ref(), remote_name, false)?;
1502
        let mut bundle = Bundle::default();
1503
        for r in seals {
1504
            let attest = self.store.get_attest(r)?;
1505
            let sigs = self.store.signatures(r)?;
1506
            bundle
1507
                .attests
1508
                .push(crate::protocol::SignedAttest { attest, sigs });
1509
        }
1510
        let results = remote.push(bundle, vec![], None, vec![])?;
1511
        if let Some(r) = results.iter().find(|r| !r.ok) {
1512
            return fail(format!("{}: {}", r.name, r.reason));
1513
        }
1514
        Ok(())
1515
    }
1516
1517
    /// The remote whose URL starts with `base` (a hub), if configured.
1518
    pub fn remote_for_url(&self, base: &str) -> Option<String> {
1519
        let base = base.trim_end_matches('/');
1520
        self.config
1521
            .remotes
1522
            .iter()
1523
            .find(|(_, url)| url.trim_end_matches('/') == base)
1524
            .map(|(name, _)| name.clone())
1525
    }
1526
1527
    /// Open a remote by name or, failing that, by URL with the default
1528
    /// credentials (see [`Repo::credentials_for`]).
1529
    pub fn open_remote_at(&self, url: &str) -> Result<(String, Box<dyn crate::protocol::Remote>)> {
1530
        if let Some(name) = self.remote_for_url(url) {
1531
            return Ok((name.clone(), self.open_named_remote(&name)?));
1532
        }
1533
        let options = RemoteOptions {
1534
            repo_id: self.id().ok(),
1535
            ..self.credentials_for(url)
1536
        };
1537
        Ok((url.to_string(), open_remote(url, &options)?))
1538
    }
1539
1540
    /// Credentials for a URL: those of the configured remote with that
1541
    /// URL, else the environment's.
1542
    pub fn options_for_url(&self, url: &str) -> RemoteOptions {
1543
        match self.remote_for_url(url) {
1544
            Some(name) => self.remote_options(&name),
1545
            None => RemoteOptions {
1546
                repo_id: self.id().ok(),
1547
                ..self.credentials_for(url)
1548
            },
1549
        }
1550
    }
1551
1552
    pub(crate) fn open_named_remote(&self, name: &str) -> Result<Box<dyn crate::protocol::Remote>> {
1553
        open_remote(&self.remote_url(name)?, &self.remote_options(name))
1554
    }
1555
1556
    fn serial_path(&self, remote: &str, bookmark: &str) -> PathBuf {
1557
        self.oxy
1558
            .join("remotes")
1559
            .join(remote)
1560
            .join(format!("{bookmark}.serial"))
1561
    }
1562
1563
    fn last_seen_serial(&self, remote: &str, bookmark: &str) -> Option<u64> {
1564
        fs::read_to_string(self.serial_path(remote, bookmark))
1565
            .ok()
1566
            .and_then(|s| s.trim().parse().ok())
1567
    }
1568
1569
    fn record_serial(&self, remote: &str, bookmark: &str, serial: u64) -> Result<()> {
1570
        let p = self.serial_path(remote, bookmark);
1571
        fs::create_dir_all(p.parent().unwrap())?;
1572
        write_atomic(&p, serial.to_string().as_bytes())
1573
    }
1574
1575
    fn policy_serial_path(&self, remote: &str) -> PathBuf {
1576
        self.oxy.join("remotes").join(remote).join("policy.serial")
1577
    }
1578
1579
    /// The highest policy serial `remote` has ever presented.
1580
    fn remote_policy_serial(&self, remote: &str) -> Option<u64> {
1581
        fs::read_to_string(self.policy_serial_path(remote))
1582
            .ok()
1583
            .and_then(|s| s.trim().parse().ok())
1584
    }
1585
1586
    fn record_policy_serial(&self, remote: &str, serial: u64) -> Result<()> {
1587
        let p = self.policy_serial_path(remote);
1588
        fs::create_dir_all(p.parent().unwrap())?;
1589
        write_atomic(&p, serial.to_string().as_bytes())
1590
    }
1591
1592
    fn lazy_dir(&self) -> PathBuf {
1593
        self.oxy.join("lazy")
1594
    }
1595
1596
    /// Remember that `blob` was withheld by `remote` (spec §13.5).
1597
    fn register_lazy(&self, blob: &ObjectRef, remote: &str) -> Result<()> {
1598
        fs::create_dir_all(self.lazy_dir())?;
1599
        write_atomic(&self.lazy_dir().join(blob.to_hex()), remote.as_bytes())
1600
    }
1601
1602
    /// Which remote promised an absent blob, if any.
1603
    pub fn lazy_remote(&self, blob: &ObjectRef) -> Option<String> {
1604
        fs::read_to_string(self.lazy_dir().join(blob.to_hex()))
1605
            .ok()
1606
            .map(|s| s.trim().to_string())
1607
    }
1608
1609
    /// Fetch an absent blob (and its chunks) from the remote that promised it.
1610
    pub(crate) fn fetch_lazy_blob(&self, blob: &ObjectRef) -> Result<()> {
1611
        let Some(remote_name) = self.lazy_remote(blob) else {
1612
            return Err(RepoError::Store(StoreError::NotFound(*blob)));
1613
        };
1614
        let remote = self.open_named_remote(&remote_name)?;
1615
        let objects = remote.objects(&[*blob])?;
1616
        for (ty, body) in objects {
1617
            if matches!(ty, ObjectType::Blob | ObjectType::Chunk) {
1618
                self.store.put(ty, &body)?;
1619
            }
1620
        }
1621
        if !self.store.has(blob) {
1622
            return fail(format!(
1623
                "{remote_name} did not send blob {}",
1624
                blob.short(12)
1625
            ));
1626
        }
1627
        let _ = fs::remove_file(self.lazy_dir().join(blob.to_hex()));
1628
        Ok(())
1629
    }
1630
1631
    /// After importing from `remote`, record every blob the changes reference
1632
    /// that the store lacks as lazily available from that remote.
1633
    fn register_missing_blobs(&self, changes: &[ChangeHash], remote: &str) -> Result<usize> {
1634
        let mut n = 0;
1635
        for h in changes {
1636
            let c = self.store.get_change(h)?;
1637
            for e in &c.edits {
1638
                if let Edit::Replace { old, new, .. } = e {
1639
                    for b in [old, new].into_iter().flatten() {
1640
                        if !self.store.has(b) {
1641
                            self.register_lazy(b, remote)?;
1642
                            n += 1;
1643
                        }
1644
                    }
1645
                }
1646
            }
1647
        }
1648
        Ok(n)
1649
    }
1650
1651
    /// Every head this repository knows: bookmarks, remote tracking, working copy.
1652
    pub fn known_heads(&self, view: &View) -> Result<Vec<ChangeHash>> {
1653
        let mut have: BTreeSet<ChangeHash> = BTreeSet::new();
1654
        for st in view
1655
            .bookmarks
1656
            .values()
1657
            .chain(view.remotes.values().flat_map(|b| b.values()))
1658
        {
1659
            if let Ok(s) = self.store.get_state(st) {
1660
                have.extend(s.heads);
1661
            }
1662
        }
1663
        have.extend(view.workspaces.values().copied());
1664
        Ok(have.into_iter().collect())
1665
    }
1666
1667
    /// Fetch every bookmark of a remote into `remotes/<name>/bookmarks/*` (spec §10.2).
1668
    pub fn fetch(&self, remote_name: &str) -> Result<FetchReport> {
1669
        let remote = self.open_named_remote(remote_name)?;
1670
        self.check_remote_id(remote.as_ref(), remote_name, false)?;
1671
        let resp = remote.refs()?;
1672
        // The host's transparency log (spec §14.8): a fork fails here.
1673
        let log = self.check_log(
1674
            &self.remote_url(remote_name)?,
1675
            self.config.remote_cas.get(remote_name).map(Path::new),
1676
            &resp,
1677
        )?;
1678
        let mut log_notes = log.notes.clone();
1679
        // Trust policy (spec §14.5). Adopt the remote's policy when it is newer
1680
        // and signed by roots we trust, or on first contact. A remote that
1681
        // once presented a policy may never present none or an older one:
1682
        // that is a downgrade. A remote that never presented one is
1683
        // unverified, and this fetch is checked only as far as possible.
1684
        let local_policy = self.current_policy()?.map(|(_, sp)| sp.policy);
1685
        let seen = self.remote_policy_serial(remote_name);
1686
        let mut unverified = false;
1687
        match &resp.policy {
1688
            Some(sp) => {
1689
                if seen.is_some_and(|s| sp.policy.serial < s) {
1690
                    return fail(format!("{remote_name} presents policy serial {} but presented {} before: refusing the downgrade", sp.policy.serial, seen.unwrap()));
1691
                }
1692
                match &local_policy {
1693
                    Some(l) if sp.policy.serial < l.serial => {}
1694
                    Some(l) if sp.policy.serial == l.serial => {
1695
                        if sp.policy.hash() != l.hash() {
1696
                            return fail(format!(
1697
                                "{remote_name} presents a different policy with the same serial {}",
1698
                                l.serial
1699
                            ));
1700
                        }
1701
                    }
1702
                    _ => {
1703
                        self.adopt_policy_chain(
1704
                            &resp.history,
1705
                            sp,
1706
                            &format!("oxy fetch {remote_name}"),
1707
                        )
1708
                        .map_err(|e| RepoError::Msg(format!("policy from {remote_name}: {e}")))?;
1709
                    }
1710
                }
1711
                self.record_policy_serial(remote_name, sp.policy.serial)?;
1712
            }
1713
            None => {
1714
                if seen.is_some() {
1715
                    return fail(format!("{remote_name} presented a policy before but has none now: refusing the downgrade"));
1716
                }
1717
                unverified = true;
1718
            }
1719
        }
1720
        let policy = if unverified {
1721
            None
1722
        } else {
1723
            self.current_policy()?.map(|(_, sp)| sp.policy)
1724
        };
1725
        let (op_ref, op) = self.head_op()?;
1726
        let mut view = op.view.clone();
1727
        let have = self.known_heads(&view)?;
1728
        let mut report = Vec::new();
1729
        let mut skipped = Vec::new();
1730
        let max_blob = self.config.max_blob;
1731
        for r in resp.refs {
1732
            let needed = {
1733
                let _t = crate::timing::phase("fetch: negotiate");
1734
                remote.negotiate(&have, &r.heads)?
1735
            };
1736
            let bundle = {
1737
                let _t = crate::timing::phase("fetch: bundle");
1738
                remote.fetch(&needed, max_blob)?
1739
            };
1740
            // Changes are not individually re-verified here: the bookmark's
1741
            // attestation (checked below) binds its whole state, and the
1742
            // receiver of the push verified the added changes at that time.
1743
            // A shallow repository takes changes whose dependencies it never
1744
            // held (below what it cloned), as its clone did (#137).
1745
            let shallow = self.is_shallow();
1746
            let imported = {
1747
                let _t = crate::timing::phase("fetch: import");
1748
                if shallow {
1749
                    self.import_bundle_shallow(&bundle, false)?
1750
                } else {
1751
                    self.import_bundle(&bundle, false)?
1752
                }
1753
            };
1754
            if shallow {
1755
                self.refresh_boundary(&imported)?;
1756
            }
1757
            if max_blob.is_some() {
1758
                self.register_missing_blobs(&imported, remote_name)?;
1759
            }
1760
            let boundary = if shallow {
1761
                self.boundary()
1762
            } else {
1763
                BTreeSet::new()
1764
            };
1765
            for h in &r.heads {
1766
                if !self.store.has(h) && !boundary.contains(h) {
1767
                    return fail(format!("remote did not send head {}", h.short(12)));
1768
                }
1769
            }
1770
            let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
1771
            if state != r.state {
1772
                return fail(format!(
1773
                    "remote state {} does not match its heads",
1774
                    r.state.short(12)
1775
                ));
1776
            }
1777
            if let Some(p) = &policy {
1778
                let Some(sa) = &r.attestation else {
1779
                    return fail(format!(
1780
                        "{remote_name}/{} has no attestation, which its policy requires",
1781
                        r.name
1782
                    ));
1783
                };
1784
                let signer = sa
1785
                    .sigs
1786
                    .first()
1787
                    .map(|s| s.key)
1788
                    .ok_or_else(|| RepoError::Msg("attestation carries no signature".into()))?;
1789
                if !p.authorizes(&signer, &r.name) {
1790
                    self.authorize_in_time(p, remote_name, &r, sa, &signer, &log, &mut log_notes)?;
1791
                }
1792
            }
1793
            if let Some(sa) = &r.attestation {
1794
                let AttestBody::Ref {
1795
                    old_state,
1796
                    new_state,
1797
                    ..
1798
                } = &sa.attest.body
1799
                else {
1800
                    return fail("remote sent a non-ref attestation for a bookmark");
1801
                };
1802
                let (old_state, new_state) = (*old_state, *new_state);
1803
                let serial = self
1804
                    .check_ref_attest_body(
1805
                        sa,
1806
                        &r.name,
1807
                        &old_state,
1808
                        &new_state,
1809
                        self.last_seen_serial(remote_name, &r.name),
1810
                        true,
1811
                    )
1812
                    .map_err(|e| {
1813
                        RepoError::Msg(format!("attestation for {}/{}: {e}", remote_name, r.name))
1814
                    })?;
1815
                if new_state != r.state {
1816
                    return fail(format!(
1817
                        "attestation for {}/{} does not cover its current state",
1818
                        remote_name, r.name
1819
                    ));
1820
                }
1821
                // A signature this client cannot verify (its signer unknown
1822
                // here, as from a host that did not send the identity, or
1823
                // bad) fails that bookmark alone: it stays as it was and
1824
                // the rest are fetched (oxygenhub#266). The current
1825
                // bookmark, which a pull would take, still fails the fetch.
1826
                // A policy's requirements were checked above and still fail.
1827
                if let Err(e) = self.verify_attest_signature(sa) {
1828
                    let why = format!("attestation for {remote_name}/{}: {e}", r.name);
1829
                    if r.name == self.config.bookmark {
1830
                        return fail(why);
1831
                    }
1832
                    skipped.push((r.name, why));
1833
                    continue;
1834
                }
1835
                let ar = self.store.put_attest(&sa.attest)?;
1836
                for sig in &sa.sigs {
1837
                    self.store.put_signature(&ar, sig)?;
1838
                }
1839
                self.record_serial(remote_name, &r.name, serial)?;
1840
            }
1841
            view.remotes
1842
                .entry(remote_name.to_string())
1843
                .or_default()
1844
                .insert(r.name.clone(), state);
1845
            report.push((r.name, r.heads, imported.len()));
1846
        }
1847
        let mut new_tags = 0;
1848
        for t in &resp.tags {
1849
            if view.tags.contains_key(&t.name) {
1850
                continue;
1851
            }
1852
            // The tag's changes must be here: fetch them if the bookmarks did not bring them.
1853
            let missing: Vec<ChangeHash> = t
1854
                .heads
1855
                .iter()
1856
                .filter(|h| !self.store.has(h))
1857
                .copied()
1858
                .collect();
1859
            if !missing.is_empty() {
1860
                let needed = remote.negotiate(&self.known_heads(&view)?, &missing)?;
1861
                let bundle = remote.fetch(&needed, max_blob)?;
1862
                self.import_bundle(&bundle, false)?;
1863
            }
1864
            self.store.put_state(&StateObject::new(t.heads.clone()))?;
1865
            let mut check = TagCheck::Fetched {
1866
                remote: remote_name,
1867
                log: &log,
1868
                notes: &mut log_notes,
1869
            };
1870
            match self.adopt_tag(&mut view, t, policy.as_ref(), &mut check) {
1871
                Ok(()) => new_tags += 1,
1872
                // Nothing says when a short-lived key signed it: the tag is
1873
                // left out, said so, and the rest is fetched (oxygen#129).
1874
                Err(e) if e.untimed => {
1875
                    skipped.push((format!("tags/{}", t.name), e.reason));
1876
                }
1877
                Err(e) => return fail(format!("{remote_name} {}: {}", t.name, e.reason)),
1878
            }
1879
        }
1880
        if view != op.view {
1881
            self.record_op(&[op_ref], &format!("oxy fetch {remote_name}"), view)?;
1882
        }
1883
        let _ = self.run_hook(
1884
            "post-fetch",
1885
            &format!(
1886
                "{{\"event\":\"post-fetch\",\"remote\":{},\"bookmarks\":{}}}",
1887
                json_str(remote_name),
1888
                json_array(report.iter().map(|(n, heads, _)| format!(
1889
                    "{{\"name\":{},\"heads\":{}}}",
1890
                    json_str(n),
1891
                    json_array(heads.iter().map(|h| json_str(&h.to_hex())))
1892
                )))
1893
            ),
1894
        );
1895
        Ok(FetchReport {
1896
            refs: report,
1897
            unverified,
1898
            new_tags,
1899
            log_notes,
1900
            skipped,
1901
        })
1902
    }
1903
1904
    /// Shallow fetch of the remote's bookmark: its checkpoint as our base,
1905
    /// the changes within `depth` steps of its heads, and the boundary below.
1906
    pub fn fetch_shallow(&self, remote_name: &str, depth: u32) -> Result<usize> {
1907
        let remote = self.open_named_remote(remote_name)?;
1908
        self.check_remote_id(remote.as_ref(), remote_name, false)?;
1909
        let resp = remote.refs()?;
1910
        let bookmark = self.config.bookmark.clone();
1911
        if let Some(sp) = &resp.policy {
1912
            self.adopt_policy_chain(&resp.history, sp, "oxy clone --depth")
1913
                .map_err(|e| RepoError::Msg(format!("policy from {remote_name}: {e}")))?;
1914
            self.record_policy_serial(remote_name, sp.policy.serial)?;
1915
        }
1916
        let Some(r) = resp.refs.into_iter().find(|r| r.name == bookmark) else {
1917
            return fail(format!("{remote_name} has no bookmark {bookmark:?}"));
1918
        };
1919
        if r.heads.is_empty() {
1920
            return Ok(0);
1921
        }
1922
        // Base: the remote's checkpoint of exactly these heads, without its
1923
        // graph blobs, which come in parts below (#137).
1924
        let cp_bundle = {
1925
            let _t = crate::timing::phase("shallow: checkpoint request");
1926
            remote.checkpoint_index(&r.heads)?
1927
        };
1928
        // In one batch: a big state's checkpoint is tens of thousands of
1929
        // graph blobs and chunks, which stored one by one were each written
1930
        // loose and flushed (a VS Code clone ran at 27 objects a second).
1931
        let objects: Vec<(ObjectType, &[u8])> = cp_bundle
1932
            .objects
1933
            .iter()
1934
            .map(|(ty, body)| (*ty, body.as_slice()))
1935
            .collect();
1936
        let t_store = crate::timing::phase("shallow: store the checkpoint");
1937
        let batch = self.store.begin_batch()?;
1938
        let stored = self.store.put_many(&objects);
1939
        batch.finish()?;
1940
        drop(t_store);
1941
        let cp_ref = objects
1942
            .iter()
1943
            .zip(stored?)
1944
            .find(|((ty, _), _)| *ty == ObjectType::Checkpoint)
1945
            .map(|(_, r)| r);
1946
        let Some(cp_ref) = cp_ref else {
1947
            return fail(format!("{remote_name} sent no checkpoint"));
1948
        };
1949
        // The graph blobs the payload names, with their chunks, a part at
1950
        // a time (#137): one answer for a big state was gigabytes, held
1951
        // whole on both sides. A remote that sent the whole checkpoint has
1952
        // left none to ask for.
1953
        let cp = crate::objects::Checkpoint::decode(
1954
            &self.store.get_typed(&cp_ref, ObjectType::Checkpoint)?,
1955
        )?;
1956
        let graphs: Vec<ObjectRef> = self
1957
            .checkpoint_graph_blobs(&cp)?
1958
            .into_iter()
1959
            .filter(|b| !self.store.has(b))
1960
            .collect();
1961
        {
1962
            let _t = crate::timing::phase("shallow: the checkpoint's graphs, in parts");
1963
            for part in graphs.chunks(CHECKPOINT_PART) {
1964
                let objects = remote.objects(part)?;
1965
                let objects: Vec<(ObjectType, &[u8])> = objects
1966
                    .iter()
1967
                    .filter(|(ty, _)| matches!(ty, ObjectType::Blob | ObjectType::Chunk))
1968
                    .map(|(ty, body)| (*ty, body.as_slice()))
1969
                    .collect();
1970
                let batch = self.store.begin_batch()?;
1971
                let stored = self.store.put_many(&objects);
1972
                batch.finish()?;
1973
                stored?;
1974
            }
1975
            if let Some(b) = graphs.iter().find(|b| !self.store.has(b)) {
1976
                return fail(format!(
1977
                    "{remote_name} did not send graph blob {}",
1978
                    b.short(12)
1979
                ));
1980
            }
1981
        }
1982
        self.install_checkpoint(&bookmark, &cp_ref)?;
1983
        let _t_blobs = crate::timing::phase("shallow: the binary files' blobs");
1984
        // The blobs of the state's opaque files come with no change when the
1985
        // window holds none of the changes that set them (#137): fetch them
1986
        // now, or under `filter.max-blob` leave them to be fetched on read.
1987
        let base = State::from_checkpoint_history(&self.read_blob(&cp.payload)?)?;
1988
        let mut missing: Vec<ObjectRef> = base
1989
            .opaque_nodes()
1990
            .iter()
1991
            .flat_map(|n| base.live_blobs(n))
1992
            .filter(|b| !self.store.has(b))
1993
            .collect();
1994
        missing.sort();
1995
        missing.dedup();
1996
        if self.config.max_blob.is_some() {
1997
            for b in &missing {
1998
                self.register_lazy(b, remote_name)?;
1999
            }
2000
        } else {
2001
            for part in missing.chunks(512) {
2002
                let objects = remote.objects(part)?;
2003
                let objects: Vec<(ObjectType, &[u8])> = objects
2004
                    .iter()
2005
                    .filter(|(ty, _)| matches!(ty, ObjectType::Blob | ObjectType::Chunk))
2006
                    .map(|(ty, body)| (*ty, body.as_slice()))
2007
                    .collect();
2008
                let batch = self.store.begin_batch()?;
2009
                let stored = self.store.put_many(&objects);
2010
                batch.finish()?;
2011
                stored?;
2012
            }
2013
            if let Some(b) = missing.iter().find(|b| !self.store.has(b)) {
2014
                return fail(format!("{remote_name} did not send blob {}", b.short(12)));
2015
            }
2016
        }
2017
        // History window. Depth 0 takes none (#137): the checkpoint holds
2018
        // every change's metadata, and its heads become the boundary, so a
2019
        // state of many heads (33,835 in the VS Code import) costs nothing
2020
        // to fetch.
2021
        let imported = if depth == 0 {
2022
            let mut boundary = self.boundary();
2023
            boundary.extend(r.heads.iter().filter(|h| !self.store.has(h)).copied());
2024
            self.write_boundary(&boundary)?;
2025
            Vec::new()
2026
        } else {
2027
            let wanted = remote.shallow(&r.heads, depth)?;
2028
            let bundle = remote.fetch(&wanted, self.config.max_blob)?;
2029
            self.import_bundle_shallow(&bundle, false)?
2030
        };
2031
        if self.config.max_blob.is_some() {
2032
            self.register_missing_blobs(&imported, remote_name)?;
2033
        }
2034
        self.refresh_boundary(&imported)?;
2035
        if let Some(sa) = &r.attestation {
2036
            let ar = self.store.put_attest(&sa.attest)?;
2037
            for sig in &sa.sigs {
2038
                self.store.put_signature(&ar, sig)?;
2039
            }
2040
            if let AttestBody::Ref { serial, .. } = sa.attest.body {
2041
                self.record_serial(remote_name, &r.name, serial)?;
2042
            }
2043
        }
2044
        let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
2045
        let (op_ref, op) = self.head_op()?;
2046
        let mut view = op.view.clone();
2047
        view.remotes
2048
            .entry(remote_name.to_string())
2049
            .or_default()
2050
            .insert(r.name.clone(), state);
2051
        self.record_op(
2052
            &[op_ref],
2053
            &format!("oxy fetch {remote_name} --depth {depth}"),
2054
            view,
2055
        )?;
2056
        Ok(imported.len())
2057
    }
2058
2059
    /// Fetch, then take the union of the local bookmark's state with the
2060
    /// remote's: no merge change, conflicts become reified (spec §9.4).
2061
    pub fn pull(&self, remote_name: &str) -> Result<PullReport> {
2062
        let _wc = self.lock_working_copy()?;
2063
        let skipped = self.fetch(remote_name)?.skipped;
2064
        let (op_ref, op) = self.head_op()?;
2065
        let current = self.config.bookmark.clone();
2066
        let mut local_current = self.heads_of(&op.view, &current)?;
2067
        local_current.sort();
2068
        let mut view = op.view.clone();
2069
        let mut current_heads = local_current.clone();
2070
        let mut changed = false;
2071
        let mut others = Vec::new();
2072
        let tracked = op
2073
            .view
2074
            .remotes
2075
            .get(remote_name)
2076
            .cloned()
2077
            .unwrap_or_default();
2078
        for (name, remote_state) in tracked {
2079
            // Every local bookmark tracks the remote's of the same name;
2080
            // the current one may be new here.
2081
            let local_state = op.view.bookmarks.get(&name).copied();
2082
            if local_state.is_none() && name != current {
2083
                continue;
2084
            }
2085
            if local_state == Some(remote_state) {
2086
                continue;
2087
            }
2088
            let local = self.heads_of(&op.view, &name)?;
2089
            let remote_heads = self.store.get_state(&remote_state)?.heads;
2090
            if remote_heads.iter().all(|h| local.contains(h)) {
2091
                continue;
2092
            }
2093
            let mut all = local.clone();
2094
            all.extend(remote_heads);
2095
            let st = self.load_state_history(&all)?;
2096
            let mut heads = st.maximal(&all);
2097
            heads.sort();
2098
            let mut sorted_local = local;
2099
            sorted_local.sort();
2100
            if heads == sorted_local {
2101
                continue;
2102
            }
2103
            view.bookmarks.insert(
2104
                name.clone(),
2105
                self.store.put_state(&StateObject::new(heads.clone()))?,
2106
            );
2107
            if name == current {
2108
                changed = true;
2109
                current_heads = heads;
2110
            } else {
2111
                others.push(name);
2112
            }
2113
        }
2114
        if changed || !others.is_empty() {
2115
            self.record_op(&[op_ref], &format!("oxy pull {remote_name}"), view)?;
2116
        }
2117
        // The working copy follows its bookmark whenever nothing would be
2118
        // lost, also when this pull brought nothing new but it had been
2119
        // left behind (oxygen#89).
2120
        let checked_out = self.follow_bookmark(&current_heads, &current, "oxy pull")?;
2121
        Ok(PullReport {
2122
            heads: if changed {
2123
                current_heads
2124
            } else {
2125
                local_current
2126
            },
2127
            changed,
2128
            checked_out,
2129
            others,
2130
            skipped,
2131
        })
2132
    }
2133
2134
    /// How many changes the working copy's bookmark has that its base
2135
    /// lacks, when the base is part of the bookmark's state (it fell behind,
2136
    /// and `oxy checkout` or `oxy pull` brings it up); `None` when it is on
2137
    /// the bookmark or holds changes the bookmark lacks. History is read
2138
    /// only when the base and the bookmark's heads differ (oxygen#89).
2139
    pub fn working_copy_behind(&self) -> Result<Option<usize>> {
2140
        let (_, wc) = self.working_copy()?;
2141
        let mut base = wc.deps.clone();
2142
        base.sort();
2143
        let mut heads = self.bookmark_heads()?;
2144
        heads.sort();
2145
        if base == heads || heads.is_empty() {
2146
            return Ok(None);
2147
        }
2148
        let state = self.load_state_history(&heads)?;
2149
        if !base.iter().all(|h| state.contains(h)) {
2150
            return Ok(None);
2151
        }
2152
        let had = if base.is_empty() {
2153
            0
2154
        } else {
2155
            self.load_state_history(&base)?.len()
2156
        };
2157
        Ok(Some(state.len().saturating_sub(had)).filter(|n| *n > 0))
2158
    }
2159
2160
    /// Bring the working copy to `heads`, the state of its bookmark
2161
    /// `bookmark`, when nothing would be lost: its base is part of that
2162
    /// state, so the bookmark only adds to it. Clean, it is checked out;
2163
    /// with uncommitted work, the work comes along when the new changes
2164
    /// leave its paths alone (as `git pull` does), else it stays. Before,
2165
    /// it followed only from exactly the bookmark's old heads, so a working
2166
    /// copy already behind (the bookmark moved by `bookmark set`, a landing
2167
    /// or a pull from another workspace) never caught up, and nothing said
2168
    /// so (oxygen#74, #89). Returns whether it moved.
2169
    pub fn follow_bookmark(
2170
        &self,
2171
        heads: &[ChangeHash],
2172
        bookmark: &str,
2173
        command: &str,
2174
    ) -> Result<bool> {
2175
        let snap = self.snapshot(command)?;
2176
        let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
2177
        let mut base = snap.base.maximal(&snap.wc.deps);
2178
        base.sort();
2179
        let mut target = heads.to_vec();
2180
        target.sort();
2181
        if base == target {
2182
            return Ok(false);
2183
        }
2184
        let state = self.load_state_history(heads)?;
2185
        if !base.iter().all(|h| state.contains(h)) {
2186
            // The working copy holds changes the bookmark lacks: not ours to move.
2187
            return Ok(false);
2188
        }
2189
        if clean {
2190
            self.checkout(heads, false, &format!("{command} (checkout)"))?;
2191
            Ok(true)
2192
        } else {
2193
            Ok(self
2194
                .checkout_carry(heads, bookmark, &format!("{command} (checkout --carry)"))
2195
                .is_ok())
2196
        }
2197
    }
2198
2199
    /// Push the current bookmark (spec §16.3), sealing this author's unsigned
2200
    /// changes and attesting the reference when a key is available.
2201
    /// Delete `bookmark` on a remote (spec §16.4): an update to the zero
2202
    /// state, attested when we have an identity. The local bookmark, if
2203
    /// any, is kept; only the remote-tracking entry goes.
2204
    pub fn push_delete(&self, remote_name: &str, bookmark: &str) -> Result<Vec<RefResult>> {
2205
        let remote = self.open_named_remote(remote_name)?;
2206
        self.check_remote_id(remote.as_ref(), remote_name, true)?;
2207
        let resp = remote.refs()?;
2208
        let Some(r) = resp.refs.iter().find(|r| r.name == bookmark) else {
2209
            return fail(format!("{remote_name} has no bookmark {bookmark}"));
2210
        };
2211
        if resp.policy.is_some() && self.secret_key()?.is_none() {
2212
            return fail(format!("{remote_name} enforces a trust policy; deleting needs an identity (`oxy identity create`)"));
2213
        }
2214
        let attestation = match self.secret_key()? {
2215
            Some(_) => Some(self.sign_attest(&Attest {
2216
                repo: self.id()?,
2217
                hlc: Hlc::now(),
2218
                body: AttestBody::Ref {
2219
                    name: bookmark.to_string(),
2220
                    old_state: r.state,
2221
                    new_state: ObjectRef::ZERO,
2222
                    serial: r.last_serial.unwrap_or(0) + 1,
2223
                },
2224
            })?),
2225
            None => None,
2226
        };
2227
        let results = remote.push(
2228
            Bundle::default(),
2229
            vec![RefUpdate {
2230
                name: bookmark.to_string(),
2231
                old_state: r.state,
2232
                new_state: ObjectRef::ZERO,
2233
                force: true,
2234
                attestation,
2235
            }],
2236
            None,
2237
            vec![],
2238
        )?;
2239
        if results.iter().any(|x| x.ok && x.name == bookmark) {
2240
            let (op_ref, op) = self.head_op()?;
2241
            let mut view = op.view.clone();
2242
            if let Some(m) = view.remotes.get_mut(remote_name) {
2243
                m.remove(bookmark);
2244
            }
2245
            self.record_op(
2246
                &[op_ref],
2247
                &format!("oxy push {remote_name} --delete {bookmark}"),
2248
                view,
2249
            )?;
2250
        }
2251
        Ok(results)
2252
    }
2253
2254
    /// Remove remote `name` (oxygen#86): every `remote.NAME.*` key of this
2255
    /// repository's config (its URL, token, CA and store settings), its
2256
    /// tracking bookmarks in the view, and a store remote's mirror under
2257
    /// `.oxy/remotes/NAME`. Nothing stays to be read as if it were current.
2258
    pub fn remove_remote(&mut self, name: &str) -> Result<()> {
2259
        if !self.config.remotes.contains_key(name) {
2260
            return fail(format!("no remote {name}: `oxy remote list` shows them"));
2261
        }
2262
        let prefix = format!("remote.{name}.");
2263
        let keys: Vec<String> = self
2264
            .config
2265
            .repo_layer()
2266
            .keys()
2267
            .filter(|k| k.starts_with(&prefix))
2268
            .cloned()
2269
            .collect();
2270
        for k in keys {
2271
            self.config.set(&k, "")?;
2272
        }
2273
        self.config.remotes.remove(name);
2274
        self.save_config()?;
2275
        let (op_ref, op) = self.head_op()?;
2276
        if op.view.remotes.contains_key(name) {
2277
            let mut view = op.view.clone();
2278
            view.remotes.remove(name);
2279
            self.record_op(&[op_ref], &format!("oxy remote remove {name}"), view)?;
2280
        }
2281
        let mirror = self.oxy.join("remotes").join(name);
2282
        if mirror.is_dir() {
2283
            fs::remove_dir_all(&mirror)?;
2284
        }
2285
        Ok(())
2286
    }
2287
2288
    /// Send `bundle`'s blobs and chunks ahead of it, in parts of about
2289
    /// `push.batch-bytes` of objects alone, when the whole would be larger
2290
    /// (oxygen#133): one change can bring more blobs than one request may
2291
    /// carry, and a proxy in front of a hub refuses a request past its limit
2292
    /// (Cloudflare's is 100 MB). What is left, the changes with their
2293
    /// identities and states, is returned to send as before. A receiver
2294
    /// stores objects as they come, and checks a change's blobs are stored.
2295
    fn send_objects_ahead(
2296
        &self,
2297
        remote: &dyn crate::protocol::Remote,
2298
        remote_name: &str,
2299
        mut bundle: Bundle,
2300
        progress: crate::store::Progress,
2301
    ) -> Result<Bundle> {
2302
        let limit = self.config.push_batch_bytes;
2303
        let changes: u64 = bundle
2304
            .changes
2305
            .iter()
2306
            .map(|(c, _)| c.encode().len() as u64)
2307
            .sum();
2308
        let objects: u64 = bundle.objects.iter().map(|(_, b)| b.len() as u64).sum();
2309
        if changes + objects <= limit {
2310
            return Ok(bundle);
2311
        }
2312
        let (mut kept, mut parts, mut part, mut bytes) = (Vec::new(), Vec::new(), Vec::new(), 0u64);
2313
        for (ty, body) in std::mem::take(&mut bundle.objects) {
2314
            if !matches!(ty, ObjectType::Blob | ObjectType::Chunk) {
2315
                kept.push((ty, body));
2316
                continue;
2317
            }
2318
            if !part.is_empty() && bytes + body.len() as u64 > limit {
2319
                parts.push(std::mem::take(&mut part));
2320
                bytes = 0;
2321
            }
2322
            bytes += body.len() as u64;
2323
            part.push((ty, body));
2324
        }
2325
        if !part.is_empty() {
2326
            parts.push(part);
2327
        }
2328
        let n = parts.len();
2329
        for (i, objects) in parts.into_iter().enumerate() {
2330
            progress(&format!(
2331
                "objects {} of {n}: {} blobs and chunks",
2332
                i + 1,
2333
                objects.len()
2334
            ));
2335
            let results = remote.push(
2336
                Bundle {
2337
                    objects,
2338
                    ..Default::default()
2339
                },
2340
                vec![],
2341
                None,
2342
                vec![],
2343
            )?;
2344
            match results.iter().find(|r| r.name == OBJECTS_RESULT) {
2345
                Some(r) if r.ok => {}
2346
                Some(r) => {
2347
                    return fail(format!(
2348
                        "{remote_name} refused objects part {} of {n}: {}",
2349
                        i + 1,
2350
                        r.reason
2351
                    ))
2352
                }
2353
                None => {
2354
                    return fail(format!(
2355
                        "{remote_name} does not take a push in parts, and this one is too large for one: update the server's oxy"
2356
                    ))
2357
                }
2358
            }
2359
        }
2360
        bundle.objects = kept;
2361
        Ok(bundle)
2362
    }
2363
2364
    /// `changes`, in dependency order, cut into batches of about
2365
    /// `push.batch-bytes` by what each change and its texts and blobs weigh.
2366
    /// There is always one, perhaps empty.
2367
    fn push_batches(&self, changes: &[ChangeHash]) -> Result<Vec<Vec<ChangeHash>>> {
2368
        let blob_len = |r: &ObjectRef| -> u64 {
2369
            self.store
2370
                .get_typed(r, ObjectType::Blob)
2371
                .ok()
2372
                .and_then(|b| Blob::decode(&b).ok())
2373
                .map_or(0, |b| b.length)
2374
        };
2375
        let (mut out, mut batch, mut bytes) = (Vec::new(), Vec::new(), 0u64);
2376
        for h in changes {
2377
            let raw = self.store.get_typed(h, ObjectType::Change)?;
2378
            let (c, texts) = Change::decode_skeleton(&raw, *h)?;
2379
            let mut weight = raw.len() as u64;
2380
            for t in &texts {
2381
                if let crate::objects::TextRef::Blob(r, _, _) = t {
2382
                    weight += blob_len(r);
2383
                }
2384
            }
2385
            for e in &c.edits {
2386
                if let Edit::Replace { new: Some(b), .. } = e {
2387
                    weight += blob_len(b);
2388
                }
2389
            }
2390
            if !batch.is_empty() && bytes + weight > self.config.push_batch_bytes {
2391
                out.push(std::mem::take(&mut batch));
2392
                bytes = 0;
2393
            }
2394
            batch.push(*h);
2395
            bytes += weight;
2396
        }
2397
        out.push(batch);
2398
        Ok(out)
2399
    }
2400
2401
    pub fn push(&self, remote_name: &str, force: bool) -> Result<Vec<RefResult>> {
2402
        let bookmark = self.config.bookmark.clone();
2403
        self.push_bookmark(remote_name, &bookmark, force)
2404
    }
2405
2406
    /// `push` for a named bookmark (`oxy push -b NAME`).
2407
    pub fn push_bookmark(
2408
        &self,
2409
        remote_name: &str,
2410
        bookmark: &str,
2411
        force: bool,
2412
    ) -> Result<Vec<RefResult>> {
2413
        self.push_bookmark_with(remote_name, bookmark, force, &mut |_| {})
2414
    }
2415
2416
    /// `push_bookmark`, saying how a long push is going: one line per
2417
    /// batch sent (oxygen#95).
2418
    /// What a push may take the remote to hold already: the heads of the
2419
    /// bookmark pushed, and of every other bookmark it lists that are held
2420
    /// here, so a new bookmark made from main sends only what main lacks
2421
    /// (oxygen#142), not its whole history again.
2422
    pub(crate) fn remote_holds(
2423
        &self,
2424
        refs: &[crate::protocol::RemoteRef],
2425
        bookmark_heads: &[ChangeHash],
2426
    ) -> Vec<ChangeHash> {
2427
        let mut have: Vec<ChangeHash> = bookmark_heads.to_vec();
2428
        for h in refs.iter().flat_map(|r| &r.heads) {
2429
            if !have.contains(h) && self.store.has(h) {
2430
                have.push(*h);
2431
            }
2432
        }
2433
        have
2434
    }
2435
2436
    pub fn push_bookmark_with(
2437
        &self,
2438
        remote_name: &str,
2439
        bookmark: &str,
2440
        force: bool,
2441
        progress: crate::store::Progress,
2442
    ) -> Result<Vec<RefResult>> {
2443
        let (op_ref, op) = self.head_op()?;
2444
        if !op.view.bookmarks.contains_key(bookmark) {
2445
            return fail(format!("no bookmark {bookmark}"));
2446
        }
2447
        let bookmark = bookmark.to_string();
2448
        let local_heads = self.heads_of(&op.view, &bookmark)?;
2449
        // pre-push hook: may veto before the remote is contacted.
2450
        self.run_hook(
2451
            "pre-push",
2452
            &format!(
2453
                "{{\"event\":\"pre-push\",\"remote\":{},\"bookmark\":{},\"heads\":{}}}",
2454
                json_str(remote_name),
2455
                json_str(&bookmark),
2456
                json_array(local_heads.iter().map(|h| json_str(&h.to_hex())))
2457
            ),
2458
        )?;
2459
        let remote = self.open_named_remote(remote_name)?;
2460
        self.check_remote_id(remote.as_ref(), remote_name, true)?;
2461
        let resp = remote.refs()?;
2462
        let local_policy = self.current_policy()?.map(|(_, sp)| sp);
2463
        let resp_policy_serial = resp.policy.as_ref().map(|p| p.policy.serial);
2464
        let policy_to_send = match (&local_policy, &resp.policy) {
2465
            (Some(l), Some(r)) if l.policy.serial > r.policy.serial => Some(l.clone()),
2466
            (Some(l), None) => Some(l.clone()),
2467
            (Some(l), Some(r)) if l.policy.serial < r.policy.serial => {
2468
                return fail(format!(
2469
                    "{remote_name} has a newer policy (serial {}); fetch first",
2470
                    r.policy.serial
2471
                ));
2472
            }
2473
            _ => None,
2474
        };
2475
        let remote_has_policy = resp.policy.is_some() || policy_to_send.is_some();
2476
        if remote_has_policy && self.secret_key()?.is_none() {
2477
            return fail(format!("{remote_name} enforces a trust policy; pushing needs an identity (`oxy identity create`)"));
2478
        }
2479
        let remote_tags: BTreeSet<String> = resp.tags.iter().map(|t| t.name.clone()).collect();
2480
        let tags_to_send: Vec<crate::protocol::TagRef> = self
2481
            .tag_refs()?
2482
            .into_iter()
2483
            .filter(|t| !remote_tags.contains(&t.name))
2484
            .collect();
2485
        let refs = resp.refs;
2486
        let empty = self.store.put_state(&StateObject::new(vec![]))?;
2487
        let (remote_state, remote_heads, last_serial) =
2488
            match refs.iter().find(|r| r.name == bookmark) {
2489
                Some(r) => (r.state, r.heads.clone(), r.last_serial),
2490
                None => (empty, vec![], None),
2491
            };
2492
        let new_state = self
2493
            .store
2494
            .put_state(&StateObject::new(local_heads.clone()))?;
2495
        // A push that carries a policy re-attests the bookmark even when its
2496
        // state is unchanged, so the current attestation is always by a key
2497
        // the current policy authorizes (a revoked key's attestation would
2498
        // otherwise be refused by every later fetch).
2499
        if new_state == remote_state && policy_to_send.is_none() && tags_to_send.is_empty() {
2500
            return Ok(vec![RefResult {
2501
                name: bookmark,
2502
                ok: true,
2503
                reason: "already up to date".into(),
2504
            }]);
2505
        }
2506
        // A shallow repository knows the changes on its boundary without
2507
        // holding them: a depth-0 clone's are the heads it cloned (#137).
2508
        let boundary = self.boundary();
2509
        for h in &remote_heads {
2510
            if !self.store.has(h) && !boundary.contains(h) {
2511
                return fail(format!(
2512
                    "the remote has change {} which you do not; pull first (`oxy pull {remote_name}`), then push again",
2513
                    h.short(12)
2514
                ));
2515
            }
2516
        }
2517
        if !force {
2518
            let closure = self.closure_in_store(&local_heads)?;
2519
            if let Some(h) = remote_heads
2520
                .iter()
2521
                .find(|h| !closure.contains(h) && !boundary.contains(h))
2522
            {
2523
                return fail(format!("not a fast-forward: remote head {} is not in your state; pull first or use --force", h.short(12)));
2524
            }
2525
        }
2526
        let to_send = self.negotiate(&self.remote_holds(&refs, &remote_heads), &local_heads)?;
2527
        let key = self.secret_key()?.map(|(k, _)| k);
2528
        if let Some(key) = key {
2529
            // Seal this author's changes that carry no valid authenticator,
2530
            // and adopt anonymous ones (zero author, spec §14.4). The seals
2531
            // are stored, so each batch's bundle carries those it needs.
2532
            let mut unsigned = Vec::new();
2533
            for h in &to_send {
2534
                let c = self.store.get_change_meta(h)?;
2535
                if self.seals_for_author(&c.author, &key)
2536
                    && !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. })
2537
                {
2538
                    unsigned.push(*h);
2539
                }
2540
            }
2541
            for chunk in unsigned.chunks(crate::attest::MAX_SEAL_HASHES) {
2542
                self.sign_attest(&Attest::seal(self.id()?, chunk.to_vec()))?;
2543
            }
2544
        }
2545
        // A long history goes in batches, in dependency order, so neither
2546
        // side holds it whole and no request passes the remote's limit:
2547
        // all but the last carry objects alone; the last moves the bookmark
2548
        // (oxygen#95).
2549
        let batches = self.push_batches(&to_send)?;
2550
        let (last, earlier) = batches.split_last().expect("at least one batch");
2551
        for (i, batch) in earlier.iter().enumerate() {
2552
            progress(&format!(
2553
                "part {} of {}: {} changes",
2554
                i + 1,
2555
                batches.len(),
2556
                batch.len()
2557
            ));
2558
            let bundle = self.send_objects_ahead(
2559
                remote.as_ref(),
2560
                remote_name,
2561
                self.bundle_for(batch)?,
2562
                &mut *progress,
2563
            )?;
2564
            let results = remote.push(bundle, vec![], None, vec![])?;
2565
            match results.iter().find(|r| r.name == OBJECTS_RESULT) {
2566
                Some(r) if r.ok => {}
2567
                Some(r) => {
2568
                    return fail(format!(
2569
                        "{remote_name} refused part {} of the push: {}",
2570
                        i + 1,
2571
                        r.reason
2572
                    ))
2573
                }
2574
                None => {
2575
                    return fail(format!(
2576
                        "{remote_name} does not take a push in parts, and this one is too long for one: update the server's oxy"
2577
                    ))
2578
                }
2579
            }
2580
        }
2581
        if batches.len() > 1 {
2582
            progress(&format!(
2583
                "part {} of {}: {} changes",
2584
                batches.len(),
2585
                batches.len(),
2586
                last.len()
2587
            ));
2588
        }
2589
        let mut bundle = self.send_objects_ahead(
2590
            remote.as_ref(),
2591
            remote_name,
2592
            self.bundle_for(last)?,
2593
            &mut *progress,
2594
        )?;
2595
        bundle.objects.push((
2596
            ObjectType::State,
2597
            StateObject::new(local_heads.clone()).encode(),
2598
        ));
2599
        let attestation = match key {
2600
            Some(_) => Some(self.sign_attest(&Attest {
2601
                repo: self.id()?,
2602
                hlc: Hlc::now(),
2603
                body: AttestBody::Ref {
2604
                    name: bookmark.clone(),
2605
                    old_state: remote_state,
2606
                    new_state,
2607
                    serial: next_serial(last_serial),
2608
                },
2609
            })?),
2610
            None => None,
2611
        };
2612
        // Whoever signed the move, and whom a delegate acts for, travel with
2613
        // it: a receiver that has never seen a change by this delegate could
2614
        // not otherwise check its signature (oxygenhub#129).
2615
        if attestation.is_some() {
2616
            for k in [self.signing_key(), self.config.author_key]
2617
                .into_iter()
2618
                .flatten()
2619
            {
2620
                if let Ok(id) = self.identity(&k) {
2621
                    let body = id.encode();
2622
                    if !bundle
2623
                        .objects
2624
                        .iter()
2625
                        .any(|(t, b)| *t == ObjectType::Identity && *b == body)
2626
                    {
2627
                        bundle.objects.push((ObjectType::Identity, body));
2628
                    }
2629
                }
2630
            }
2631
            // A short-lived signing key's certificate goes with it (§14.7).
2632
            let signers: Vec<KeyId> = self.signing_key().into_iter().collect();
2633
            self.attach_certificates_for(&mut bundle, &signers)?;
2634
        }
2635
        let serial = attestation.as_ref().and_then(|a| match a.attest.body {
2636
            AttestBody::Ref { serial, .. } => Some(serial),
2637
            _ => None,
2638
        });
2639
        let sent_serial = policy_to_send.as_ref().map(|p| p.policy.serial);
2640
        // Tag states must be present on the remote: bundle their changes too.
2641
        for t in &tags_to_send {
2642
            let extra = self.negotiate(&remote_heads, &t.heads)?;
2643
            let extra: Vec<ChangeHash> =
2644
                extra.into_iter().filter(|h| !to_send.contains(h)).collect();
2645
            if !extra.is_empty() {
2646
                let more = self.bundle_for(&extra)?;
2647
                bundle.changes.extend(more.changes);
2648
                bundle.hashes.extend(more.hashes);
2649
                bundle.attests.extend(more.attests);
2650
                bundle.objects.extend(more.objects);
2651
            }
2652
            bundle.objects.push((
2653
                ObjectType::State,
2654
                StateObject::new(t.heads.clone()).encode(),
2655
            ));
2656
        }
2657
        // A tag's signer travels with it, as the bookmark's does: a tag made
2658
        // by `oxy tag create` under a short-lived key was signed by a key the
2659
        // receiver has never seen, so its identity and certificate go too.
2660
        let tag_signers: Vec<KeyId> = tags_to_send
2661
            .iter()
2662
            .filter_map(|t| t.attestation.as_ref())
2663
            .flat_map(|a| a.sigs.iter().map(|s| s.key))
2664
            .collect();
2665
        for k in &tag_signers {
2666
            if let Ok(id) = self.identity(k) {
2667
                let body = id.encode();
2668
                if !bundle
2669
                    .objects
2670
                    .iter()
2671
                    .any(|(t, b)| *t == ObjectType::Identity && *b == body)
2672
                {
2673
                    bundle.objects.push((ObjectType::Identity, body));
2674
                }
2675
            }
2676
        }
2677
        if !tag_signers.is_empty() {
2678
            self.attach_certificates_for(&mut bundle, &tag_signers)?;
2679
        }
2680
        if let Some(l) = &policy_to_send {
2681
            let remote_serial = resp_policy_serial;
2682
            bundle.policies = self
2683
                .policy_history()?
2684
                .into_iter()
2685
                .filter(|sp| {
2686
                    remote_serial.is_none_or(|r| sp.policy.serial > r)
2687
                        && sp.policy.serial < l.policy.serial
2688
                })
2689
                .collect();
2690
        }
2691
        let results = remote.push(
2692
            bundle,
2693
            vec![RefUpdate {
2694
                name: bookmark.clone(),
2695
                old_state: remote_state,
2696
                new_state,
2697
                force,
2698
                attestation,
2699
            }],
2700
            policy_to_send,
2701
            tags_to_send,
2702
        )?;
2703
        if let (Some(serial), true) = (
2704
            sent_serial,
2705
            results.iter().any(|r| r.ok && r.name == "policy"),
2706
        ) {
2707
            self.record_policy_serial(remote_name, serial)?;
2708
        }
2709
        if results.iter().any(|r| r.ok && r.name == bookmark) {
2710
            let mut view = op.view.clone();
2711
            view.remotes
2712
                .entry(remote_name.to_string())
2713
                .or_default()
2714
                .insert(bookmark.clone(), new_state);
2715
            self.record_op(&[op_ref], &format!("oxy push {remote_name}"), view)?;
2716
            if let Some(s) = serial {
2717
                self.record_serial(remote_name, &bookmark, s)?;
2718
            }
2719
        }
2720
        Ok(results)
2721
    }
2722
2723
    /// Clone a remote into `dest`: same repository ID, `origin` remote,
2724
    /// bookmark set to the remote's, working copy checked out.
2725
    pub fn clone(url: &str, dest: &Path) -> Result<Repo> {
2726
        Self::clone_with(url, dest, None, &[])
2727
    }
2728
2729
    /// Clone, leaving blobs above `max_blob` bytes to be fetched on first
2730
    /// read, and materializing only `sparse` paths (which implies lazy blobs
2731
    /// unless a size is given, so out-of-scope binaries are never fetched).
2732
    pub fn clone_with(
2733
        url: &str,
2734
        dest: &Path,
2735
        max_blob: Option<u64>,
2736
        sparse: &[String],
2737
    ) -> Result<Repo> {
2738
        Self::clone_full(url, dest, max_blob, sparse, None)
2739
    }
2740
2741
    /// Shallow clone (spec §17): a checkpoint of the bookmark's heads plus the
2742
    /// changes within `depth` steps of them; the rest is the boundary.
2743
    pub fn clone_shallow(
2744
        url: &str,
2745
        dest: &Path,
2746
        depth: u32,
2747
        max_blob: Option<u64>,
2748
        sparse: &[String],
2749
    ) -> Result<Repo> {
2750
        Self::clone_full(url, dest, max_blob, sparse, Some(depth))
2751
    }
2752
2753
    fn clone_full(
2754
        url: &str,
2755
        dest: &Path,
2756
        max_blob: Option<u64>,
2757
        sparse: &[String],
2758
        depth: Option<u32>,
2759
    ) -> Result<Repo> {
2760
        Ok(Self::clone_reporting(url, dest, max_blob, sparse, depth)?.0)
2761
    }
2762
2763
    /// Clone as [`Repo::clone_with`] (or [`Repo::clone_shallow`] with a
2764
    /// `depth`), also returning the bookmarks the fetch left out because
2765
    /// their attestation could not be verified ([`FetchReport::skipped`]),
2766
    /// for the caller to warn about. The bookmark checked out is never one
2767
    /// of them: that fails the clone.
2768
    pub fn clone_reporting(
2769
        url: &str,
2770
        dest: &Path,
2771
        max_blob: Option<u64>,
2772
        sparse: &[String],
2773
        depth: Option<u32>,
2774
    ) -> Result<(Repo, Vec<(String, String)>)> {
2775
        let options = crate::credentials::Credentials::user().options_for(url);
2776
        Self::clone_using(url, dest, max_blob, sparse, depth, &options)
2777
    }
2778
2779
    /// Clone a subrepo with the credentials the parent would use for its
2780
    /// hub (see [`Repo::credentials_for`]). Nothing is recorded in the
2781
    /// subrepo: it finds the same ones again through its parent.
2782
    pub(crate) fn clone_as_subrepo(&self, url: &str, dest: &Path) -> Result<Repo> {
2783
        let options = self.credentials_for(url);
2784
        Ok(Self::clone_using(url, dest, None, &[], None, &options)?.0)
2785
    }
2786
2787
    /// The name of a remote of this repository on the same hub (scheme,
2788
    /// host and port) as `url`.
2789
    fn remote_on_same_hub(&self, url: &str) -> Option<String> {
2790
        let origin = |u: &str| -> Option<String> {
2791
            let (scheme, rest) = u.split_once("://")?;
2792
            if !matches!(scheme, "http" | "https") {
2793
                return None;
2794
            }
2795
            let host = rest.split('/').next()?;
2796
            Some(format!(
2797
                "{}://{}",
2798
                scheme.to_ascii_lowercase(),
2799
                host.to_ascii_lowercase()
2800
            ))
2801
        };
2802
        let want = origin(url)?;
2803
        self.config
2804
            .remotes
2805
            .iter()
2806
            .find(|(_, u)| origin(u).as_deref() == Some(want.as_str()))
2807
            .map(|(name, _)| name.clone())
2808
    }
2809
2810
    /// The token and CA to reach `url` with: those of a remote of this
2811
    /// repository on the same hub (scheme, host and port), else the
2812
    /// environment's (`OXY_TOKEN`, `OXY_CA`), the user's credential store
2813
    /// and enclosing checkouts', as a clone would use them.
2814
    pub(crate) fn credentials_for(&self, url: &str) -> RemoteOptions {
2815
        match self.remote_on_same_hub(url) {
2816
            Some(name) => RemoteOptions {
2817
                repo_id: None,
2818
                cache_dir: None,
2819
                ..self.remote_options(&name)
2820
            },
2821
            None => {
2822
                let mut options = RemoteOptions::default();
2823
                self.fill_credentials(url, &mut options);
2824
                options
2825
            }
2826
        }
2827
    }
2828
2829
    fn clone_using(
2830
        url: &str,
2831
        dest: &Path,
2832
        max_blob: Option<u64>,
2833
        sparse: &[String],
2834
        depth: Option<u32>,
2835
        options: &RemoteOptions,
2836
    ) -> Result<(Repo, Vec<(String, String)>)> {
2837
        let remote = open_remote(url, options)?;
2838
        let id = remote.repo_id()?;
2839
        if id == crate::protocol::UNBOUND_ID {
2840
            return fail(format!(
2841
                "{url} is empty and unbound: push a repository to it first"
2842
            ));
2843
        }
2844
        let mut repo = Repo::init_with_id(dest, id)?;
2845
        repo.config
2846
            .set("remote.origin.url", &crate::protocol::stored_url(url))?;
2847
        if let Some(m) = max_blob {
2848
            repo.config.set("filter.max-blob", &m.to_string())?;
2849
        } else if !sparse.is_empty() {
2850
            repo.config.set("filter.max-blob", "0")?;
2851
        }
2852
        if !sparse.is_empty() {
2853
            repo.config.set("filter.paths", &sparse.join(","))?;
2854
        }
2855
        repo.save_config()?;
2856
        let skipped = if let Some(depth) = depth {
2857
            repo.fetch_shallow("origin", depth)?;
2858
            Vec::new()
2859
        } else {
2860
            repo.fetch("origin")?.skipped
2861
        };
2862
        let (op_ref, op) = repo.head_op()?;
2863
        // The remote's default bookmark, else ours (main), else its only
2864
        // one; a bookmark the fetch skipped counts, so the clone fails
2865
        // rather than check out another (oxygenhub#266).
2866
        let mut theirs = op.view.remotes.get("origin").cloned().unwrap_or_default();
2867
        for (name, _) in &skipped {
2868
            theirs.insert(name.clone(), ObjectRef::ZERO);
2869
        }
2870
        // Asked of the remote as now configured, so a store-backed one
2871
        // answers from the mirror the fetch just filled.
2872
        let advertised = repo
2873
            .open_named_remote("origin")
2874
            .ok()
2875
            .and_then(|r| r.refs().ok())
2876
            .and_then(|r| r.default_bookmark);
2877
        let bookmark = advertised
2878
            .filter(|b| theirs.contains_key(b))
2879
            .or_else(|| {
2880
                theirs
2881
                    .contains_key(&repo.config.bookmark)
2882
                    .then(|| repo.config.bookmark.clone())
2883
            })
2884
            .or_else(|| {
2885
                (theirs.len() == 1)
2886
                    .then(|| theirs.keys().next().cloned())
2887
                    .flatten()
2888
            })
2889
            .unwrap_or_else(|| repo.config.bookmark.clone());
2890
        if let Some((_, why)) = skipped.iter().find(|(name, _)| *name == bookmark) {
2891
            return fail(format!("cannot check out {bookmark}: {why}"));
2892
        }
2893
        let initial = repo.config.bookmark.clone();
2894
        if bookmark != repo.config.bookmark {
2895
            fs::create_dir_all(repo.workspace_bookmark_path().parent().unwrap())?;
2896
            write_atomic(&repo.workspace_bookmark_path(), bookmark.as_bytes())?;
2897
            repo.config.bookmark = bookmark.clone();
2898
        }
2899
        if let Some(state) = theirs.get(&bookmark).copied() {
2900
            let heads = repo.store.get_state(&state)?.heads;
2901
            let mut view = op.view.clone();
2902
            // The empty bookmark `init` made goes when another is checked out.
2903
            if initial != bookmark && repo.heads_of(&view, &initial)?.is_empty() {
2904
                view.bookmarks.remove(&initial);
2905
            }
2906
            view.bookmarks.insert(bookmark, state);
2907
            repo.record_op(&[op_ref], &format!("oxy clone {url}"), view)?;
2908
            repo.checkout(&heads, false, "oxy clone (checkout)")?;
2909
        }
2910
        Ok((repo, skipped))
2911
    }
2912
}
2913
2914
/// The serial of a reference's next attestation: one more than the last the
2915
/// remote reports, or, for a bookmark the remote does not have, the clock in
2916
/// milliseconds, so a bookmark created again after a deletion goes past the
2917
/// deletion's serial the remote keeps.
2918
fn next_serial(last: Option<u64>) -> u64 {
2919
    match last {
2920
        Some(l) => l + 1,
2921
        None => Hlc::now().wall_ms.max(1),
2922
    }
2923
}