OXYGENHUB
◆ v0.11.0 oxygen / crates / oxy-core / src / repo / sync.rs
124.4 KiB · Rust · last changed Fetch every bookmark in one exchange, and none when nothing is new: a no-op pull of 160 bookmarks 169 s to 3 s, a clone 324 s to 26 s (closes #158) 4 days ago source blame history raw
98286406 Andy McCall ↶ 1
//! Serving and talking to remotes (spec §10.2, §16).
2
1ec6fdbc Andy McCall ↶ 3
use super::tags::TagCheck;
98286406 Andy McCall ↶ 4
use super::*;
5
7aa2dc93 Andy McCall ↶ 6
/// What a push did, for hosts (hub spec §6.12).
7
#[derive(Clone, Debug, Default)]
8
pub struct PushReport {
9
    pub results: Vec<RefResult>,
10
    /// Accepted reference updates, in request order.
11
    pub updated: Vec<RefUpdated>,
12
    /// Changes this push stored that were not already present.
13
    pub imported: Vec<ChangeHash>,
14
}
15
16
#[derive(Clone, Debug)]
17
pub struct RefUpdated {
18
    pub name: String,
19
    pub old_state: ObjectRef,
20
    pub new_state: ObjectRef,
21
    pub heads: Vec<ChangeHash>,
22
}
23
83e0677b Andy McCall ↶ 24
/// What a fetch brought in.
25
#[derive(Debug, Clone, PartialEq, Eq, Default)]
26
pub struct FetchReport {
27
    /// Bookmark name, its heads, and how many changes were new.
28
    pub refs: Vec<(String, Vec<ChangeHash>, usize)>,
132b884b Andy McCall ↶ 29
    /// Tags adopted from the remote.
30
    pub new_tags: usize,
b63fc7a7 Andy McCall ↶ 31
    /// Tags the remote withdrew, and so withdrawn here (oxygenhub#301).
32
    pub withdrawn_tags: Vec<String>,
83e0677b Andy McCall ↶ 33
    /// The remote has never presented a policy, so nothing was verified
34
    /// against one (spec §14.5: warn loudly).
35
    pub unverified: bool,
4b9d0677 Andy McCall ↶ 36
    /// What checking the host's transparency log found short of a fork
37
    /// (spec §14.8): attestations not logged yet, a log that could not be
38
    /// read. A fork fails the fetch instead.
39
    pub log_notes: Vec<String>,
d36ba2f7 Andy McCall ↶ 40
    /// Bookmarks left as they were because their attestation could not be
41
    /// verified (its signer unknown here, or its signature bad), each with
42
    /// why (oxygenhub#266). The current bookmark is never skipped: its
43
    /// fetch fails instead.
44
    pub skipped: Vec<(String, String)>,
83e0677b Andy McCall ↶ 45
}
46
98286406 Andy McCall ↶ 47
#[derive(Debug, Clone, PartialEq, Eq)]
48
pub struct PullReport {
14edbaaf Andy McCall ↶ 49
    /// The current bookmark's heads after the pull.
98286406 Andy McCall ↶ 50
    pub heads: Vec<ChangeHash>,
14edbaaf Andy McCall ↶ 51
    /// Whether the current bookmark changed.
98286406 Andy McCall ↶ 52
    pub changed: bool,
53
    pub checked_out: bool,
14edbaaf Andy McCall ↶ 54
    /// Other bookmarks the pull moved: each local bookmark tracks the
55
    /// remote's of the same name.
56
    pub others: Vec<String>,
d36ba2f7 Andy McCall ↶ 57
    /// What the fetch left as it was ([`FetchReport::skipped`]).
58
    pub skipped: Vec<(String, String)>,
98286406 Andy McCall ↶ 59
}
60
cb8b8082 Andy McCall ↶ 61
/// The result a push of objects alone reports (a batch of a long history):
62
/// not a reference.
63
pub const OBJECTS_RESULT: &str = "objects";
64
80b069fa Andy McCall ↶ 65
/// Graph blobs asked for in one `objects` request when a shallow clone
66
/// fetches its checkpoint in parts (#137): their chunk lists and chunks,
67
/// a few MiB a request rather than one answer of gigabytes.
68
const CHECKPOINT_PART: usize = 2048;
69
98286406 Andy McCall ↶ 70
impl Repo {
71
    // ------------------------------------------------------------ serving (spec §16)
72
73
    fn ref_attest_path(&self, bookmark: &str) -> PathBuf {
54cd08a1 Andy McCall ↶ 74
        self.oxy.join("attest").join(name_file(bookmark))
98286406 Andy McCall ↶ 75
    }
76
77
    /// The latest reference attestation stored for a bookmark.
78
    pub fn latest_ref_attest(&self, bookmark: &str) -> Result<Option<(ObjectRef, SignedAttest)>> {
79
        match fs::read_to_string(self.ref_attest_path(bookmark)) {
80
            Ok(h) => {
75167672 Andy McCall 81
                let r = ObjectRef::from_hex(h.trim())
82
                    .ok_or_else(|| RepoError::Msg("corrupt attestation pointer".into()))?;
83
                Ok(Some((
84
                    r,
85
                    SignedAttest {
86
                        attest: self.store.get_attest(&r)?,
87
                        sigs: self.store.signatures(&r)?,
88
                    },
89
                )))
98286406 Andy McCall ↶ 90
            }
91
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
92
            Err(e) => Err(e.into()),
93
        }
94
    }
95
9f9e72b9 Andy McCall ↶ 96
    pub(crate) fn set_ref_attest(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
98286406 Andy McCall ↶ 97
        fs::create_dir_all(self.oxy.join("attest"))?;
98
        write_atomic(&self.ref_attest_path(bookmark), r.to_hex().as_bytes())
99
    }
100
83e0677b Andy McCall ↶ 101
    pub fn remote_refs(&self) -> Result<RefsResponse> {
98286406 Andy McCall ↶ 102
        let view = self.view()?;
103
        let mut out = Vec::new();
104
        for (name, state) in &view.bookmarks {
105
            let latest = self.latest_ref_attest(name)?.map(|(_, a)| a);
106
            let last_serial = latest.as_ref().and_then(|a| match a.attest.body {
107
                AttestBody::Ref { serial, .. } => Some(serial),
108
                _ => None,
109
            });
110
            let attestation = latest.filter(|a| matches!(&a.attest.body, AttestBody::Ref { new_state, .. } if new_state == state));
f53cbe7a Andy McCall ↶ 111
            let accepted = match &attestation {
112
                Some(a) => self.acceptance_of(&a.attest.hash())?,
113
                None => None,
114
            };
75167672 Andy McCall 115
            out.push(RemoteRef {
116
                name: name.clone(),
117
                state: *state,
118
                heads: self.store.get_state(state)?.heads,
119
                attestation,
120
                last_serial,
f53cbe7a Andy McCall ↶ 121
                accepted,
4b9d0677 Andy McCall ↶ 122
                logged: None,
028545df Andy McCall ↶ 123
                integration: vec![],
75167672 Andy McCall 124
            });
98286406 Andy McCall ↶ 125
        }
28fc23d7 Andy McCall ↶ 126
        let policy = self.current_policy()?.map(|(_, sp)| sp);
75167672 Andy McCall 127
        let history = self
128
            .policy_history()?
129
            .into_iter()
130
            .filter(|sp| {
131
                policy
132
                    .as_ref()
133
                    .is_some_and(|p| sp.policy.serial < p.policy.serial)
134
            })
135
            .collect();
136
        Ok(RefsResponse {
137
            refs: out,
138
            policy,
139
            tags: self.tag_refs()?,
140
            history,
14edbaaf Andy McCall ↶ 141
            default_bookmark: Some(self.config.bookmark.clone()),
4b9d0677 Andy McCall ↶ 142
            log: None,
028545df Andy McCall ↶ 143
            witnesses: vec![],
b63fc7a7 Andy McCall ↶ 144
            withdrawn: self.withdrawn_refs()?,
75167672 Andy McCall 145
        })
98286406 Andy McCall ↶ 146
    }
147
148
    /// Every change reachable from `heads` that is in the store.
149
    fn closure_in_store(&self, heads: &[ChangeHash]) -> Result<BTreeSet<ChangeHash>> {
150
        let mut seen = BTreeSet::new();
151
        let mut stack: Vec<ChangeHash> = heads.to_vec();
152
        while let Some(h) = stack.pop() {
153
            if seen.contains(&h) || !self.store.has(&h) {
154
                continue;
155
            }
156
            seen.insert(h);
69816d68 Andy McCall ↶ 157
            stack.extend(self.store.get_change_meta(&h)?.deps);
98286406 Andy McCall ↶ 158
        }
159
        Ok(seen)
160
    }
161
2863456a Andy McCall ↶ 162
    /// The changes within `depth` dependency steps of `heads` (spec §17),
163
    /// nearest first; depth 1 is the heads themselves.
164
    pub fn shallow_changes(&self, heads: &[ChangeHash], depth: u32) -> Result<Vec<ChangeHash>> {
165
        let mut out = Vec::new();
166
        let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
167
        let mut layer: Vec<ChangeHash> = heads.to_vec();
168
        for _ in 0..depth.max(1) {
169
            let mut next = Vec::new();
170
            for h in layer {
171
                if !seen.insert(h) || !self.store.has(&h) {
172
                    continue;
173
                }
174
                let c = self.store.get_change(&h)?;
175
                next.extend(c.deps.iter().copied());
176
                out.push(h);
177
            }
178
            if next.is_empty() {
179
                break;
180
            }
181
            layer = next;
182
        }
183
        out.reverse();
184
        Ok(out)
185
    }
186
187
    fn shallow_path(&self) -> PathBuf {
188
        self.oxy.join("shallow")
189
    }
190
191
    /// The shallow boundary: dependencies this repository does not hold (spec §17).
192
    pub fn boundary(&self) -> BTreeSet<ChangeHash> {
75167672 Andy McCall 193
        fs::read_to_string(self.shallow_path())
194
            .map(|t| t.lines().filter_map(ObjectRef::from_hex).collect())
195
            .unwrap_or_default()
2863456a Andy McCall ↶ 196
    }
197
198
    pub fn is_shallow(&self) -> bool {
199
        !self.boundary().is_empty()
200
    }
201
202
    fn write_boundary(&self, boundary: &BTreeSet<ChangeHash>) -> Result<()> {
203
        if boundary.is_empty() {
204
            let _ = fs::remove_file(self.shallow_path());
205
            return Ok(());
206
        }
75167672 Andy McCall 207
        let text: String = boundary
208
            .iter()
209
            .map(|h| format!("{}\n", h.to_hex()))
210
            .collect();
2863456a Andy McCall ↶ 211
        write_atomic(&self.shallow_path(), text.as_bytes())
212
    }
213
214
    /// Recompute the boundary from what is stored: every dependency of a
215
    /// stored change that is not itself stored.
216
    fn refresh_boundary(&self, imported: &[ChangeHash]) -> Result<()> {
217
        let mut boundary = self.boundary();
218
        boundary.retain(|h| !self.store.has(h));
219
        for h in imported {
69816d68 Andy McCall ↶ 220
            for d in self.store.get_change_meta(h)?.deps {
2863456a Andy McCall ↶ 221
                if !self.store.has(&d) {
222
                    boundary.insert(d);
223
                }
224
            }
225
        }
226
        self.write_boundary(&boundary)
227
    }
228
229
    /// Fetch `depth` more levels of history below the shallow boundary.
230
    pub fn deepen(&self, remote_name: &str, depth: u32) -> Result<usize> {
231
        let boundary = self.boundary();
232
        if boundary.is_empty() {
233
            return Ok(0);
234
        }
235
        let remote = self.open_named_remote(remote_name)?;
236
        let wanted = remote.shallow(&boundary.iter().copied().collect::<Vec<_>>(), depth)?;
237
        let bundle = remote.fetch(&wanted, self.config.max_blob)?;
238
        let imported = self.import_bundle_shallow(&bundle, false)?;
239
        if self.config.max_blob.is_some() {
240
            self.register_missing_blobs(&imported, remote_name)?;
241
        }
242
        self.refresh_boundary(&imported)?;
243
        Ok(imported.len())
244
    }
245
98286406 Andy McCall ↶ 246
    /// `closure(want) \ closure(have)`, dependencies first (spec §16.2).
247
    pub fn negotiate(&self, have: &[ChangeHash], want: &[ChangeHash]) -> Result<Vec<ChangeHash>> {
248
        let have = self.closure_in_store(have)?;
2863456a Andy McCall ↶ 249
        // In a shallow repository the walk stops at the boundary, whose
250
        // changes the remote is assumed to hold.
251
        let boundary = self.boundary();
98286406 Andy McCall ↶ 252
        let mut out = Vec::new();
253
        let mut done: BTreeSet<ChangeHash> = have.clone();
2863456a Andy McCall ↶ 254
        done.extend(boundary.iter().copied());
98286406 Andy McCall ↶ 255
        // Iterative post-order DFS.
f3811eb2 Andy McCall ↶ 256
        let shallow = self.is_shallow();
98286406 Andy McCall ↶ 257
        let mut stack: Vec<(ChangeHash, bool)> = want.iter().rev().map(|h| (*h, false)).collect();
258
        while let Some((h, expanded)) = stack.pop() {
259
            if done.contains(&h) {
260
                continue;
261
            }
262
            if !self.store.has(&h) {
f3811eb2 Andy McCall ↶ 263
                // A shallow repository holds only part of its history; a
264
                // change it lacks is below what it cloned (in a depth-0
265
                // clone, any change of the checkpoint), which the remote
266
                // is assumed to hold, as it is for the boundary (#137).
267
                if shallow {
268
                    done.insert(h);
269
                    continue;
270
                }
98286406 Andy McCall ↶ 271
                return fail(format!("change {} is not in this repository", h.short(12)));
272
            }
273
            if expanded {
274
                done.insert(h);
275
                out.push(h);
276
                continue;
277
            }
278
            stack.push((h, true));
69816d68 Andy McCall ↶ 279
            for d in self.store.get_change_meta(&h)?.deps {
98286406 Andy McCall ↶ 280
                if !done.contains(&d) {
281
                    stack.push((d, false));
282
                }
283
            }
284
        }
285
        Ok(out)
286
    }
287
288
    /// Bundle changes (in the given dependency order) with their signatures,
289
    /// seals, identities, blobs, chunks.
290
    pub fn bundle_for(&self, changes: &[ChangeHash]) -> Result<Bundle> {
d36ba2f7 Andy McCall ↶ 291
        self.bundle_of(changes, None, false, false)
b3c5a59a Andy McCall ↶ 292
    }
293
294
    /// Objects by reference for the `objects` endpoint; a blob brings its chunks.
295
    pub fn objects_for(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>> {
80b069fa Andy McCall ↶ 296
        // In two reads in pack order (#137): a checkpoint's graph blobs come
297
        // this way in parts, tens of thousands for a big state.
298
        let mut found = Vec::with_capacity(refs.len());
299
        for got in self.store.get_many(refs) {
300
            match got {
301
                Ok(x) => found.push(x),
302
                Err(StoreError::NotFound(_)) => {}
b3c5a59a Andy McCall ↶ 303
                Err(e) => return Err(e.into()),
80b069fa Andy McCall ↶ 304
            }
305
        }
306
        let mut chunks = Vec::new();
307
        for (ty, body) in &found {
308
            if *ty == ObjectType::Blob {
309
                chunks.extend(Blob::decode(body)?.chunks);
310
            }
311
        }
312
        let mut out = Vec::with_capacity(found.len() + chunks.len());
313
        for got in self.store.get_many(&chunks) {
314
            match got {
315
                Ok(x) => out.push(x),
316
                Err(StoreError::NotFound(_)) => {}
317
                Err(e) => return Err(e.into()),
b3c5a59a Andy McCall ↶ 318
            }
319
        }
80b069fa Andy McCall ↶ 320
        out.extend(found);
b3c5a59a Andy McCall ↶ 321
        Ok(out)
322
    }
323
324
    /// Like `bundle_for`, leaving out blobs larger than `max_blob` bytes
325
    /// (and their chunks) so the receiver fetches them lazily (spec §13.5).
d36ba2f7 Andy McCall ↶ 326
    ///
327
    /// This is what a server answers a fetch with, so it also carries what
328
    /// verifies the attestations it serves beside it (see
329
    /// [`Repo::served_attest_signers`]), also when `changes` is empty.
75167672 Andy McCall 330
    pub fn bundle_for_filtered(
331
        &self,
332
        changes: &[ChangeHash],
333
        max_blob: Option<u64>,
334
    ) -> Result<Bundle> {
d36ba2f7 Andy McCall ↶ 335
        self.bundle_of(changes, max_blob, false, true)
2c75672f Andy McCall ↶ 336
    }
337
338
    /// Like [`Repo::bundle_for_filtered`] for a fetch between repositories
339
    /// on one machine: changes as they are stored ([`Bundle::raw_changes`]),
340
    /// with their text blobs and chunks among the objects, so neither side
341
    /// decodes or encodes their text.
342
    pub fn bundle_raw_for(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle> {
d36ba2f7 Andy McCall ↶ 343
        self.bundle_of(changes, max_blob, true, true)
2c75672f Andy McCall ↶ 344
    }
345
d36ba2f7 Andy McCall ↶ 346
    /// The keys that signed the attestations this repository serves with
347
    /// its references: each bookmark's latest and each tag's. A fetch
348
    /// bundle carries their identities and certificates (spec §16.2), so a
349
    /// client can check an attestation whose signer signed none of the
350
    /// changes it receives, a delegate or a short-lived key that only
351
    /// moved a bookmark (oxygenhub#266). One pointer and one signature
e10ba800 Andy McCall ↶ 352
    /// list per reference, computed once per operation head and kept
353
    /// (every bookmark move and tag is an operation): a client sends one
354
    /// fetch per bookmark, and on a bucket store each signature list is a
355
    /// LIST request, so recomputing per fetch cost metalgear2 (145
356
    /// bookmarks) 14 s a fetch.
d36ba2f7 Andy McCall ↶ 357
    pub fn served_attest_signers(&self) -> Result<BTreeSet<KeyId>> {
e10ba800 Andy McCall ↶ 358
        type Signers = std::collections::HashMap<(PathBuf, ObjectRef), BTreeSet<KeyId>>;
359
        static SERVED: std::sync::Mutex<Option<Signers>> = std::sync::Mutex::new(None);
360
        let key = (self.oxy.clone(), self.head_op()?.0);
361
        if let Some(keys) = SERVED
362
            .lock()
363
            .unwrap_or_else(|p| p.into_inner())
364
            .as_ref()
365
            .and_then(|m| m.get(&key))
366
        {
367
            return Ok(keys.clone());
368
        }
369
        let keys = self.attest_signers_now()?;
370
        let mut served = SERVED.lock().unwrap_or_else(|p| p.into_inner());
371
        let map = served.get_or_insert_with(Default::default);
372
        // One entry per repository: an older operation head is never asked
373
        // for again once a newer one is current.
374
        map.retain(|(oxy, _), _| *oxy != key.0);
375
        if map.len() >= 1024 {
376
            map.clear();
377
        }
378
        map.insert(key, keys.clone());
379
        Ok(keys)
380
    }
381
382
    fn attest_signers_now(&self) -> Result<BTreeSet<KeyId>> {
d36ba2f7 Andy McCall ↶ 383
        let view = self.view()?;
384
        let pointers = view
385
            .bookmarks
386
            .keys()
387
            .map(|b| self.ref_attest_path(b))
388
            .chain(view.tags.keys().map(|t| self.tag_attest_path(t)));
389
        let mut keys = BTreeSet::new();
390
        for p in pointers {
391
            let Ok(text) = fs::read_to_string(&p) else {
392
                continue;
393
            };
394
            let Some(r) = ObjectRef::from_hex(text.trim()) else {
395
                continue;
396
            };
397
            keys.extend(self.store.signatures(&r)?.into_iter().map(|s| s.key));
398
        }
399
        Ok(keys)
400
    }
401
402
    /// `serving`: a fetch's answer, which carries the identities and
403
    /// certificates of [`Repo::served_attest_signers`] too.
2c75672f Andy McCall ↶ 404
    fn bundle_of(
405
        &self,
406
        changes: &[ChangeHash],
407
        max_blob: Option<u64>,
408
        raw: bool,
d36ba2f7 Andy McCall ↶ 409
        serving: bool,
2c75672f Andy McCall ↶ 410
    ) -> Result<Bundle> {
98286406 Andy McCall ↶ 411
        let mut bundle = Bundle::default();
412
        let mut objects: BTreeSet<ObjectRef> = BTreeSet::new();
413
        let mut seals: BTreeSet<ObjectRef> = BTreeSet::new();
414
        let add_object = |r: &ObjectRef, objects: &mut BTreeSet<ObjectRef>| {
415
            if !r.is_zero() && self.store.has(r) {
416
                objects.insert(*r);
417
            }
418
        };
419
        for h in changes {
2c75672f Andy McCall ↶ 420
            // Raw: the stored bytes, and a skeleton for the rest.
421
            let (c, encoded) = if raw {
422
                let bytes = self.store.get_typed(h, ObjectType::Change)?;
423
                let (skel, texts) = Change::decode_skeleton(&bytes, *h)?;
424
                for t in texts {
0f93f49c Andy McCall ↶ 425
                    if let crate::objects::TextRef::Blob(r, _, _) = t {
2c75672f Andy McCall ↶ 426
                        let blob = Blob::decode(&self.store.get_typed(&r, ObjectType::Blob)?)?;
427
                        objects.insert(r);
428
                        objects.extend(blob.chunks);
429
                    }
430
                }
431
                (skel, Some(bytes))
432
            } else {
433
                (self.store.get_change(h)?, None)
434
            };
98286406 Andy McCall ↶ 435
            let sigs = self.store.signatures(h)?;
436
            add_object(&c.author, &mut objects);
437
            for s in &sigs {
438
                add_object(&s.key, &mut objects);
439
            }
570e3531 Andy McCall ↶ 440
            // A replacement's new content only: its old content is the new
441
            // content of a change it depends on, which the receiver holds or
442
            // which this bundle carries (oxygen#136: removing ~1,100 binary
443
            // files sent them all again).
98286406 Andy McCall ↶ 444
            for e in &c.edits {
570e3531 Andy McCall ↶ 445
                if let Edit::Replace { new: Some(b), .. } = e {
446
                    if self.store.has(b) {
447
                        if let Ok(blob) = Blob::decode(&self.store.get_typed(b, ObjectType::Blob)?)
448
                        {
449
                            if max_blob.is_some_and(|m| blob.length > m) {
450
                                continue;
98286406 Andy McCall ↶ 451
                            }
570e3531 Andy McCall ↶ 452
                            objects.insert(*b);
453
                            objects.extend(blob.chunks.iter().copied());
98286406 Andy McCall ↶ 454
                        }
455
                    }
456
                }
457
            }
458
            for s in self.store.seals_for(h)? {
459
                seals.insert(s);
2c75672f Andy McCall ↶ 460
            }
461
            match encoded {
462
                Some(bytes) => bundle.raw_changes.push((bytes, sigs)),
463
                None => {
464
                    bundle.changes.push((c, sigs));
465
                    bundle.hashes.push(*h);
466
                }
98286406 Andy McCall ↶ 467
            }
468
        }
469
        for r in seals {
470
            let attest = self.store.get_attest(&r)?;
471
            let sigs = self.store.signatures(&r)?;
472
            for s in &sigs {
473
                add_object(&s.key, &mut objects);
474
            }
475
            bundle.attests.push(SignedAttest { attest, sigs });
476
        }
d36ba2f7 Andy McCall ↶ 477
        let signers: Vec<KeyId> = if serving {
478
            let signers = self.served_attest_signers()?;
479
            for k in &signers {
480
                add_object(k, &mut objects);
481
            }
482
            signers.into_iter().collect()
483
        } else {
484
            Vec::new()
485
        };
98286406 Andy McCall ↶ 486
        for r in objects {
487
            let (ty, body) = self.store.get(&r)?;
488
            bundle.objects.push((ty, body));
489
        }
d36ba2f7 Andy McCall ↶ 490
        self.attach_certificates_for(&mut bundle, &signers)?;
98286406 Andy McCall ↶ 491
        Ok(bundle)
492
    }
493
f0a517d2 Andy McCall ↶ 494
    /// Add the certificates of short-lived keys that signed what `bundle`
495
    /// carries, and their issuers' identities, so a receiver can tell who
496
    /// they stand for (spec §14.7). `also` names more signers (a reference
497
    /// attestation sent beside the bundle).
498
    pub(crate) fn attach_certificates_for(
499
        &self,
500
        bundle: &mut Bundle,
501
        also: &[KeyId],
502
    ) -> Result<()> {
503
        let mut keys: BTreeSet<KeyId> = also.iter().copied().collect();
504
        keys.extend(
505
            bundle
506
                .changes
507
                .iter()
508
                .flat_map(|(_, s)| s.iter().map(|s| s.key)),
509
        );
510
        keys.extend(
511
            bundle
512
                .raw_changes
513
                .iter()
514
                .flat_map(|(_, s)| s.iter().map(|s| s.key)),
515
        );
516
        keys.extend(
517
            bundle
518
                .attests
519
                .iter()
520
                .flat_map(|a| a.sigs.iter().map(|s| s.key)),
521
        );
522
        for k in keys {
523
            let Some(r) = self.store.certificate_of(&k)? else {
524
                continue;
525
            };
526
            let cert = self.store.get_certificate(&r)?;
527
            if bundle
528
                .certificates
529
                .iter()
530
                .any(|c| c.cert.hash() == cert.cert.hash())
531
            {
532
                continue;
533
            }
f0ef9778 Andy McCall ↶ 534
            // An agent's certificate brings its grant's signer too, so the
535
            // receiver can check the grant (oxygen#134).
536
            let grantor = cert.cert.grant.as_ref().map(|g| g.grant.grantor);
537
            for id in [Some(k), Some(cert.cert.issuer), grantor].iter().flatten() {
f0a517d2 Andy McCall ↶ 538
                if let Ok(body) = self.store.get_typed(id, ObjectType::Identity) {
539
                    if !bundle
540
                        .objects
541
                        .iter()
542
                        .any(|(t, b)| *t == ObjectType::Identity && *b == body)
543
                    {
544
                        bundle.objects.push((ObjectType::Identity, body));
545
                    }
546
                }
547
            }
548
            bundle.certificates.push(cert);
549
        }
550
        Ok(())
551
    }
552
553
    /// Keep the certificates a bundle brings once each verifies with the
554
    /// identity of the issuer it names; one that does not refuses the bundle.
555
    fn adopt_certificates(&self, certs: &[crate::cert::SignedCertificate]) -> Result<()> {
556
        for c in certs {
557
            let issuer = self.store.get_identity(&c.cert.issuer).map_err(|_| {
558
                RepoError::Msg(format!(
559
                    "a certificate names issuer {}, whose identity did not come with it",
560
                    c.cert.issuer.short(12)
561
                ))
562
            })?;
563
            c.verify(&issuer)
564
                .map_err(|e| RepoError::Msg(format!("a certificate does not verify: {e}")))?;
565
            self.store.put_identity(&c.cert.key)?;
566
            self.store.put_certificate(c)?;
567
        }
568
        Ok(())
569
    }
570
571
    /// A signer authorized for `reference` by what an issuer certified it
572
    /// as (spec §14.7): a short-lived key with a kept certificate that
573
    /// verifies with its issuer, gives `scope`, covers `at_ms` (a time the
574
    /// signer did not choose) and names a subject the policy grants.
575
    pub(crate) fn certified_authorized(
576
        &self,
577
        p: &crate::policy::Policy,
578
        signer: &KeyId,
579
        reference: &str,
580
        scope: Scope,
581
        at_ms: u64,
582
    ) -> bool {
583
        let Ok(Some(r)) = self.store.certificate_of(signer) else {
584
            return false;
585
        };
586
        let Ok(c) = self.store.get_certificate(&r) else {
587
            return false;
588
        };
589
        let Some(issuer) = p
590
            .identity(&c.cert.issuer)
591
            .cloned()
592
            .or_else(|| self.store.get_identity(&c.cert.issuer).ok())
593
        else {
594
            return false;
595
        };
596
        c.verify(&issuer).is_ok()
597
            && c.cert.key_id() == *signer
598
            && c.cert.covers(scope, at_ms)
599
            && p.subject_grant(reference, &c.cert.issuer, &c.cert.subject, scope)
600
    }
601
98286406 Andy McCall ↶ 602
    /// Store a bundle's contents. With `strict`, every change must
603
    /// authenticate (signature or seal by its author or a delegate).
604
    pub fn import_bundle(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
2863456a Andy McCall ↶ 605
        self.import_bundle_inner(bundle, strict, false)
606
    }
607
608
    /// Like `import_bundle`, accepting changes whose dependencies are absent
609
    /// (they join the shallow boundary).
610
    pub fn import_bundle_shallow(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
611
        self.import_bundle_inner(bundle, strict, true)
612
    }
613
75167672 Andy McCall 614
    fn import_bundle_inner(
615
        &self,
616
        bundle: &Bundle,
617
        strict: bool,
618
        shallow: bool,
619
    ) -> Result<Vec<ChangeHash>> {
8f10a07a Andy McCall ↶ 620
        let batch = self.store.begin_batch()?;
621
        let result = self.import_bundle_batched(bundle, strict, shallow);
622
        batch.finish()?;
623
        result
624
    }
625
75167672 Andy McCall 626
    fn import_bundle_batched(
627
        &self,
628
        bundle: &Bundle,
629
        strict: bool,
630
        shallow: bool,
631
    ) -> Result<Vec<ChangeHash>> {
2c75672f Andy McCall ↶ 632
        for (ty, _) in &bundle.objects {
633
            if !matches!(
634
                ty,
75167672 Andy McCall 635
                ObjectType::Chunk
2c75672f Andy McCall ↶ 636
                    | ObjectType::Blob
637
                    | ObjectType::Identity
638
                    | ObjectType::State
639
                    | ObjectType::Checkpoint
640
            ) {
641
                return fail(format!("bundle may not carry {} objects", ty.name()));
98286406 Andy McCall ↶ 642
            }
643
        }
2c75672f Andy McCall ↶ 644
        // All at once: in a large batch they are compressed on every core.
645
        let objects: Vec<(ObjectType, &[u8])> = bundle
646
            .objects
647
            .iter()
648
            .map(|(ty, body)| (*ty, body.as_slice()))
649
            .collect();
650
        self.store.put_many(&objects)?;
f0a517d2 Andy McCall ↶ 651
        self.adopt_certificates(&bundle.certificates)?;
98286406 Andy McCall ↶ 652
        for sa in &bundle.attests {
653
            let r = self.store.put_attest(&sa.attest)?;
654
            for sig in &sa.sigs {
655
                self.store.put_signature(&r, sig)?;
656
            }
657
            if let AttestBody::Seal { hashes } = &sa.attest.body {
658
                for h in hashes {
659
                    self.store.index_seal(h, &r)?;
660
                }
661
            }
662
        }
598009a7 Andy McCall ↶ 663
        // Each change's hash: from the bundle when its maker knew it, else
664
        // worked out once (for a change adding a whole tree, hashing is
665
        // chunking all its text).
666
        let hashes: Vec<ChangeHash> = if bundle.hashes.len() == bundle.changes.len() {
667
            bundle.hashes.clone()
668
        } else {
669
            bundle.changes.iter().map(|(c, _)| c.hash()).collect()
670
        };
2c75672f Andy McCall ↶ 671
        let raw_hashes: Vec<ChangeHash> = bundle
672
            .raw_changes
673
            .iter()
674
            .map(|(b, _)| hash_object(ObjectType::Change, b))
675
            .collect();
676
        let incoming: BTreeSet<ChangeHash> = hashes.iter().chain(&raw_hashes).copied().collect();
2863456a Andy McCall ↶ 677
        let boundary = self.boundary();
98286406 Andy McCall ↶ 678
        let mut imported = Vec::new();
2c75672f Andy McCall ↶ 679
        // Changes as stored: checked, then stored as they are, their text
680
        // blobs having come with the objects.
681
        for ((bytes, sigs), h) in bundle.raw_changes.iter().zip(raw_hashes) {
682
            let (skel, texts) = Change::decode_skeleton(bytes, h)?;
683
            for d in &skel.deps {
684
                if !self.store.has(d) && !incoming.contains(d) && !shallow && !boundary.contains(d)
685
                {
686
                    return fail(format!(
687
                        "change {} depends on {}, which is neither stored nor in the bundle",
688
                        h.short(12),
689
                        d.short(12)
690
                    ));
691
                }
692
            }
693
            for t in texts {
0f93f49c Andy McCall ↶ 694
                if let crate::objects::TextRef::Blob(r, _, _) = t {
2c75672f Andy McCall ↶ 695
                    if !self.store.has(&r) {
696
                        return fail(format!(
697
                            "change {} came without its text blob {}",
698
                            h.short(12),
699
                            r.short(12)
700
                        ));
701
                    }
702
                }
703
            }
704
            self.store.put(ObjectType::Change, bytes)?;
705
            for sig in sigs {
706
                self.store.put_signature(&h, sig)?;
707
            }
708
            imported.push(h);
709
        }
598009a7 Andy McCall ↶ 710
        for ((c, sigs), h) in bundle.changes.iter().zip(hashes) {
98286406 Andy McCall ↶ 711
            for d in &c.deps {
75167672 Andy McCall 712
                if !self.store.has(d) && !incoming.contains(d) && !shallow && !boundary.contains(d)
713
                {
714
                    return fail(format!(
715
                        "change {} depends on {}, which is neither stored nor in the bundle",
716
                        h.short(12),
717
                        d.short(12)
718
                    ));
98286406 Andy McCall ↶ 719
                }
720
            }
598009a7 Andy McCall ↶ 721
            let stored = self.store.put_change(c)?;
722
            if stored != h {
723
                return fail(format!(
724
                    "change {} is not the change the bundle names",
725
                    h.short(12)
726
                ));
727
            }
98286406 Andy McCall ↶ 728
            for sig in sigs {
729
                self.store.put_signature(&h, sig)?;
730
            }
731
            imported.push(h);
732
        }
733
        if strict {
734
            for h in &imported {
735
                if !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. }) {
75167672 Andy McCall 736
                    return fail(format!(
737
                        "change {} is not authenticated by its author",
738
                        h.short(12)
739
                    ));
98286406 Andy McCall ↶ 740
                }
741
            }
742
        }
743
        Ok(imported)
744
    }
745
746
    /// Verify a reference attestation against an update it claims to describe.
747
    /// `allow_equal` lets a client re-read the attestation it already has; a
748
    /// server receiving a push requires a strictly greater serial.
84b9e9ac Andy McCall ↶ 749
    /// A reference attestation signed by a delegate of an identity the
750
    /// rule lists is authorized as that identity (spec §14.5): the identity
7ba155f2 Andy McCall ↶ 751
    /// named the delegate with the `refs` scope, still in force at `at_ms`
752
    /// (the attestation's wall time), and the policy names the identity.
753
    /// Revoked keys authorize nothing either way.
eae59556 Andy McCall ↶ 754
    pub(crate) fn delegate_authorized(
755
        &self,
756
        p: &crate::policy::Policy,
757
        signer: &KeyId,
758
        reference: &str,
7ba155f2 Andy McCall ↶ 759
        at_ms: u64,
eae59556 Andy McCall ↶ 760
    ) -> bool {
84b9e9ac Andy McCall ↶ 761
        if p.is_revoked(signer) {
762
            return false;
763
        }
eae59556 Andy McCall ↶ 764
        let Some(rule) = p.rule_for(reference) else {
765
            return false;
766
        };
84b9e9ac Andy McCall ↶ 767
        rule.keys.iter().any(|k| {
768
            !p.is_revoked(k)
769
                && p.identity(k)
770
                    .cloned()
771
                    .or_else(|| self.store.get_identity(k).ok())
7ba155f2 Andy McCall ↶ 772
                    .is_some_and(|id| id.delegates_to(signer, Scope::REFS, at_ms))
84b9e9ac Andy McCall ↶ 773
        })
774
    }
775
e5d9b051 Andy McCall ↶ 776
    /// A lasting attestation (a tag) signed by a key the policy has since
777
    /// rotated away from (spec §14.2) is authorized as the key that
778
    /// succeeded it: a key the rule lists reaches the signer through
779
    /// `previous` links, and the attestation is dated (`at_ms`) before the
780
    /// signer's successor was made, so a superseded key makes nothing new.
781
    /// Revoked keys authorize nothing.
782
    pub(crate) fn predecessor_authorized(
783
        &self,
784
        p: &crate::policy::Policy,
785
        signer: &KeyId,
786
        reference: &str,
787
        at_ms: u64,
788
    ) -> bool {
789
        if p.is_revoked(signer) {
790
            return false;
791
        }
792
        let Some(rule) = p.rule_for(reference) else {
793
            return false;
794
        };
795
        let identity = |k: &KeyId| {
796
            p.identity(k)
797
                .cloned()
798
                .or_else(|| self.store.get_identity(k).ok())
799
        };
800
        rule.keys.iter().any(|k| {
801
            if p.is_revoked(k) {
802
                return false;
803
            }
804
            let mut cur = identity(k);
805
            for _ in 0..32 {
806
                let Some(id) = cur else {
807
                    return false;
808
                };
809
                match id.previous {
810
                    Some(prev) if prev == *signer => return at_ms < id.created,
811
                    Some(prev) if !p.is_revoked(&prev) => cur = identity(&prev),
812
                    _ => return false,
813
                }
814
            }
815
            false
816
        })
817
    }
818
75167672 Andy McCall 819
    fn check_ref_attest(
d36ba2f7 Andy McCall ↶ 820
        &self,
821
        sa: &SignedAttest,
822
        name: &str,
823
        old_state: &ObjectRef,
824
        new_state: &ObjectRef,
825
        last_serial: Option<u64>,
826
        allow_equal: bool,
827
    ) -> std::result::Result<u64, String> {
828
        let serial =
829
            self.check_ref_attest_body(sa, name, old_state, new_state, last_serial, allow_equal)?;
830
        self.verify_attest_signature(sa)?;
831
        Ok(serial)
832
    }
833
834
    /// What [`Repo::check_ref_attest`] checks short of the signature: the
835
    /// attestation describes this update of this repository with a serial
836
    /// past the last seen.
837
    fn check_ref_attest_body(
75167672 Andy McCall 838
        &self,
839
        sa: &SignedAttest,
840
        name: &str,
841
        old_state: &ObjectRef,
842
        new_state: &ObjectRef,
843
        last_serial: Option<u64>,
844
        allow_equal: bool,
845
    ) -> std::result::Result<u64, String> {
846
        let AttestBody::Ref {
847
            name: n,
848
            old_state: o,
849
            new_state: nw,
850
            serial,
851
        } = &sa.attest.body
852
        else {
98286406 Andy McCall ↶ 853
            return Err("attestation is not a ref attestation".into());
854
        };
855
        if n != name || o != old_state || nw != new_state {
856
            return Err("attestation does not describe this update".into());
857
        }
858
        if sa.attest.repo != self.id().map_err(|e| e.to_string())? {
859
            return Err("attestation is for a different repository".into());
860
        }
861
        if let Some(last) = last_serial {
862
            if *serial < last || (*serial == last && !allow_equal) {
75167672 Andy McCall 863
                return Err(format!(
864
                    "attestation serial {serial} is not greater than the last seen {last}"
865
                ));
98286406 Andy McCall ↶ 866
            }
867
        }
d36ba2f7 Andy McCall ↶ 868
        Ok(*serial)
869
    }
870
871
    /// An attestation's first signature verifies with its signer's
872
    /// identity, which must be stored here.
873
    fn verify_attest_signature(&self, sa: &SignedAttest) -> std::result::Result<(), String> {
98286406 Andy McCall ↶ 874
        let r = sa.attest.hash();
875
        let sig = sa.sigs.first().ok_or("attestation carries no signature")?;
75167672 Andy McCall 876
        let id = self
877
            .store
878
            .get_identity(&sig.key)
879
            .map_err(|_| format!("unknown key {}", sig.key.short(12)))?;
d36ba2f7 Andy McCall ↶ 880
        verify_signature(&id, sig, Domain::Attest, &r).map_err(|e| e.to_string())
98286406 Andy McCall ↶ 881
    }
882
83e0677b Andy McCall ↶ 883
    /// Accept a push (spec §16.4). Without a trust policy every repository
98286406 Andy McCall ↶ 884
    /// is unverified, so unsigned changes and unattested updates are accepted;
83e0677b Andy McCall ↶ 885
    /// attestations that are present must still be valid. With a policy,
886
    /// every change and update is checked against it. A newer policy in the
887
    /// push is applied first when it is signed by the current roots.
75167672 Andy McCall 888
    pub fn receive_push(
889
        &self,
890
        bundle: Bundle,
891
        updates: Vec<RefUpdate>,
892
        policy: Option<SignedPolicy>,
893
        tags: Vec<crate::protocol::TagRef>,
894
    ) -> Result<Vec<RefResult>> {
119bbf50 Andy McCall ↶ 895
        Ok(self
896
            .receive_push_report(bundle, updates, policy, tags)?
897
            .results)
7aa2dc93 Andy McCall ↶ 898
    }
899
900
    /// `receive_push` with what a host wants to log: each accepted
901
    /// reference's old and new state and heads, and the changes this push
902
    /// stored (hub spec §6.12).
903
    pub fn receive_push_report(
904
        &self,
905
        bundle: Bundle,
906
        updates: Vec<RefUpdate>,
907
        policy: Option<SignedPolicy>,
908
        tags: Vec<crate::protocol::TagRef>,
909
    ) -> Result<PushReport> {
767ac6cd Andy McCall 910
        // Updates are checked against the head as it is now, not as this
911
        // handle first read it (oxygenhub#249): another push may have
912
        // landed since.
913
        self.refresh()?;
7aa2dc93 Andy McCall ↶ 914
        let mut report = PushReport::default();
915
        let results = &mut report.results;
83e0677b Andy McCall ↶ 916
        if let Some(sp) = policy {
28fc23d7 Andy McCall ↶ 917
            match self.adopt_policy_chain(&bundle.policies, &sp, "push received") {
83e0677b Andy McCall ↶ 918
                Ok(()) => {
75167672 Andy McCall 919
                    results.push(RefResult {
920
                        name: "policy".into(),
921
                        ok: true,
922
                        reason: format!("serial {}", sp.policy.serial),
923
                    });
83e0677b Andy McCall ↶ 924
                }
925
                Err(e) => {
75167672 Andy McCall 926
                    results.push(RefResult {
927
                        name: "policy".into(),
928
                        ok: false,
929
                        reason: e,
930
                    });
83e0677b Andy McCall ↶ 931
                    for u in updates {
75167672 Andy McCall 932
                        results.push(RefResult {
933
                            name: u.name,
934
                            ok: false,
935
                            reason: "policy update was rejected".into(),
936
                        });
83e0677b Andy McCall ↶ 937
                    }
7aa2dc93 Andy McCall ↶ 938
                    return Ok(report);
83e0677b Andy McCall ↶ 939
                }
940
            }
941
        }
942
        let policy = self.current_policy()?.map(|(_, sp)| sp.policy);
cb8b8082 Andy McCall ↶ 943
        // A push of objects alone (one batch of a long history, oxygen#95)
944
        // names no reference: its outcome is reported as `objects`.
945
        let objects_only = updates.is_empty() && tags.is_empty();
7aa2dc93 Andy McCall ↶ 946
        match self.import_bundle(&bundle, policy.is_some()) {
cb8b8082 Andy McCall ↶ 947
            Ok(imported) => {
948
                if objects_only {
949
                    results.push(RefResult {
950
                        name: OBJECTS_RESULT.into(),
951
                        ok: true,
952
                        reason: format!("{} changes stored", imported.len()),
953
                    });
954
                }
955
                report.imported = imported;
956
            }
7aa2dc93 Andy McCall ↶ 957
            Err(e) => {
cb8b8082 Andy McCall ↶ 958
                if objects_only {
959
                    results.push(RefResult {
960
                        name: OBJECTS_RESULT.into(),
961
                        ok: false,
962
                        reason: e.to_string(),
963
                    });
964
                }
7aa2dc93 Andy McCall ↶ 965
                for u in updates {
966
                    results.push(RefResult {
967
                        name: u.name,
968
                        ok: false,
969
                        reason: e.to_string(),
970
                    });
971
                }
972
                return Ok(report);
83e0677b Andy McCall ↶ 973
            }
974
        }
98286406 Andy McCall ↶ 975
        let (op_ref, op) = self.head_op()?;
976
        let mut view = op.view.clone();
977
        let mut names = Vec::new();
978
        for u in updates {
83e0677b Andy McCall ↶ 979
            let outcome = self.apply_update(&mut view, &u, policy.as_ref());
98286406 Andy McCall ↶ 980
            let (ok, reason) = match outcome {
981
                Ok(()) => (true, String::new()),
982
                Err(e) => (false, e),
983
            };
984
            if ok {
985
                names.push(u.name.clone());
119bbf50 Andy McCall ↶ 986
                let heads = self
987
                    .store
988
                    .get_state(&u.new_state)
989
                    .map(|s| s.heads)
990
                    .unwrap_or_default();
991
                report.updated.push(RefUpdated {
992
                    name: u.name.clone(),
993
                    old_state: u.old_state,
994
                    new_state: u.new_state,
995
                    heads,
996
                });
98286406 Andy McCall ↶ 997
            }
75167672 Andy McCall 998
            results.push(RefResult {
999
                name: u.name,
1000
                ok,
1001
                reason,
1002
            });
98286406 Andy McCall ↶ 1003
        }
132b884b Andy McCall ↶ 1004
        for t in &tags {
1005
            let name = format!("tags/{}", t.name);
b63fc7a7 Andy McCall ↶ 1006
            if t.is_withdrawal() {
1007
                match self.adopt_withdrawal(&mut view, t, policy.as_ref(), &mut TagCheck::Received)
1008
                {
1009
                    Ok(removed) => {
1010
                        // A withdrawal is an update to nothing, as a
1011
                        // bookmark's deletion is.
1012
                        if removed {
1013
                            report.updated.push(RefUpdated {
1014
                                name: name.clone(),
1015
                                old_state: t.state,
1016
                                new_state: ObjectRef::ZERO,
1017
                                heads: vec![],
1018
                            });
1019
                        }
1020
                        names.push(name.clone());
1021
                        results.push(RefResult {
1022
                            name,
1023
                            ok: true,
1024
                            reason: if removed {
1025
                                "withdrawn"
1026
                            } else {
1027
                                "already withdrawn"
1028
                            }
1029
                            .into(),
1030
                        });
1031
                    }
1032
                    Err(e) => results.push(RefResult {
1033
                        name,
1034
                        ok: false,
1035
                        reason: e.reason,
1036
                    }),
1037
                }
1038
                continue;
1039
            }
b2ef93bb Andy McCall ↶ 1040
            let new = !view.tags.contains_key(&t.name);
1ec6fdbc Andy McCall ↶ 1041
            match self.adopt_tag(&mut view, t, policy.as_ref(), &mut TagCheck::Received) {
132b884b Andy McCall ↶ 1042
                Ok(()) => {
b2ef93bb Andy McCall ↶ 1043
                    // A tag new here is an update too, so a hub can start
1044
                    // its pipeline's tag jobs (oxygenflow#24); one it
1045
                    // already had, sent again, is not.
1046
                    if new {
1047
                        report.updated.push(RefUpdated {
1048
                            name: name.clone(),
1049
                            old_state: ObjectRef::ZERO,
1050
                            new_state: t.state,
1051
                            heads: t.heads.clone(),
1052
                        });
1053
                    }
132b884b Andy McCall ↶ 1054
                    names.push(name.clone());
75167672 Andy McCall 1055
                    results.push(RefResult {
1056
                        name,
1057
                        ok: true,
1058
                        reason: String::new(),
1059
                    });
132b884b Andy McCall ↶ 1060
                }
75167672 Andy McCall 1061
                Err(e) => results.push(RefResult {
1062
                    name,
1063
                    ok: false,
1ec6fdbc Andy McCall ↶ 1064
                    reason: e.reason,
75167672 Andy McCall 1065
                }),
132b884b Andy McCall ↶ 1066
            }
1067
        }
98286406 Andy McCall ↶ 1068
        if !names.is_empty() {
75167672 Andy McCall 1069
            self.record_op(
1070
                &[op_ref],
1071
                &format!("push received: {}", names.join(" ")),
1072
                view,
1073
            )?;
98286406 Andy McCall ↶ 1074
        }
7aa2dc93 Andy McCall ↶ 1075
        Ok(report)
98286406 Andy McCall ↶ 1076
    }
1077
be451356 Andy McCall ↶ 1078
    /// Check an update's attestation against the latest one for its
1079
    /// reference and store it as the new latest. Failures say what was
1080
    /// being done, for which reference, so a refused push can be diagnosed
1081
    /// from its reason alone.
1082
    fn record_update_attest(
1083
        &self,
1084
        u: &RefUpdate,
1085
        sa: &SignedAttest,
1086
    ) -> std::result::Result<(), String> {
1087
        let name = &u.name;
1088
        let last = self
1089
            .latest_ref_attest(name)
1090
            .map_err(|e| format!("reading the last attestation of {name:?}: {e}"))?
1091
            .and_then(|(_, a)| match a.attest.body {
1092
                AttestBody::Ref { serial, .. } => Some(serial),
1093
                _ => None,
1094
            });
1095
        self.check_ref_attest(sa, name, &u.old_state, &u.new_state, last, false)?;
1096
        let store = |e: &dyn std::fmt::Display| format!("storing the attestation of {name:?}: {e}");
1097
        let r = self.store.put_attest(&sa.attest).map_err(|e| store(&e))?;
1098
        for sig in &sa.sigs {
1099
            self.store.put_signature(&r, sig).map_err(|e| store(&e))?;
f53cbe7a Andy McCall ↶ 1100
        }
1101
        self.set_ref_attest(name, &r).map_err(|e| store(&e))?;
1102
        // Signed by a short-lived key: a host records when it accepted it.
1103
        let certified = sa
1104
            .sigs
1105
            .first()
1106
            .is_some_and(|s| self.store.certificate_of(&s.key).ok().flatten().is_some());
1107
        if certified {
1108
            self.record_acceptance(&r).map_err(|e| store(&e))?;
be451356 Andy McCall ↶ 1109
        }
f53cbe7a Andy McCall ↶ 1110
        Ok(())
be451356 Andy McCall ↶ 1111
    }
1112
f53cbe7a Andy McCall ↶ 1113
    /// Record that this host accepted attestation `r` now, signed with its
1114
    /// acceptor key (spec §14.7); nothing without one.
1115
    pub(crate) fn record_acceptance(&self, r: &ObjectRef) -> Result<()> {
1116
        let Some(acceptor) = &self.acceptor else {
1117
            return Ok(());
1118
        };
1119
        let (id, sk) = (&acceptor.0, &acceptor.1);
1120
        let attest = Attest {
1121
            repo: self.id()?,
1122
            hlc: Hlc::now(),
1123
            body: AttestBody::Accepted { attest: *r },
1124
        };
1125
        let sig = sk.sign(Domain::Attest, &id.hash(), &attest.hash());
1126
        self.store.put_identity(id)?;
1127
        let a = self.store.put_attest(&attest)?;
1128
        self.store.put_signature(&a, &sig)?;
1129
        self.store.index_add_pub("accepted", r, &a)?;
1130
        Ok(())
1131
    }
1132
1133
    /// The host's acceptance of attestation `r`, if one was recorded.
1134
    pub fn acceptance_of(&self, r: &ObjectRef) -> Result<Option<SignedAttest>> {
1135
        let Some(a) = self.store.index_get_pub("accepted", r)?.into_iter().next() else {
1136
            return Ok(None);
1137
        };
1138
        Ok(Some(SignedAttest {
1139
            attest: self.store.get_attest(&a)?,
1140
            sigs: self.store.signatures(&a)?,
1141
        }))
1142
    }
1143
1144
    /// When a certified signer's attestation `sa` is checked on a fetch: the
1145
    /// host's acceptance time, if `accepted` is signed by the issuer of the
1146
    /// signer's certificate and accepts `sa`; else the attestation's own
1147
    /// time, which the signer chose.
1148
    pub(crate) fn certified_time(
1149
        &self,
1150
        signer: &KeyId,
1151
        sa: &SignedAttest,
1152
        accepted: Option<&SignedAttest>,
1153
    ) -> u64 {
1ec6fdbc Andy McCall ↶ 1154
        self.issuer_acceptance(signer, sa, accepted)
1155
            .unwrap_or(sa.attest.hlc.wall_ms)
1156
    }
1157
1158
    /// The time of `accepted` when it accepts exactly `sa` and is signed by
1159
    /// the issuer of `signer`'s kept certificate (spec §14.7); else none.
1160
    pub(crate) fn issuer_acceptance(
1161
        &self,
1162
        signer: &KeyId,
1163
        sa: &SignedAttest,
1164
        accepted: Option<&SignedAttest>,
1165
    ) -> Option<u64> {
1166
        let acc = accepted?;
1167
        let c = self.store.certificate_of(signer).ok().flatten()?;
1168
        let cert = self.store.get_certificate(&c).ok()?;
f53cbe7a Andy McCall ↶ 1169
        let AttestBody::Accepted { attest } = &acc.attest.body else {
1ec6fdbc Andy McCall ↶ 1170
            return None;
f53cbe7a Andy McCall ↶ 1171
        };
1ec6fdbc Andy McCall ↶ 1172
        let sig = acc.sigs.iter().find(|s| s.key == cert.cert.issuer)?;
1173
        let issuer = self.store.get_identity(&cert.cert.issuer).ok()?;
1174
        (*attest == sa.attest.hash()
1175
            && verify_signature(&issuer, sig, Domain::Attest, &acc.attest.hash()).is_ok())
1176
        .then_some(acc.attest.hlc.wall_ms)
1177
    }
1178
1179
    /// Keep an acceptance this repository was given for attestation `r`
1180
    /// (one [`Repo::issuer_acceptance`] took), so it is served again with
1181
    /// what it accepts.
1182
    pub(crate) fn keep_acceptance(&self, r: &ObjectRef, acc: &SignedAttest) -> Result<()> {
1183
        let a = self.store.put_attest(&acc.attest)?;
1184
        for sig in &acc.sigs {
1185
            self.store.put_signature(&a, sig)?;
f53cbe7a Andy McCall ↶ 1186
        }
1ec6fdbc Andy McCall ↶ 1187
        self.store.index_add_pub("accepted", r, &a)?;
1188
        Ok(())
f53cbe7a Andy McCall ↶ 1189
    }
1190
028545df Andy McCall ↶ 1191
    /// A fetched bookmark's attestation by a signer the policy does not
1192
    /// list itself: a delegate or a short-lived key, whose authority
1193
    /// depends on when it signed (spec §14.5, §14.8). Checked at the
1194
    /// attestation's integrated time when the host's log gives one; else
1195
    /// as before, at the host's acceptance for a certified signer, when it
1196
    /// countersigned one, or at the attestation's own time, and said so
1197
    /// when the host keeps a log. Failing at the integrated time but not
1198
    /// as before is a note while logs are new; a large gap between the
1199
    /// claimed and the integrated time is a note.
1200
    #[allow(clippy::too_many_arguments)]
1201
    pub(crate) fn authorize_in_time(
1202
        &self,
1203
        p: &crate::policy::Policy,
1204
        remote_name: &str,
1205
        r: &crate::protocol::RemoteRef,
1206
        sa: &SignedAttest,
1207
        signer: &KeyId,
1208
        log: &super::transparency::LogCheck,
1209
        notes: &mut Vec<String>,
1210
    ) -> Result<()> {
1211
        use super::transparency::{gap_note, judge, Timed};
1212
        let own = sa.attest.hlc.wall_ms;
1213
        let at = self.certified_time(signer, sa, r.accepted.as_ref());
1214
        let time = log.time_of(&r.name);
1215
        let check = |t_delegate: u64, t_cert: u64| {
1216
            self.delegate_authorized(p, signer, &r.name, t_delegate)
1217
                || self.certified_authorized(p, signer, &r.name, Scope::REFS, t_cert)
1218
        };
1219
        let before = if at != own {
1220
            "the host's acceptance"
1221
        } else {
1222
            "the time it claims"
1223
        };
1224
        let what = format!("{remote_name}/{}", r.name);
1225
        match judge(time.as_ref().ok().map(|i| i.ms()), |t| check(t, t), || check(own, at)) {
1226
            Timed::Refused => {
1227
                return fail(format!(
1228
                    "{what}: attested by {}, which the policy does not authorize for this bookmark; a policy owner can grant it with `oxy policy grant`",
1229
                    signer.short(12)
1230
                ))
1231
            }
1232
            Timed::Integrated => {}
1233
            Timed::Disagrees => notes.push(format!(
1234
                "{what}: {}'s authority had lapsed when witness {} first saw its attestation, though not at {before}; accepted while transparency logs are new",
1235
                signer.short(12),
1236
                time.as_ref().map(|i| i.witness.as_str()).unwrap_or("?")
1237
            )),
1238
            Timed::Fallback if log.kept => notes.push(format!(
1239
                "{what}: {} checked at {before}, not at a witnessed time: {}",
1240
                signer.short(12),
1241
                time.as_ref().err().map(String::as_str).unwrap_or("none")
1242
            )),
1243
            Timed::Fallback => {}
1244
        }
1245
        if let Some(n) = time.ok().and_then(|i| gap_note(own, i)) {
1246
            notes.push(format!("{what}'s attestation {n}"));
1247
        }
1248
        Ok(())
1249
    }
1250
f8c69576 Andy McCall ↶ 1251
    /// A deletion (spec §16.4): `new_state` is zero. The bookmark must exist
8c30d814 Andy McCall ↶ 1252
    /// at `old_state`, is never the bookmark a non-bare repository has
1253
    /// checked out (a bare one's default is its host's to protect, see
1254
    /// `oxy_server::Backend::default_bookmark`), and under a policy needs
1255
    /// an attestation by a key that may update it.
f8c69576 Andy McCall ↶ 1256
    /// The deletion's attestation stays the bookmark's latest, so an older
1257
    /// one cannot be replayed to bring it back.
1258
    fn apply_deletion(
1259
        &self,
1260
        view: &mut View,
1261
        u: &RefUpdate,
1262
        policy: Option<&crate::policy::Policy>,
1263
    ) -> std::result::Result<(), String> {
1264
        let Some(current) = view.bookmarks.get(&u.name).copied() else {
1265
            return Err(format!("no bookmark {:?} to delete", u.name));
1266
        };
1267
        if current != u.old_state {
1268
            return Err("stale: the remote bookmark has moved; pull first".into());
1269
        }
8c30d814 Andy McCall ↶ 1270
        if !self.is_bare() && u.name == self.config.bookmark {
f8c69576 Andy McCall ↶ 1271
            return Err(format!(
8c30d814 Andy McCall ↶ 1272
                "{:?} is this repository's default bookmark (checked out there) and cannot be deleted",
f8c69576 Andy McCall ↶ 1273
                u.name
1274
            ));
1275
        }
1276
        if let Some(p) = policy {
1277
            let Some(sa) = &u.attestation else {
1278
                return Err(format!(
1279
                    "deleting {:?} requires a signed reference attestation under this repository's policy",
1280
                    u.name
1281
                ));
1282
            };
1283
            let signer = sa
1284
                .sigs
1285
                .first()
1286
                .map(|s| s.key)
1287
                .ok_or("attestation carries no signature")?;
f0a517d2 Andy McCall ↶ 1288
            // Received now: the receiver's own clock is the trusted time.
1289
            let now = Hlc::now().wall_ms;
7ba155f2 Andy McCall ↶ 1290
            if !p.authorizes(&signer, &u.name)
1291
                && !self.delegate_authorized(p, &signer, &u.name, sa.attest.hlc.wall_ms)
f0a517d2 Andy McCall ↶ 1292
                && !self.certified_authorized(p, &signer, &u.name, Scope::REFS, now)
7ba155f2 Andy McCall ↶ 1293
            {
f8c69576 Andy McCall ↶ 1294
                return Err(format!(
1295
                    "key {} may not delete {:?} under this repository's policy",
1296
                    signer.short(12),
1297
                    u.name
1298
                ));
1299
            }
1300
        }
1301
        if let Some(sa) = &u.attestation {
be451356 Andy McCall ↶ 1302
            self.record_update_attest(u, sa)?;
f8c69576 Andy McCall ↶ 1303
        }
1304
        view.bookmarks.remove(&u.name);
1305
        Ok(())
1306
    }
1307
75167672 Andy McCall 1308
    fn apply_update(
1309
        &self,
1310
        view: &mut View,
1311
        u: &RefUpdate,
1312
        policy: Option<&crate::policy::Policy>,
1313
    ) -> std::result::Result<(), String> {
83e0677b Andy McCall ↶ 1314
        if u.name == "policy" {
75167672 Andy McCall 1315
            return Err(
1316
                "the policy reference is updated by sending a policy, not a bookmark update".into(),
1317
            );
83e0677b Andy McCall ↶ 1318
        }
75167672 Andy McCall 1319
        let empty = self
1320
            .store
1321
            .put_state(&StateObject::new(vec![]))
1322
            .map_err(|e| e.to_string())?;
f8c69576 Andy McCall ↶ 1323
        if u.new_state.is_zero() {
1324
            return self.apply_deletion(view, u, policy);
1325
        }
98286406 Andy McCall ↶ 1326
        let current = view.bookmarks.get(&u.name).copied().unwrap_or(empty);
1327
        if current != u.old_state {
1328
            return Err("stale: the remote bookmark has moved; pull first".into());
1329
        }
75167672 Andy McCall 1330
        let new_state = self
1331
            .store
1332
            .get_state(&u.new_state)
1333
            .map_err(|_| "new state object is missing".to_string())?;
98286406 Andy McCall ↶ 1334
        for h in &new_state.heads {
1335
            if !self.store.has(h) {
1336
                return Err(format!("head {} is missing", h.short(12)));
1337
            }
1338
        }
1339
        if !u.force {
75167672 Andy McCall 1340
            let old_heads = self
1341
                .store
1342
                .get_state(&current)
1343
                .map_err(|e| e.to_string())?
1344
                .heads;
1345
            let closure = self
1346
                .closure_in_store(&new_state.heads)
1347
                .map_err(|e| e.to_string())?;
98286406 Andy McCall ↶ 1348
            if let Some(h) = old_heads.iter().find(|h| !closure.contains(h)) {
75167672 Andy McCall 1349
                return Err(format!(
1350
                    "not a fast-forward: current head {} would be dropped (use --force)",
1351
                    h.short(12)
1352
                ));
98286406 Andy McCall ↶ 1353
            }
1354
        }
83e0677b Andy McCall ↶ 1355
        if let Some(p) = policy {
1356
            let Some(sa) = &u.attestation else {
75167672 Andy McCall 1357
                return Err(format!(
1358
                    "{:?} requires a signed reference attestation under this repository's policy",
1359
                    u.name
1360
                ));
83e0677b Andy McCall ↶ 1361
            };
75167672 Andy McCall 1362
            let signer = sa
1363
                .sigs
1364
                .first()
1365
                .map(|s| s.key)
1366
                .ok_or("attestation carries no signature")?;
f0a517d2 Andy McCall ↶ 1367
            // Received now: the receiver's own clock is the trusted time.
1368
            let now = Hlc::now().wall_ms;
7ba155f2 Andy McCall ↶ 1369
            if !p.authorizes(&signer, &u.name)
1370
                && !self.delegate_authorized(p, &signer, &u.name, sa.attest.hlc.wall_ms)
f0a517d2 Andy McCall ↶ 1371
                && !self.certified_authorized(p, &signer, &u.name, Scope::REFS, now)
7ba155f2 Andy McCall ↶ 1372
            {
75167672 Andy McCall 1373
                return Err(format!(
1374
                    "key {} may not update {:?} under this repository's policy",
1375
                    signer.short(12),
1376
                    u.name
1377
                ));
83e0677b Andy McCall ↶ 1378
            }
75167672 Andy McCall 1379
            let old_heads = self
1380
                .store
1381
                .get_state(&current)
1382
                .map_err(|e| e.to_string())?
1383
                .heads;
83e0677b Andy McCall ↶ 1384
            self.state_allowed(&old_heads, &new_state.heads, p, &u.name)?;
1385
        }
98286406 Andy McCall ↶ 1386
        if let Some(sa) = &u.attestation {
be451356 Andy McCall ↶ 1387
            self.record_update_attest(u, sa)?;
98286406 Andy McCall ↶ 1388
        }
1389
        view.bookmarks.insert(u.name.clone(), u.new_state);
1390
        Ok(())
1391
    }
1392
1393
    // ------------------------------------------------------------ client side
1394
1395
    fn remote_url(&self, name: &str) -> Result<String> {
75167672 Andy McCall 1396
        self.config.remotes.get(name).cloned().ok_or_else(|| {
42ccf504 Andy McCall ↶ 1397
            let names: Vec<&str> = self.config.remotes.keys().map(String::as_str).collect();
1398
            RepoError::Msg(match names.as_slice() {
1399
                [] => format!("no remote named {name:?}; add one with `oxy remote add`"),
1400
                _ if name == "origin" && names.len() > 1 && self.config.default_remote.is_none() => format!(
1401
                    "this repository has no remote named origin, and several others ({}): name one (`oxy pull {}`), or make one the default with `oxy config core.remote {}`",
1402
                    names.join(", "),
1403
                    names[0],
1404
                    names[0]
1405
                ),
1406
                _ => format!(
1407
                    "no remote named {name:?}; this repository has {}",
1408
                    names.join(", ")
1409
                ),
1410
            })
75167672 Andy McCall 1411
        })
98286406 Andy McCall ↶ 1412
    }
1413
75167672 Andy McCall 1414
    /// Refuse a remote that is another repository. An **unbound** remote
1415
    /// (id all zero: a hosted repository nobody has pushed to) is accepted
1416
    /// for pushes, which bind it to this repository's id (spec §16.1).
05e71ed6 Andy McCall ↶ 1417
    fn check_remote_id(
1418
        &self,
1419
        remote: &dyn crate::protocol::Remote,
1420
        remote_name: &str,
1421
        allow_unbound: bool,
1422
    ) -> Result<()> {
75167672 Andy McCall 1423
        let theirs = remote.repo_id()?;
1424
        if theirs == self.id()? {
1425
            return Ok(());
1426
        }
1427
        if theirs == crate::protocol::UNBOUND_ID {
1428
            if allow_unbound {
1429
                return Ok(());
1430
            }
05e71ed6 Andy McCall ↶ 1431
            return fail(format!(
1432
                "remote {remote_name:?} is empty and unbound: push to it first"
1433
            ));
75167672 Andy McCall 1434
        }
05e71ed6 Andy McCall ↶ 1435
        fail(format!(
3e5772cc Andy McCall ↶ 1436
            "remote {remote_name:?} is a different repository (repository IDs differ); check its URL with `oxy remote list`"
05e71ed6 Andy McCall ↶ 1437
        ))
75167672 Andy McCall 1438
    }
1439
ff1f62fe Andy McCall ↶ 1440
    /// Credentials and trust for a remote: `remote.<name>.token` and
1441
    /// `remote.<name>.ca`, else the defaults for its URL
1442
    /// ([`Repo::fill_credentials`]).
3e5772cc Andy McCall ↶ 1443
    pub(crate) fn remote_options(&self, name: &str) -> RemoteOptions {
ff1f62fe Andy McCall ↶ 1444
        let url = self.config.remotes.get(name).cloned().unwrap_or_default();
1445
        let mut options = RemoteOptions {
75167672 Andy McCall 1446
            repo_id: self.id().ok(),
ff1f62fe Andy McCall ↶ 1447
            token: self.config.remote_tokens.get(name).cloned(),
1448
            ca: self.config.remote_cas.get(name).map(PathBuf::from),
9f9e72b9 Andy McCall ↶ 1449
            cache_dir: Some(self.oxy.join("remotes").join(name).join("strata")),
68a92484 Andy McCall ↶ 1450
            ..RemoteOptions::default()
ff1f62fe Andy McCall ↶ 1451
        };
1452
        self.fill_credentials(&url, &mut options);
1453
        options
1454
    }
1455
1456
    /// The user's credential store (`credentials.toml` beside the user
1457
    /// configuration file).
1458
    pub fn credentials(&self) -> crate::credentials::Credentials {
1459
        crate::credentials::Credentials::at(
1460
            self.user_config
1461
                .parent()
1462
                .map(Path::to_path_buf)
1463
                .unwrap_or_else(default_home),
1464
        )
1465
    }
1466
1467
    /// Fill in what `options` lacks for `url`: `$OXY_TOKEN` and `$OXY_CA`,
1468
    /// then the user's credential store for its hub, then a remote on the
1469
    /// same hub in an enclosing checkout (a subrepo borrows its parent's).
68a92484 Andy McCall ↶ 1470
    /// When the token is not this remote's own or the environment's, the
1471
    /// other tokens enclosing checkouts hold for the hub are kept to try
1472
    /// if it is refused: a stale stored login must not hide a parent's
1473
    /// good token (oxygen#112).
ff1f62fe Andy McCall ↶ 1474
    fn fill_credentials(&self, url: &str, options: &mut RemoteOptions) {
1475
        if options.token.is_some() && options.ca.is_some() {
1476
            return;
1477
        }
569802eb Andy McCall ↶ 1478
        let own = options.token.is_some();
1479
        let env = std::env::var("OXY_TOKEN").is_ok_and(|t| !t.is_empty());
1480
        let explicit = own || env;
ff1f62fe Andy McCall ↶ 1481
        let stored = self.credentials().options_for(url);
0588de69 Andy McCall ↶ 1482
        // A checkout's own token refused as expired or unknown (401) falls
1483
        // back to this computer's sign-in for the hub, never to another
1484
        // checkout's, and never when it is refused for permission (#155).
569802eb Andy McCall ↶ 1485
        if own && !env {
1486
            if let Some(t) = stored
1487
                .token
1488
                .clone()
1489
                .filter(|t| options.token.as_ref() != Some(t))
1490
            {
1491
                options.fallback_tokens.push(t);
0588de69 Andy McCall ↶ 1492
                options.fallback_on_401_only = true;
569802eb Andy McCall ↶ 1493
            }
1494
        }
ff1f62fe Andy McCall ↶ 1495
        options.token = options.token.take().or(stored.token);
1496
        options.ca = options.ca.take().or(stored.ca);
68a92484 Andy McCall ↶ 1497
        let enclosing = self.enclosing_credentials(url);
1498
        if options.token.is_none() {
1499
            options.token = enclosing.iter().find_map(|(t, _)| t.clone());
1500
        }
1501
        if options.ca.is_none() {
1502
            options.ca = enclosing.iter().find_map(|(_, c)| c.clone());
1503
        }
1504
        if !explicit {
1505
            for t in enclosing.into_iter().filter_map(|(t, _)| t) {
1506
                if options.token.as_ref() != Some(&t) && !options.fallback_tokens.contains(&t) {
1507
                    options.fallback_tokens.push(t);
1508
                }
1509
            }
569802eb Andy McCall ↶ 1510
        }
1511
    }
1512
1513
    /// What to do when `remote`'s own token was refused (`message` says
1514
    /// HTTP 401) and so was this computer's sign-in for its hub, which it
1515
    /// falls back to (oxygen#155).
1516
    pub fn stale_token_note(&self, remote: &str, message: &str) -> Option<String> {
1517
        if !message.contains("HTTP 401") {
1518
            return None;
2b5ee371 Andy McCall ↶ 1519
        }
569802eb Andy McCall ↶ 1520
        let own = self.config.remote_tokens.get(remote)?;
1521
        let url = self.config.remotes.get(remote)?;
1522
        let stored = self.credentials().options_for(url).token?;
1523
        (stored != *own).then(|| {
1524
            format!(
1525
                "the hub refused both this checkout's own token for {remote} (remote.{remote}.token) and your sign-in: \
1526
                 `oxy auth login HUB`, then `oxy config --unset remote.{remote}.token` so the sign-in is used"
1527
            )
1528
        })
68a92484 Andy McCall ↶ 1529
    }
1530
1531
    /// The tokens and CAs of remotes on `url`'s hub in enclosing
1532
    /// checkouts, nearest first.
1533
    fn enclosing_credentials(&self, url: &str) -> Vec<(Option<String>, Option<PathBuf>)> {
1534
        let mut out = Vec::new();
ff1f62fe Andy McCall ↶ 1535
        let Some(hub) = crate::credentials::hub_of(url) else {
68a92484 Andy McCall ↶ 1536
            return out;
ff1f62fe Andy McCall ↶ 1537
        };
1538
        let mut at = self.root.parent();
1539
        while let Some(dir) = at {
1540
            let Ok(cfg) = read_config_file(&dir.join(OXY_DIR).join("config")) else {
1541
                break;
1542
            };
1543
            for (key, remote_url) in &cfg {
1544
                let Some(name) = key
1545
                    .strip_prefix("remote.")
1546
                    .and_then(|k| k.strip_suffix(".url"))
1547
                else {
1548
                    continue;
1549
                };
1550
                if crate::credentials::hub_of(remote_url).as_deref() != Some(hub.as_str()) {
1551
                    continue;
1552
                }
68a92484 Andy McCall ↶ 1553
                out.push((
1554
                    cfg.get(&format!("remote.{name}.token")).cloned(),
1555
                    cfg.get(&format!("remote.{name}.ca")).map(PathBuf::from),
1556
                ));
ff1f62fe Andy McCall ↶ 1557
            }
1558
            at = dir.parent();
1559
        }
68a92484 Andy McCall ↶ 1560
        out
98286406 Andy McCall ↶ 1561
    }
1562
d82e4409 Andy McCall ↶ 1563
    /// Push seal attestations on their own (hub spec §7.4): a bundle with
1564
    /// no changes and no reference updates, which the receiver stores.
1565
    pub fn push_seals(&self, remote_name: &str, seals: &[ObjectRef]) -> Result<()> {
1566
        let remote = self.open_named_remote(remote_name)?;
1567
        self.check_remote_id(remote.as_ref(), remote_name, false)?;
1568
        let mut bundle = Bundle::default();
1569
        for r in seals {
1570
            let attest = self.store.get_attest(r)?;
1571
            let sigs = self.store.signatures(r)?;
4213c41d Andy McCall ↶ 1572
            bundle
1573
                .attests
1574
                .push(crate::protocol::SignedAttest { attest, sigs });
d82e4409 Andy McCall ↶ 1575
        }
1576
        let results = remote.push(bundle, vec![], None, vec![])?;
1577
        if let Some(r) = results.iter().find(|r| !r.ok) {
1578
            return fail(format!("{}: {}", r.name, r.reason));
1579
        }
1580
        Ok(())
1581
    }
1582
1583
    /// The remote whose URL starts with `base` (a hub), if configured.
1584
    pub fn remote_for_url(&self, base: &str) -> Option<String> {
1585
        let base = base.trim_end_matches('/');
4213c41d Andy McCall ↶ 1586
        self.config
1587
            .remotes
1588
            .iter()
1589
            .find(|(_, url)| url.trim_end_matches('/') == base)
1590
            .map(|(name, _)| name.clone())
d82e4409 Andy McCall ↶ 1591
    }
1592
1593
    /// Open a remote by name or, failing that, by URL with the default
ff1f62fe Andy McCall ↶ 1594
    /// credentials (see [`Repo::credentials_for`]).
d82e4409 Andy McCall ↶ 1595
    pub fn open_remote_at(&self, url: &str) -> Result<(String, Box<dyn crate::protocol::Remote>)> {
1596
        if let Some(name) = self.remote_for_url(url) {
1597
            return Ok((name.clone(), self.open_named_remote(&name)?));
1598
        }
1599
        let options = RemoteOptions {
1600
            repo_id: self.id().ok(),
ff1f62fe Andy McCall ↶ 1601
            ..self.credentials_for(url)
d82e4409 Andy McCall ↶ 1602
        };
1603
        Ok((url.to_string(), open_remote(url, &options)?))
1604
    }
1605
1606
    /// Credentials for a URL: those of the configured remote with that
1607
    /// URL, else the environment's.
1608
    pub fn options_for_url(&self, url: &str) -> RemoteOptions {
1609
        match self.remote_for_url(url) {
1610
            Some(name) => self.remote_options(&name),
1611
            None => RemoteOptions {
1612
                repo_id: self.id().ok(),
ff1f62fe Andy McCall ↶ 1613
                ..self.credentials_for(url)
d82e4409 Andy McCall ↶ 1614
            },
1615
        }
1616
    }
1617
c8d9059e Andy McCall ↶ 1618
    pub(crate) fn open_named_remote(&self, name: &str) -> Result<Box<dyn crate::protocol::Remote>> {
2b5ee371 Andy McCall ↶ 1619
        open_remote(&self.remote_url(name)?, &self.remote_options(name))
98286406 Andy McCall ↶ 1620
    }
1621
1622
    fn serial_path(&self, remote: &str, bookmark: &str) -> PathBuf {
75167672 Andy McCall 1623
        self.oxy
1624
            .join("remotes")
1625
            .join(remote)
1626
            .join(format!("{bookmark}.serial"))
98286406 Andy McCall ↶ 1627
    }
1628
1629
    fn last_seen_serial(&self, remote: &str, bookmark: &str) -> Option<u64> {
75167672 Andy McCall 1630
        fs::read_to_string(self.serial_path(remote, bookmark))
1631
            .ok()
1632
            .and_then(|s| s.trim().parse().ok())
98286406 Andy McCall ↶ 1633
    }
1634
1635
    fn record_serial(&self, remote: &str, bookmark: &str, serial: u64) -> Result<()> {
1636
        let p = self.serial_path(remote, bookmark);
1637
        fs::create_dir_all(p.parent().unwrap())?;
1638
        write_atomic(&p, serial.to_string().as_bytes())
1639
    }
1640
83e0677b Andy McCall ↶ 1641
    fn policy_serial_path(&self, remote: &str) -> PathBuf {
1642
        self.oxy.join("remotes").join(remote).join("policy.serial")
1643
    }
1644
1645
    /// The highest policy serial `remote` has ever presented.
1646
    fn remote_policy_serial(&self, remote: &str) -> Option<u64> {
75167672 Andy McCall 1647
        fs::read_to_string(self.policy_serial_path(remote))
1648
            .ok()
1649
            .and_then(|s| s.trim().parse().ok())
83e0677b Andy McCall ↶ 1650
    }
1651
1652
    fn record_policy_serial(&self, remote: &str, serial: u64) -> Result<()> {
1653
        let p = self.policy_serial_path(remote);
1654
        fs::create_dir_all(p.parent().unwrap())?;
1655
        write_atomic(&p, serial.to_string().as_bytes())
1656
    }
1657
b3c5a59a Andy McCall ↶ 1658
    fn lazy_dir(&self) -> PathBuf {
1659
        self.oxy.join("lazy")
1660
    }
1661
1662
    /// Remember that `blob` was withheld by `remote` (spec §13.5).
1663
    fn register_lazy(&self, blob: &ObjectRef, remote: &str) -> Result<()> {
1664
        fs::create_dir_all(self.lazy_dir())?;
1665
        write_atomic(&self.lazy_dir().join(blob.to_hex()), remote.as_bytes())
1666
    }
1667
1668
    /// Which remote promised an absent blob, if any.
1669
    pub fn lazy_remote(&self, blob: &ObjectRef) -> Option<String> {
75167672 Andy McCall 1670
        fs::read_to_string(self.lazy_dir().join(blob.to_hex()))
1671
            .ok()
1672
            .map(|s| s.trim().to_string())
b3c5a59a Andy McCall ↶ 1673
    }
1674
1675
    /// Fetch an absent blob (and its chunks) from the remote that promised it.
1676
    pub(crate) fn fetch_lazy_blob(&self, blob: &ObjectRef) -> Result<()> {
1677
        let Some(remote_name) = self.lazy_remote(blob) else {
1678
            return Err(RepoError::Store(StoreError::NotFound(*blob)));
1679
        };
1680
        let remote = self.open_named_remote(&remote_name)?;
1681
        let objects = remote.objects(&[*blob])?;
1682
        for (ty, body) in objects {
1683
            if matches!(ty, ObjectType::Blob | ObjectType::Chunk) {
1684
                self.store.put(ty, &body)?;
1685
            }
1686
        }
1687
        if !self.store.has(blob) {
75167672 Andy McCall 1688
            return fail(format!(
1689
                "{remote_name} did not send blob {}",
1690
                blob.short(12)
1691
            ));
b3c5a59a Andy McCall ↶ 1692
        }
1693
        let _ = fs::remove_file(self.lazy_dir().join(blob.to_hex()));
1694
        Ok(())
1695
    }
1696
1697
    /// After importing from `remote`, record every blob the changes reference
1698
    /// that the store lacks as lazily available from that remote.
1699
    fn register_missing_blobs(&self, changes: &[ChangeHash], remote: &str) -> Result<usize> {
1700
        let mut n = 0;
1701
        for h in changes {
1702
            let c = self.store.get_change(h)?;
1703
            for e in &c.edits {
1704
                if let Edit::Replace { old, new, .. } = e {
1705
                    for b in [old, new].into_iter().flatten() {
1706
                        if !self.store.has(b) {
1707
                            self.register_lazy(b, remote)?;
1708
                            n += 1;
1709
                        }
1710
                    }
1711
                }
1712
            }
1713
        }
1714
        Ok(n)
1715
    }
1716
98286406 Andy McCall ↶ 1717
    /// Every head this repository knows: bookmarks, remote tracking, working copy.
d82e4409 Andy McCall ↶ 1718
    pub fn known_heads(&self, view: &View) -> Result<Vec<ChangeHash>> {
98286406 Andy McCall ↶ 1719
        let mut have: BTreeSet<ChangeHash> = BTreeSet::new();
75167672 Andy McCall 1720
        for st in view
1721
            .bookmarks
1722
            .values()
1723
            .chain(view.remotes.values().flat_map(|b| b.values()))
1724
        {
98286406 Andy McCall ↶ 1725
            if let Ok(s) = self.store.get_state(st) {
1726
                have.extend(s.heads);
1727
            }
1728
        }
1729
        have.extend(view.workspaces.values().copied());
1730
        Ok(have.into_iter().collect())
1731
    }
1732
1733
    /// Fetch every bookmark of a remote into `remotes/<name>/bookmarks/*` (spec §10.2).
83e0677b Andy McCall ↶ 1734
    pub fn fetch(&self, remote_name: &str) -> Result<FetchReport> {
98286406 Andy McCall ↶ 1735
        let remote = self.open_named_remote(remote_name)?;
75167672 Andy McCall 1736
        self.check_remote_id(remote.as_ref(), remote_name, false)?;
2f0b4b06 Andy McCall ↶ 1737
        let resp = {
1738
            let _t = crate::timing::phase("fetch: refs");
1739
            remote.refs()?
1740
        };
4b9d0677 Andy McCall ↶ 1741
        // The host's transparency log (spec §14.8): a fork fails here.
2f0b4b06 Andy McCall ↶ 1742
        let log_phase = crate::timing::phase("fetch: transparency log");
028545df Andy McCall ↶ 1743
        let log = self.check_log(
4b9d0677 Andy McCall ↶ 1744
            &self.remote_url(remote_name)?,
1745
            self.config.remote_cas.get(remote_name).map(Path::new),
1746
            &resp,
1747
        )?;
2f0b4b06 Andy McCall ↶ 1748
        drop(log_phase);
028545df Andy McCall ↶ 1749
        let mut log_notes = log.notes.clone();
83e0677b Andy McCall ↶ 1750
        // Trust policy (spec §14.5). Adopt the remote's policy when it is newer
1751
        // and signed by roots we trust, or on first contact. A remote that
1752
        // once presented a policy may never present none or an older one:
1753
        // that is a downgrade. A remote that never presented one is
1754
        // unverified, and this fetch is checked only as far as possible.
1755
        let local_policy = self.current_policy()?.map(|(_, sp)| sp.policy);
1756
        let seen = self.remote_policy_serial(remote_name);
1757
        let mut unverified = false;
1758
        match &resp.policy {
1759
            Some(sp) => {
1760
                if seen.is_some_and(|s| sp.policy.serial < s) {
1761
                    return fail(format!("{remote_name} presents policy serial {} but presented {} before: refusing the downgrade", sp.policy.serial, seen.unwrap()));
1762
                }
1763
                match &local_policy {
1764
                    Some(l) if sp.policy.serial < l.serial => {}
1765
                    Some(l) if sp.policy.serial == l.serial => {
1766
                        if sp.policy.hash() != l.hash() {
75167672 Andy McCall 1767
                            return fail(format!(
1768
                                "{remote_name} presents a different policy with the same serial {}",
1769
                                l.serial
1770
                            ));
83e0677b Andy McCall ↶ 1771
                        }
1772
                    }
28fc23d7 Andy McCall ↶ 1773
                    _ => {
75167672 Andy McCall 1774
                        self.adopt_policy_chain(
1775
                            &resp.history,
1776
                            sp,
1777
                            &format!("oxy fetch {remote_name}"),
1778
                        )
1779
                        .map_err(|e| RepoError::Msg(format!("policy from {remote_name}: {e}")))?;
83e0677b Andy McCall ↶ 1780
                    }
1781
                }
1782
                self.record_policy_serial(remote_name, sp.policy.serial)?;
1783
            }
1784
            None => {
1785
                if seen.is_some() {
1786
                    return fail(format!("{remote_name} presented a policy before but has none now: refusing the downgrade"));
1787
                }
1788
                unverified = true;
1789
            }
1790
        }
75167672 Andy McCall 1791
        let policy = if unverified {
1792
            None
1793
        } else {
1794
            self.current_policy()?.map(|(_, sp)| sp.policy)
1795
        };
98286406 Andy McCall ↶ 1796
        let (op_ref, op) = self.head_op()?;
1797
        let mut view = op.view.clone();
1798
        let have = self.known_heads(&view)?;
1799
        let mut report = Vec::new();
d36ba2f7 Andy McCall ↶ 1800
        let mut skipped = Vec::new();
b3c5a59a Andy McCall ↶ 1801
        let max_blob = self.config.max_blob;
2f0b4b06 Andy McCall ↶ 1802
        // One negotiation and one bundle for every bookmark's heads at
1803
        // once (oxygen#158): a bookmark at a time cost a round trip each and
1804
        // sent again the changes bookmarks share, as `have` did not grow
1805
        // between them (160 bookmarks: 160 bundles, 49k changes encoded for
1806
        // a few hundred). Heads already here, with every attestation's
1807
        // signer known, need no round trip at all.
1808
        let mut wanted: Vec<ChangeHash> = resp
1809
            .refs
1810
            .iter()
1811
            .flat_map(|r| r.heads.iter().copied())
1812
            .filter(|h| !self.store.has(h))
1813
            .collect();
1814
        wanted.sort();
1815
        wanted.dedup();
1816
        // A bundle also carries the identities and certificates of the keys
1817
        // that signed the remote's attestations (spec §16.2): with no change
1818
        // wanted, ask anyway when one of those signers is unknown here.
1819
        let unknown_signer = resp
1820
            .refs
1821
            .iter()
1822
            .filter_map(|r| r.attestation.as_ref())
1823
            .chain(resp.tags.iter().filter_map(|t| t.attestation.as_ref()))
1824
            .flat_map(|a| a.sigs.iter().map(|s| s.key))
1825
            .any(|k| {
1826
                self.store.get_identity(&k).is_err()
1827
                    && !matches!(self.store.certificate_of(&k), Ok(Some(_)))
1828
            });
1829
        let shallow = self.is_shallow();
1830
        let imported: Vec<ChangeHash> = if wanted.is_empty() && !unknown_signer {
1831
            Vec::new()
1832
        } else {
091fef80 Andy McCall ↶ 1833
            let needed = {
1834
                let _t = crate::timing::phase("fetch: negotiate");
2f0b4b06 Andy McCall ↶ 1835
                remote.negotiate(&have, &wanted)?
091fef80 Andy McCall ↶ 1836
            };
1837
            let bundle = {
1838
                let _t = crate::timing::phase("fetch: bundle");
1839
                remote.fetch(&needed, max_blob)?
1840
            };
2f0b4b06 Andy McCall ↶ 1841
            // Changes are not individually re-verified here: each bookmark's
83e0677b Andy McCall ↶ 1842
            // attestation (checked below) binds its whole state, and the
1843
            // receiver of the push verified the added changes at that time.
f3811eb2 Andy McCall ↶ 1844
            // A shallow repository takes changes whose dependencies it never
1845
            // held (below what it cloned), as its clone did (#137).
2f0b4b06 Andy McCall ↶ 1846
            let _t = crate::timing::phase("fetch: import");
f3811eb2 Andy McCall ↶ 1847
            if shallow {
2f0b4b06 Andy McCall ↶ 1848
                self.import_bundle_shallow(&bundle, false)?
1849
            } else {
1850
                self.import_bundle(&bundle, false)?
f3811eb2 Andy McCall ↶ 1851
            }
2f0b4b06 Andy McCall ↶ 1852
        };
1853
        if shallow {
1854
            self.refresh_boundary(&imported)?;
1855
        }
1856
        if max_blob.is_some() {
1857
            self.register_missing_blobs(&imported, remote_name)?;
1858
        }
1859
        let boundary = if shallow {
1860
            self.boundary()
1861
        } else {
1862
            BTreeSet::new()
1863
        };
1864
        // Each bookmark's new changes: those just imported that its heads
1865
        // reach, for the report.
1866
        let mut new_deps: HashMap<ChangeHash, Vec<ChangeHash>> = HashMap::new();
1867
        for h in &imported {
1868
            if let Ok(m) = self.store.get_change_meta(h) {
1869
                new_deps.insert(*h, m.deps);
b3c5a59a Andy McCall ↶ 1870
            }
2f0b4b06 Andy McCall ↶ 1871
        }
1872
        let new_in = |heads: &[ChangeHash]| -> usize {
1873
            let mut seen: HashSet<ChangeHash> = HashSet::new();
1874
            let mut stack: Vec<ChangeHash> = heads
1875
                .iter()
1876
                .filter(|h| new_deps.contains_key(*h))
1877
                .copied()
1878
                .collect();
1879
            while let Some(h) = stack.pop() {
1880
                if seen.insert(h) {
1881
                    stack.extend(
1882
                        new_deps[&h]
1883
                            .iter()
1884
                            .filter(|d| new_deps.contains_key(*d) && !seen.contains(*d)),
1885
                    );
1886
                }
1887
            }
1888
            seen.len()
1889
        };
1890
        let checks = crate::timing::phase("fetch: check each bookmark");
1891
        for r in resp.refs {
98286406 Andy McCall ↶ 1892
            for h in &r.heads {
f3811eb2 Andy McCall ↶ 1893
                if !self.store.has(h) && !boundary.contains(h) {
98286406 Andy McCall ↶ 1894
                    return fail(format!("remote did not send head {}", h.short(12)));
1895
                }
1896
            }
1897
            let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
1898
            if state != r.state {
75167672 Andy McCall 1899
                return fail(format!(
1900
                    "remote state {} does not match its heads",
1901
                    r.state.short(12)
1902
                ));
98286406 Andy McCall ↶ 1903
            }
83e0677b Andy McCall ↶ 1904
            if let Some(p) = &policy {
1905
                let Some(sa) = &r.attestation else {
75167672 Andy McCall 1906
                    return fail(format!(
1907
                        "{remote_name}/{} has no attestation, which its policy requires",
1908
                        r.name
1909
                    ));
83e0677b Andy McCall ↶ 1910
                };
75167672 Andy McCall 1911
                let signer = sa
1912
                    .sigs
1913
                    .first()
1914
                    .map(|s| s.key)
1915
                    .ok_or_else(|| RepoError::Msg("attestation carries no signature".into()))?;
028545df Andy McCall ↶ 1916
                if !p.authorizes(&signer, &r.name) {
1917
                    self.authorize_in_time(p, remote_name, &r, sa, &signer, &log, &mut log_notes)?;
83e0677b Andy McCall ↶ 1918
                }
1919
            }
98286406 Andy McCall ↶ 1920
            if let Some(sa) = &r.attestation {
75167672 Andy McCall 1921
                let AttestBody::Ref {
1922
                    old_state,
1923
                    new_state,
1924
                    ..
1925
                } = &sa.attest.body
1926
                else {
98286406 Andy McCall ↶ 1927
                    return fail("remote sent a non-ref attestation for a bookmark");
1928
                };
1929
                let (old_state, new_state) = (*old_state, *new_state);
1930
                let serial = self
d36ba2f7 Andy McCall ↶ 1931
                    .check_ref_attest_body(
75167672 Andy McCall 1932
                        sa,
1933
                        &r.name,
1934
                        &old_state,
1935
                        &new_state,
1936
                        self.last_seen_serial(remote_name, &r.name),
1937
                        true,
1938
                    )
1939
                    .map_err(|e| {
1940
                        RepoError::Msg(format!("attestation for {}/{}: {e}", remote_name, r.name))
1941
                    })?;
98286406 Andy McCall ↶ 1942
                if new_state != r.state {
75167672 Andy McCall 1943
                    return fail(format!(
1944
                        "attestation for {}/{} does not cover its current state",
1945
                        remote_name, r.name
1946
                    ));
98286406 Andy McCall ↶ 1947
                }
d36ba2f7 Andy McCall ↶ 1948
                // A signature this client cannot verify (its signer unknown
1949
                // here, as from a host that did not send the identity, or
1950
                // bad) fails that bookmark alone: it stays as it was and
1951
                // the rest are fetched (oxygenhub#266). The current
1952
                // bookmark, which a pull would take, still fails the fetch.
1953
                // A policy's requirements were checked above and still fail.
1954
                if let Err(e) = self.verify_attest_signature(sa) {
1955
                    let why = format!("attestation for {remote_name}/{}: {e}", r.name);
1956
                    if r.name == self.config.bookmark {
1957
                        return fail(why);
1958
                    }
1959
                    skipped.push((r.name, why));
1960
                    continue;
1961
                }
98286406 Andy McCall ↶ 1962
                let ar = self.store.put_attest(&sa.attest)?;
1963
                for sig in &sa.sigs {
1964
                    self.store.put_signature(&ar, sig)?;
1965
                }
1966
                self.record_serial(remote_name, &r.name, serial)?;
1967
            }
75167672 Andy McCall 1968
            view.remotes
1969
                .entry(remote_name.to_string())
1970
                .or_default()
1971
                .insert(r.name.clone(), state);
2f0b4b06 Andy McCall ↶ 1972
            let n = new_in(&r.heads);
1973
            report.push((r.name, r.heads, n));
98286406 Andy McCall ↶ 1974
        }
2f0b4b06 Andy McCall ↶ 1975
        drop(checks);
1976
        let tags_phase = crate::timing::phase("fetch: tags");
132b884b Andy McCall ↶ 1977
        let mut new_tags = 0;
1978
        for t in &resp.tags {
1979
            if view.tags.contains_key(&t.name) {
b63fc7a7 Andy McCall ↶ 1980
                continue;
1981
            }
1982
            // Withdrawn here: the name is not used again.
1983
            if self.tag_withdrawal(&t.name)?.is_some() {
132b884b Andy McCall ↶ 1984
                continue;
1985
            }
1986
            // The tag's changes must be here: fetch them if the bookmarks did not bring them.
75167672 Andy McCall 1987
            let missing: Vec<ChangeHash> = t
1988
                .heads
1989
                .iter()
1990
                .filter(|h| !self.store.has(h))
1991
                .copied()
1992
                .collect();
132b884b Andy McCall ↶ 1993
            if !missing.is_empty() {
1994
                let needed = remote.negotiate(&self.known_heads(&view)?, &missing)?;
1995
                let bundle = remote.fetch(&needed, max_blob)?;
1996
                self.import_bundle(&bundle, false)?;
1997
            }
1998
            self.store.put_state(&StateObject::new(t.heads.clone()))?;
1ec6fdbc Andy McCall ↶ 1999
            let mut check = TagCheck::Fetched {
2000
                remote: remote_name,
2001
                log: &log,
2002
                notes: &mut log_notes,
2003
            };
2004
            match self.adopt_tag(&mut view, t, policy.as_ref(), &mut check) {
2005
                Ok(()) => new_tags += 1,
2006
                // Nothing says when a short-lived key signed it: the tag is
2007
                // left out, said so, and the rest is fetched (oxygen#129).
2008
                Err(e) if e.untimed => {
2009
                    skipped.push((format!("tags/{}", t.name), e.reason));
2010
                }
2011
                Err(e) => return fail(format!("{remote_name} {}: {}", t.name, e.reason)),
2012
            }
132b884b Andy McCall ↶ 2013
        }
b63fc7a7 Andy McCall ↶ 2014
        // The remote withdrew a tag this repository has: so does this one,
2015
        // once the withdrawal is checked as a push's would be; one that is
2016
        // not is left as it is, said so (oxygenhub#301).
2f0b4b06 Andy McCall ↶ 2017
        drop(tags_phase);
b63fc7a7 Andy McCall ↶ 2018
        let mut withdrawn_tags = Vec::new();
2019
        for t in &resp.withdrawn {
2020
            if !view.tags.contains_key(&t.name) {
2021
                continue;
2022
            }
2023
            let mut check = TagCheck::Fetched {
2024
                remote: remote_name,
2025
                log: &log,
2026
                notes: &mut log_notes,
2027
            };
2028
            match self.adopt_withdrawal(&mut view, t, policy.as_ref(), &mut check) {
2029
                Ok(true) => withdrawn_tags.push(t.name.clone()),
2030
                Ok(false) => {}
2031
                Err(e) => skipped.push((
2032
                    format!("tags/{}", t.name),
2033
                    format!("withdrawn on {remote_name}, kept here: {}", e.reason),
2034
                )),
2035
            }
2036
        }
98286406 Andy McCall ↶ 2037
        if view != op.view {
2038
            self.record_op(&[op_ref], &format!("oxy fetch {remote_name}"), view)?;
2039
        }
6d3eefbc Andy McCall ↶ 2040
        let _ = self.run_hook(
2041
            "post-fetch",
2042
            &format!(
2043
                "{{\"event\":\"post-fetch\",\"remote\":{},\"bookmarks\":{}}}",
2044
                json_str(remote_name),
75167672 Andy McCall 2045
                json_array(report.iter().map(|(n, heads, _)| format!(
2046
                    "{{\"name\":{},\"heads\":{}}}",
2047
                    json_str(n),
2048
                    json_array(heads.iter().map(|h| json_str(&h.to_hex())))
2049
                )))
6d3eefbc Andy McCall ↶ 2050
            ),
2051
        );
75167672 Andy McCall 2052
        Ok(FetchReport {
2053
            refs: report,
2054
            unverified,
2055
            new_tags,
b63fc7a7 Andy McCall ↶ 2056
            withdrawn_tags,
4b9d0677 Andy McCall ↶ 2057
            log_notes,
d36ba2f7 Andy McCall ↶ 2058
            skipped,
75167672 Andy McCall 2059
        })
98286406 Andy McCall ↶ 2060
    }
2061
2863456a Andy McCall ↶ 2062
    /// Shallow fetch of the remote's bookmark: its checkpoint as our base,
2063
    /// the changes within `depth` steps of its heads, and the boundary below.
2064
    pub fn fetch_shallow(&self, remote_name: &str, depth: u32) -> Result<usize> {
2065
        let remote = self.open_named_remote(remote_name)?;
75167672 Andy McCall 2066
        self.check_remote_id(remote.as_ref(), remote_name, false)?;
2863456a Andy McCall ↶ 2067
        let resp = remote.refs()?;
2068
        let bookmark = self.config.bookmark.clone();
2069
        if let Some(sp) = &resp.policy {
75167672 Andy McCall 2070
            self.adopt_policy_chain(&resp.history, sp, "oxy clone --depth")
2071
                .map_err(|e| RepoError::Msg(format!("policy from {remote_name}: {e}")))?;
2863456a Andy McCall ↶ 2072
            self.record_policy_serial(remote_name, sp.policy.serial)?;
2073
        }
2074
        let Some(r) = resp.refs.into_iter().find(|r| r.name == bookmark) else {
2075
            return fail(format!("{remote_name} has no bookmark {bookmark:?}"));
2076
        };
2077
        if r.heads.is_empty() {
2078
            return Ok(0);
2079
        }
80b069fa Andy McCall ↶ 2080
        // Base: the remote's checkpoint of exactly these heads, without its
2081
        // graph blobs, which come in parts below (#137).
f3811eb2 Andy McCall ↶ 2082
        let cp_bundle = {
2083
            let _t = crate::timing::phase("shallow: checkpoint request");
80b069fa Andy McCall ↶ 2084
            remote.checkpoint_index(&r.heads)?
f3811eb2 Andy McCall ↶ 2085
        };
7bd805ff Andy McCall ↶ 2086
        // In one batch: a big state's checkpoint is tens of thousands of
2087
        // graph blobs and chunks, which stored one by one were each written
2088
        // loose and flushed (a VS Code clone ran at 27 objects a second).
2089
        let objects: Vec<(ObjectType, &[u8])> = cp_bundle
2090
            .objects
2091
            .iter()
2092
            .map(|(ty, body)| (*ty, body.as_slice()))
2093
            .collect();
f3811eb2 Andy McCall ↶ 2094
        let t_store = crate::timing::phase("shallow: store the checkpoint");
7bd805ff Andy McCall ↶ 2095
        let batch = self.store.begin_batch()?;
2096
        let stored = self.store.put_many(&objects);
2097
        batch.finish()?;
f3811eb2 Andy McCall ↶ 2098
        drop(t_store);
7bd805ff Andy McCall ↶ 2099
        let cp_ref = objects
2100
            .iter()
2101
            .zip(stored?)
2102
            .find(|((ty, _), _)| *ty == ObjectType::Checkpoint)
2103
            .map(|(_, r)| r);
75167672 Andy McCall 2104
        let Some(cp_ref) = cp_ref else {
2105
            return fail(format!("{remote_name} sent no checkpoint"));
2106
        };
80b069fa Andy McCall ↶ 2107
        // The graph blobs the payload names, with their chunks, a part at
2108
        // a time (#137): one answer for a big state was gigabytes, held
2109
        // whole on both sides. A remote that sent the whole checkpoint has
2110
        // left none to ask for.
2111
        let cp = crate::objects::Checkpoint::decode(
2112
            &self.store.get_typed(&cp_ref, ObjectType::Checkpoint)?,
2113
        )?;
2114
        let graphs: Vec<ObjectRef> = self
2115
            .checkpoint_graph_blobs(&cp)?
2116
            .into_iter()
2117
            .filter(|b| !self.store.has(b))
2118
            .collect();
2119
        {
2120
            let _t = crate::timing::phase("shallow: the checkpoint's graphs, in parts");
2121
            for part in graphs.chunks(CHECKPOINT_PART) {
2122
                let objects = remote.objects(part)?;
2123
                let objects: Vec<(ObjectType, &[u8])> = objects
2124
                    .iter()
2125
                    .filter(|(ty, _)| matches!(ty, ObjectType::Blob | ObjectType::Chunk))
2126
                    .map(|(ty, body)| (*ty, body.as_slice()))
2127
                    .collect();
2128
                let batch = self.store.begin_batch()?;
2129
                let stored = self.store.put_many(&objects);
2130
                batch.finish()?;
2131
                stored?;
2132
            }
2133
            if let Some(b) = graphs.iter().find(|b| !self.store.has(b)) {
2134
                return fail(format!(
2135
                    "{remote_name} did not send graph blob {}",
2136
                    b.short(12)
2137
                ));
2138
            }
2139
        }
2863456a Andy McCall ↶ 2140
        self.install_checkpoint(&bookmark, &cp_ref)?;
f3811eb2 Andy McCall ↶ 2141
        let _t_blobs = crate::timing::phase("shallow: the binary files' blobs");
7bd805ff Andy McCall ↶ 2142
        // The blobs of the state's opaque files come with no change when the
2143
        // window holds none of the changes that set them (#137): fetch them
2144
        // now, or under `filter.max-blob` leave them to be fetched on read.
2145
        let base = State::from_checkpoint_history(&self.read_blob(&cp.payload)?)?;
2146
        let mut missing: Vec<ObjectRef> = base
2147
            .opaque_nodes()
2148
            .iter()
2149
            .flat_map(|n| base.live_blobs(n))
2150
            .filter(|b| !self.store.has(b))
2151
            .collect();
2152
        missing.sort();
2153
        missing.dedup();
2154
        if self.config.max_blob.is_some() {
2155
            for b in &missing {
2156
                self.register_lazy(b, remote_name)?;
2157
            }
2158
        } else {
2159
            for part in missing.chunks(512) {
2160
                let objects = remote.objects(part)?;
2161
                let objects: Vec<(ObjectType, &[u8])> = objects
2162
                    .iter()
2163
                    .filter(|(ty, _)| matches!(ty, ObjectType::Blob | ObjectType::Chunk))
2164
                    .map(|(ty, body)| (*ty, body.as_slice()))
2165
                    .collect();
2166
                let batch = self.store.begin_batch()?;
2167
                let stored = self.store.put_many(&objects);
2168
                batch.finish()?;
2169
                stored?;
2170
            }
2171
            if let Some(b) = missing.iter().find(|b| !self.store.has(b)) {
2172
                return fail(format!("{remote_name} did not send blob {}", b.short(12)));
2173
            }
2174
        }
f3811eb2 Andy McCall ↶ 2175
        // History window. Depth 0 takes none (#137): the checkpoint holds
2176
        // every change's metadata, and its heads become the boundary, so a
2177
        // state of many heads (33,835 in the VS Code import) costs nothing
2178
        // to fetch.
2179
        let imported = if depth == 0 {
2180
            let mut boundary = self.boundary();
2181
            boundary.extend(r.heads.iter().filter(|h| !self.store.has(h)).copied());
2182
            self.write_boundary(&boundary)?;
2183
            Vec::new()
2184
        } else {
2185
            let wanted = remote.shallow(&r.heads, depth)?;
2186
            let bundle = remote.fetch(&wanted, self.config.max_blob)?;
2187
            self.import_bundle_shallow(&bundle, false)?
2188
        };
2863456a Andy McCall ↶ 2189
        if self.config.max_blob.is_some() {
2190
            self.register_missing_blobs(&imported, remote_name)?;
2191
        }
2192
        self.refresh_boundary(&imported)?;
2193
        if let Some(sa) = &r.attestation {
2194
            let ar = self.store.put_attest(&sa.attest)?;
2195
            for sig in &sa.sigs {
2196
                self.store.put_signature(&ar, sig)?;
2197
            }
2198
            if let AttestBody::Ref { serial, .. } = sa.attest.body {
2199
                self.record_serial(remote_name, &r.name, serial)?;
2200
            }
2201
        }
2202
        let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
2203
        let (op_ref, op) = self.head_op()?;
2204
        let mut view = op.view.clone();
75167672 Andy McCall 2205
        view.remotes
2206
            .entry(remote_name.to_string())
2207
            .or_default()
2208
            .insert(r.name.clone(), state);
2209
        self.record_op(
2210
            &[op_ref],
2211
            &format!("oxy fetch {remote_name} --depth {depth}"),
2212
            view,
2213
        )?;
2863456a Andy McCall ↶ 2214
        Ok(imported.len())
2215
    }
2216
98286406 Andy McCall ↶ 2217
    /// Fetch, then take the union of the local bookmark's state with the
2218
    /// remote's: no merge change, conflicts become reified (spec §9.4).
2219
    pub fn pull(&self, remote_name: &str) -> Result<PullReport> {
6caacfff Andy McCall ↶ 2220
        let _wc = self.lock_working_copy()?;
d36ba2f7 Andy McCall ↶ 2221
        let skipped = self.fetch(remote_name)?.skipped;
98286406 Andy McCall ↶ 2222
        let (op_ref, op) = self.head_op()?;
14edbaaf Andy McCall ↶ 2223
        let current = self.config.bookmark.clone();
2224
        let mut local_current = self.heads_of(&op.view, &current)?;
2225
        local_current.sort();
2226
        let mut view = op.view.clone();
2227
        let mut current_heads = local_current.clone();
2228
        let mut changed = false;
2229
        let mut others = Vec::new();
2230
        let tracked = op
75167672 Andy McCall 2231
            .view
2232
            .remotes
2233
            .get(remote_name)
14edbaaf Andy McCall ↶ 2234
            .cloned()
2235
            .unwrap_or_default();
2236
        for (name, remote_state) in tracked {
2237
            // Every local bookmark tracks the remote's of the same name;
2238
            // the current one may be new here.
2239
            let local_state = op.view.bookmarks.get(&name).copied();
2240
            if local_state.is_none() && name != current {
2241
                continue;
2242
            }
2243
            if local_state == Some(remote_state) {
2244
                continue;
2245
            }
2246
            let local = self.heads_of(&op.view, &name)?;
2247
            let remote_heads = self.store.get_state(&remote_state)?.heads;
2248
            if remote_heads.iter().all(|h| local.contains(h)) {
2249
                continue;
2250
            }
2251
            let mut all = local.clone();
2252
            all.extend(remote_heads);
2253
            let st = self.load_state_history(&all)?;
2254
            let mut heads = st.maximal(&all);
2255
            heads.sort();
2256
            let mut sorted_local = local;
2257
            sorted_local.sort();
2258
            if heads == sorted_local {
2259
                continue;
2260
            }
2261
            view.bookmarks.insert(
2262
                name.clone(),
2263
                self.store.put_state(&StateObject::new(heads.clone()))?,
2264
            );
2265
            if name == current {
2266
                changed = true;
2267
                current_heads = heads;
2268
            } else {
2269
                others.push(name);
2270
            }
2271
        }
f6dccc45 Andy McCall ↶ 2272
        if changed || !others.is_empty() {
2273
            self.record_op(&[op_ref], &format!("oxy pull {remote_name}"), view)?;
98286406 Andy McCall ↶ 2274
        }
f6dccc45 Andy McCall ↶ 2275
        // The working copy follows its bookmark whenever nothing would be
2276
        // lost, also when this pull brought nothing new but it had been
2277
        // left behind (oxygen#89).
2278
        let checked_out = self.follow_bookmark(&current_heads, &current, "oxy pull")?;
75167672 Andy McCall 2279
        Ok(PullReport {
f6dccc45 Andy McCall ↶ 2280
            heads: if changed {
2281
                current_heads
2282
            } else {
2283
                local_current
2284
            },
14edbaaf Andy McCall ↶ 2285
            changed,
75167672 Andy McCall 2286
            checked_out,
14edbaaf Andy McCall ↶ 2287
            others,
d36ba2f7 Andy McCall ↶ 2288
            skipped,
75167672 Andy McCall 2289
        })
f6dccc45 Andy McCall ↶ 2290
    }
2291
2292
    /// How many changes the working copy's bookmark has that its base
2293
    /// lacks, when the base is part of the bookmark's state (it fell behind,
2294
    /// and `oxy checkout` or `oxy pull` brings it up); `None` when it is on
2295
    /// the bookmark or holds changes the bookmark lacks. History is read
2296
    /// only when the base and the bookmark's heads differ (oxygen#89).
2297
    pub fn working_copy_behind(&self) -> Result<Option<usize>> {
2298
        let (_, wc) = self.working_copy()?;
2299
        let mut base = wc.deps.clone();
2300
        base.sort();
2301
        let mut heads = self.bookmark_heads()?;
2302
        heads.sort();
2303
        if base == heads || heads.is_empty() {
2304
            return Ok(None);
2305
        }
2306
        let state = self.load_state_history(&heads)?;
2307
        if !base.iter().all(|h| state.contains(h)) {
2308
            return Ok(None);
2309
        }
2310
        let had = if base.is_empty() {
2311
            0
2312
        } else {
2313
            self.load_state_history(&base)?.len()
2314
        };
2315
        Ok(Some(state.len().saturating_sub(had)).filter(|n| *n > 0))
2316
    }
2317
2318
    /// Bring the working copy to `heads`, the state of its bookmark
2319
    /// `bookmark`, when nothing would be lost: its base is part of that
2320
    /// state, so the bookmark only adds to it. Clean, it is checked out;
2321
    /// with uncommitted work, the work comes along when the new changes
2322
    /// leave its paths alone (as `git pull` does), else it stays. Before,
2323
    /// it followed only from exactly the bookmark's old heads, so a working
2324
    /// copy already behind (the bookmark moved by `bookmark set`, a landing
2325
    /// or a pull from another workspace) never caught up, and nothing said
2326
    /// so (oxygen#74, #89). Returns whether it moved.
2327
    pub fn follow_bookmark(
2328
        &self,
2329
        heads: &[ChangeHash],
2330
        bookmark: &str,
2331
        command: &str,
2332
    ) -> Result<bool> {
2333
        let snap = self.snapshot(command)?;
2334
        let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
2335
        let mut base = snap.base.maximal(&snap.wc.deps);
2336
        base.sort();
2337
        let mut target = heads.to_vec();
2338
        target.sort();
2339
        if base == target {
2340
            return Ok(false);
2341
        }
2342
        let state = self.load_state_history(heads)?;
2343
        if !base.iter().all(|h| state.contains(h)) {
2344
            // The working copy holds changes the bookmark lacks: not ours to move.
2345
            return Ok(false);
2346
        }
2347
        if clean {
2348
            self.checkout(heads, false, &format!("{command} (checkout)"))?;
2349
            Ok(true)
2350
        } else {
2351
            Ok(self
2352
                .checkout_carry(heads, bookmark, &format!("{command} (checkout --carry)"))
2353
                .is_ok())
b63fc7a7 Andy McCall ↶ 2354
        }
2355
    }
2356
2357
    /// Withdraw tag `tag` on `remote_name` (`oxy push --delete tags/NAME`,
2358
    /// oxygenhub#301): withdrawn here first when it still is a tag here
2359
    /// (`oxy tag delete`), then the withdrawal alone is sent, with its
2360
    /// signer's identity and certificate.
2361
    pub fn push_tag_withdrawal(&self, remote_name: &str, tag: &str) -> Result<Vec<RefResult>> {
2362
        let remote = self.open_named_remote(remote_name)?;
2363
        self.check_remote_id(remote.as_ref(), remote_name, true)?;
2364
        let resp = remote.refs()?;
2365
        let name = format!("tags/{tag}");
2366
        if resp.withdrawn.iter().any(|t| t.name == tag) {
2367
            return Ok(vec![RefResult {
2368
                name,
2369
                ok: true,
2370
                reason: "already withdrawn".into(),
2371
            }]);
2372
        }
2373
        let Some(there) = resp.tags.iter().find(|t| t.name == tag) else {
2374
            return fail(format!("{remote_name} has no tag {tag}"));
2375
        };
2376
        if resp.policy.is_some() && self.secret_key()?.is_none() {
2377
            return fail(format!("{remote_name} enforces a trust policy; withdrawing a tag needs an identity (`oxy identity create`)"));
f6dccc45 Andy McCall ↶ 2378
        }
b63fc7a7 Andy McCall ↶ 2379
        if self.tag_withdrawal(tag)?.is_none() {
2380
            match self.view()?.tags.get(tag) {
2381
                Some(st) if *st == there.state => {
2382
                    self.tag_delete(tag, "")?;
2383
                }
2384
                Some(_) => {
2385
                    return fail(format!(
2386
                        "tag {tag} names another state here than on {remote_name}"
2387
                    ))
2388
                }
2389
                None => {
2390
                    return fail(format!(
2391
                        "there is no tag {tag} here to withdraw: `oxy fetch {remote_name}` first"
2392
                    ))
2393
                }
2394
            }
2395
        }
2396
        let Some(w) = self
2397
            .withdrawn_refs()?
2398
            .into_iter()
2399
            .find(|w| w.name == tag && w.state == there.state)
2400
        else {
2401
            return fail(format!(
2402
                "tag {tag} was withdrawn here at another state than {remote_name}'s"
2403
            ));
2404
        };
2405
        let mut bundle = Bundle::default();
2406
        let signers: Vec<KeyId> = w
2407
            .attestation
2408
            .iter()
2409
            .flat_map(|a| a.sigs.iter().map(|s| s.key))
2410
            .collect();
2411
        for k in &signers {
2412
            if let Ok(id) = self.identity(k) {
2413
                bundle.objects.push((ObjectType::Identity, id.encode()));
2414
            }
2415
        }
2416
        self.attach_certificates_for(&mut bundle, &signers)?;
2417
        remote.push(bundle, vec![], None, vec![w])
98286406 Andy McCall ↶ 2418
    }
2419
2420
    /// Push the current bookmark (spec §16.3), sealing this author's unsigned
2421
    /// changes and attesting the reference when a key is available.
f8c69576 Andy McCall ↶ 2422
    /// Delete `bookmark` on a remote (spec §16.4): an update to the zero
2423
    /// state, attested when we have an identity. The local bookmark, if
2424
    /// any, is kept; only the remote-tracking entry goes.
2425
    pub fn push_delete(&self, remote_name: &str, bookmark: &str) -> Result<Vec<RefResult>> {
b63fc7a7 Andy McCall ↶ 2426
        if let Some(tag) = bookmark.strip_prefix("tags/") {
2427
            return self.push_tag_withdrawal(remote_name, tag);
2428
        }
f8c69576 Andy McCall ↶ 2429
        let remote = self.open_named_remote(remote_name)?;
2430
        self.check_remote_id(remote.as_ref(), remote_name, true)?;
2431
        let resp = remote.refs()?;
2432
        let Some(r) = resp.refs.iter().find(|r| r.name == bookmark) else {
2433
            return fail(format!("{remote_name} has no bookmark {bookmark}"));
2434
        };
2435
        if resp.policy.is_some() && self.secret_key()?.is_none() {
2436
            return fail(format!("{remote_name} enforces a trust policy; deleting needs an identity (`oxy identity create`)"));
2437
        }
2438
        let attestation = match self.secret_key()? {
2439
            Some(_) => Some(self.sign_attest(&Attest {
2440
                repo: self.id()?,
2441
                hlc: Hlc::now(),
2442
                body: AttestBody::Ref {
2443
                    name: bookmark.to_string(),
2444
                    old_state: r.state,
2445
                    new_state: ObjectRef::ZERO,
2446
                    serial: r.last_serial.unwrap_or(0) + 1,
2447
                },
2448
            })?),
2449
            None => None,
2450
        };
2451
        let results = remote.push(
2452
            Bundle::default(),
2453
            vec![RefUpdate {
2454
                name: bookmark.to_string(),
2455
                old_state: r.state,
2456
                new_state: ObjectRef::ZERO,
2457
                force: true,
2458
                attestation,
2459
            }],
2460
            None,
2461
            vec![],
2462
        )?;
2463
        if results.iter().any(|x| x.ok && x.name == bookmark) {
2464
            let (op_ref, op) = self.head_op()?;
2465
            let mut view = op.view.clone();
2466
            if let Some(m) = view.remotes.get_mut(remote_name) {
2467
                m.remove(bookmark);
2468
            }
2469
            self.record_op(
2470
                &[op_ref],
2471
                &format!("oxy push {remote_name} --delete {bookmark}"),
2472
                view,
2473
            )?;
2474
        }
2475
        Ok(results)
2476
    }
2477
34fad4a9 Andy McCall ↶ 2478
    /// Remove remote `name` (oxygen#86): every `remote.NAME.*` key of this
2479
    /// repository's config (its URL, token, CA and store settings), its
2480
    /// tracking bookmarks in the view, and a store remote's mirror under
2481
    /// `.oxy/remotes/NAME`. Nothing stays to be read as if it were current.
2482
    pub fn remove_remote(&mut self, name: &str) -> Result<()> {
2483
        if !self.config.remotes.contains_key(name) {
2484
            return fail(format!("no remote {name}: `oxy remote list` shows them"));
2485
        }
2486
        let prefix = format!("remote.{name}.");
2487
        let keys: Vec<String> = self
2488
            .config
2489
            .repo_layer()
2490
            .keys()
2491
            .filter(|k| k.starts_with(&prefix))
2492
            .cloned()
2493
            .collect();
2494
        for k in keys {
2495
            self.config.set(&k, "")?;
2496
        }
2497
        self.config.remotes.remove(name);
2498
        self.save_config()?;
2499
        let (op_ref, op) = self.head_op()?;
2500
        if op.view.remotes.contains_key(name) {
2501
            let mut view = op.view.clone();
2502
            view.remotes.remove(name);
2503
            self.record_op(&[op_ref], &format!("oxy remote remove {name}"), view)?;
2504
        }
2505
        let mirror = self.oxy.join("remotes").join(name);
2506
        if mirror.is_dir() {
2507
            fs::remove_dir_all(&mirror)?;
2508
        }
2509
        Ok(())
2510
    }
2511
d3b27090 Andy McCall ↶ 2512
    /// Send `bundle`'s blobs and chunks ahead of it, in parts of about
2513
    /// `push.batch-bytes` of objects alone, when the whole would be larger
2514
    /// (oxygen#133): one change can bring more blobs than one request may
2515
    /// carry, and a proxy in front of a hub refuses a request past its limit
2516
    /// (Cloudflare's is 100 MB). What is left, the changes with their
2517
    /// identities and states, is returned to send as before. A receiver
2518
    /// stores objects as they come, and checks a change's blobs are stored.
2519
    fn send_objects_ahead(
2520
        &self,
2521
        remote: &dyn crate::protocol::Remote,
2522
        remote_name: &str,
2523
        mut bundle: Bundle,
2524
        progress: crate::store::Progress,
2525
    ) -> Result<Bundle> {
2526
        let limit = self.config.push_batch_bytes;
2527
        let changes: u64 = bundle
2528
            .changes
2529
            .iter()
2530
            .map(|(c, _)| c.encode().len() as u64)
2531
            .sum();
2532
        let objects: u64 = bundle.objects.iter().map(|(_, b)| b.len() as u64).sum();
2533
        if changes + objects <= limit {
2534
            return Ok(bundle);
2535
        }
2536
        let (mut kept, mut parts, mut part, mut bytes) = (Vec::new(), Vec::new(), Vec::new(), 0u64);
2537
        for (ty, body) in std::mem::take(&mut bundle.objects) {
2538
            if !matches!(ty, ObjectType::Blob | ObjectType::Chunk) {
2539
                kept.push((ty, body));
2540
                continue;
2541
            }
2542
            if !part.is_empty() && bytes + body.len() as u64 > limit {
2543
                parts.push(std::mem::take(&mut part));
2544
                bytes = 0;
2545
            }
2546
            bytes += body.len() as u64;
2547
            part.push((ty, body));
2548
        }
2549
        if !part.is_empty() {
2550
            parts.push(part);
2551
        }
2552
        let n = parts.len();
2553
        for (i, objects) in parts.into_iter().enumerate() {
2554
            progress(&format!(
2555
                "objects {} of {n}: {} blobs and chunks",
2556
                i + 1,
2557
                objects.len()
2558
            ));
2559
            let results = remote.push(
2560
                Bundle {
2561
                    objects,
2562
                    ..Default::default()
2563
                },
2564
                vec![],
2565
                None,
2566
                vec![],
2567
            )?;
2568
            match results.iter().find(|r| r.name == OBJECTS_RESULT) {
2569
                Some(r) if r.ok => {}
2570
                Some(r) => {
2571
                    return fail(format!(
2572
                        "{remote_name} refused objects part {} of {n}: {}",
2573
                        i + 1,
2574
                        r.reason
2575
                    ))
2576
                }
2577
                None => {
2578
                    return fail(format!(
2579
                        "{remote_name} does not take a push in parts, and this one is too large for one: update the server's oxy"
2580
                    ))
2581
                }
2582
            }
2583
        }
2584
        bundle.objects = kept;
2585
        Ok(bundle)
2586
    }
2587
cb8b8082 Andy McCall ↶ 2588
    /// `changes`, in dependency order, cut into batches of about
2589
    /// `push.batch-bytes` by what each change and its texts and blobs weigh.
2590
    /// There is always one, perhaps empty.
2591
    fn push_batches(&self, changes: &[ChangeHash]) -> Result<Vec<Vec<ChangeHash>>> {
2592
        let blob_len = |r: &ObjectRef| -> u64 {
2593
            self.store
2594
                .get_typed(r, ObjectType::Blob)
2595
                .ok()
2596
                .and_then(|b| Blob::decode(&b).ok())
2597
                .map_or(0, |b| b.length)
2598
        };
2599
        let (mut out, mut batch, mut bytes) = (Vec::new(), Vec::new(), 0u64);
2600
        for h in changes {
2601
            let raw = self.store.get_typed(h, ObjectType::Change)?;
2602
            let (c, texts) = Change::decode_skeleton(&raw, *h)?;
2603
            let mut weight = raw.len() as u64;
2604
            for t in &texts {
2605
                if let crate::objects::TextRef::Blob(r, _, _) = t {
2606
                    weight += blob_len(r);
2607
                }
2608
            }
2609
            for e in &c.edits {
2610
                if let Edit::Replace { new: Some(b), .. } = e {
2611
                    weight += blob_len(b);
2612
                }
2613
            }
2614
            if !batch.is_empty() && bytes + weight > self.config.push_batch_bytes {
2615
                out.push(std::mem::take(&mut batch));
2616
                bytes = 0;
2617
            }
2618
            batch.push(*h);
2619
            bytes += weight;
2620
        }
2621
        out.push(batch);
2622
        Ok(out)
2623
    }
2624
98286406 Andy McCall ↶ 2625
    pub fn push(&self, remote_name: &str, force: bool) -> Result<Vec<RefResult>> {
6d3eefbc Andy McCall ↶ 2626
        let bookmark = self.config.bookmark.clone();
7b9809e6 Andy McCall ↶ 2627
        self.push_bookmark(remote_name, &bookmark, force)
2628
    }
2629
2630
    /// `push` for a named bookmark (`oxy push -b NAME`).
75167672 Andy McCall 2631
    pub fn push_bookmark(
cb8b8082 Andy McCall ↶ 2632
        &self,
2633
        remote_name: &str,
2634
        bookmark: &str,
2635
        force: bool,
2636
    ) -> Result<Vec<RefResult>> {
2637
        self.push_bookmark_with(remote_name, bookmark, force, &mut |_| {})
2638
    }
2639
2640
    /// `push_bookmark`, saying how a long push is going: one line per
2641
    /// batch sent (oxygen#95).
c8d9059e Andy McCall ↶ 2642
    /// What a push may take the remote to hold already: the heads of the
2643
    /// bookmark pushed, and of every other bookmark it lists that are held
2644
    /// here, so a new bookmark made from main sends only what main lacks
2645
    /// (oxygen#142), not its whole history again.
2646
    pub(crate) fn remote_holds(
2647
        &self,
2648
        refs: &[crate::protocol::RemoteRef],
2649
        bookmark_heads: &[ChangeHash],
2650
    ) -> Vec<ChangeHash> {
2651
        let mut have: Vec<ChangeHash> = bookmark_heads.to_vec();
2652
        for h in refs.iter().flat_map(|r| &r.heads) {
2653
            if !have.contains(h) && self.store.has(h) {
2654
                have.push(*h);
2655
            }
2656
        }
2657
        have
2658
    }
2659
cb8b8082 Andy McCall ↶ 2660
    pub fn push_bookmark_with(
75167672 Andy McCall 2661
        &self,
2662
        remote_name: &str,
2663
        bookmark: &str,
2664
        force: bool,
cb8b8082 Andy McCall ↶ 2665
        progress: crate::store::Progress,
75167672 Andy McCall 2666
    ) -> Result<Vec<RefResult>> {
7b9809e6 Andy McCall ↶ 2667
        let (op_ref, op) = self.head_op()?;
2668
        if !op.view.bookmarks.contains_key(bookmark) {
2669
            return fail(format!("no bookmark {bookmark}"));
2670
        }
2671
        let bookmark = bookmark.to_string();
6d3eefbc Andy McCall ↶ 2672
        let local_heads = self.heads_of(&op.view, &bookmark)?;
2673
        // pre-push hook: may veto before the remote is contacted.
2674
        self.run_hook(
2675
            "pre-push",
75167672 Andy McCall 2676
            &format!(
2677
                "{{\"event\":\"pre-push\",\"remote\":{},\"bookmark\":{},\"heads\":{}}}",
2678
                json_str(remote_name),
2679
                json_str(&bookmark),
2680
                json_array(local_heads.iter().map(|h| json_str(&h.to_hex())))
2681
            ),
6d3eefbc Andy McCall ↶ 2682
        )?;
98286406 Andy McCall ↶ 2683
        let remote = self.open_named_remote(remote_name)?;
75167672 Andy McCall 2684
        self.check_remote_id(remote.as_ref(), remote_name, true)?;
83e0677b Andy McCall ↶ 2685
        let resp = remote.refs()?;
2686
        let local_policy = self.current_policy()?.map(|(_, sp)| sp);
28fc23d7 Andy McCall ↶ 2687
        let resp_policy_serial = resp.policy.as_ref().map(|p| p.policy.serial);
83e0677b Andy McCall ↶ 2688
        let policy_to_send = match (&local_policy, &resp.policy) {
2689
            (Some(l), Some(r)) if l.policy.serial > r.policy.serial => Some(l.clone()),
2690
            (Some(l), None) => Some(l.clone()),
2691
            (Some(l), Some(r)) if l.policy.serial < r.policy.serial => {
75167672 Andy McCall 2692
                return fail(format!(
2693
                    "{remote_name} has a newer policy (serial {}); fetch first",
2694
                    r.policy.serial
2695
                ));
83e0677b Andy McCall ↶ 2696
            }
2697
            _ => None,
2698
        };
2699
        let remote_has_policy = resp.policy.is_some() || policy_to_send.is_some();
2700
        if remote_has_policy && self.secret_key()?.is_none() {
2701
            return fail(format!("{remote_name} enforces a trust policy; pushing needs an identity (`oxy identity create`)"));
2702
        }
b63fc7a7 Andy McCall ↶ 2703
        let remote_tags: BTreeSet<String> = resp
2704
            .tags
2705
            .iter()
2706
            .chain(&resp.withdrawn)
2707
            .map(|t| t.name.clone())
2708
            .collect();
75167672 Andy McCall 2709
        let tags_to_send: Vec<crate::protocol::TagRef> = self
2710
            .tag_refs()?
2711
            .into_iter()
2712
            .filter(|t| !remote_tags.contains(&t.name))
b63fc7a7 Andy McCall ↶ 2713
            .collect();
2714
        // Tags withdrawn here that the remote still has, at the same state
2715
        // (oxygenhub#301).
2716
        let withdrawals_to_send: Vec<crate::protocol::TagRef> = self
2717
            .withdrawn_refs()?
2718
            .into_iter()
2719
            .filter(|w| {
2720
                resp.tags
2721
                    .iter()
2722
                    .any(|t| t.name == w.name && t.state == w.state)
2723
            })
75167672 Andy McCall 2724
            .collect();
83e0677b Andy McCall ↶ 2725
        let refs = resp.refs;
98286406 Andy McCall ↶ 2726
        let empty = self.store.put_state(&StateObject::new(vec![]))?;
75167672 Andy McCall 2727
        let (remote_state, remote_heads, last_serial) =
2728
            match refs.iter().find(|r| r.name == bookmark) {
2729
                Some(r) => (r.state, r.heads.clone(), r.last_serial),
2730
                None => (empty, vec![], None),
2731
            };
2732
        let new_state = self
2733
            .store
2734
            .put_state(&StateObject::new(local_heads.clone()))?;
83e0677b Andy McCall ↶ 2735
        // A push that carries a policy re-attests the bookmark even when its
2736
        // state is unchanged, so the current attestation is always by a key
2737
        // the current policy authorizes (a revoked key's attestation would
2738
        // otherwise be refused by every later fetch).
b63fc7a7 Andy McCall ↶ 2739
        if new_state == remote_state
2740
            && policy_to_send.is_none()
2741
            && tags_to_send.is_empty()
2742
            && withdrawals_to_send.is_empty()
2743
        {
75167672 Andy McCall 2744
            return Ok(vec![RefResult {
2745
                name: bookmark,
2746
                ok: true,
2747
                reason: "already up to date".into(),
2748
            }]);
98286406 Andy McCall ↶ 2749
        }
f3811eb2 Andy McCall ↶ 2750
        // A shallow repository knows the changes on its boundary without
2751
        // holding them: a depth-0 clone's are the heads it cloned (#137).
2752
        let boundary = self.boundary();
98286406 Andy McCall ↶ 2753
        for h in &remote_heads {
f3811eb2 Andy McCall ↶ 2754
            if !self.store.has(h) && !boundary.contains(h) {
75167672 Andy McCall 2755
                return fail(format!(
3e5772cc Andy McCall ↶ 2756
                    "the remote has change {} which you do not; pull first (`oxy pull {remote_name}`), then push again",
75167672 Andy McCall 2757
                    h.short(12)
2758
                ));
98286406 Andy McCall ↶ 2759
            }
2760
        }
2761
        if !force {
2762
            let closure = self.closure_in_store(&local_heads)?;
f3811eb2 Andy McCall ↶ 2763
            if let Some(h) = remote_heads
2764
                .iter()
2765
                .find(|h| !closure.contains(h) && !boundary.contains(h))
2766
            {
98286406 Andy McCall ↶ 2767
                return fail(format!("not a fast-forward: remote head {} is not in your state; pull first or use --force", h.short(12)));
2768
            }
2769
        }
c8d9059e Andy McCall ↶ 2770
        let to_send = self.negotiate(&self.remote_holds(&refs, &remote_heads), &local_heads)?;
98286406 Andy McCall ↶ 2771
        let key = self.secret_key()?.map(|(k, _)| k);
2772
        if let Some(key) = key {
05e71ed6 Andy McCall ↶ 2773
            // Seal this author's changes that carry no valid authenticator,
cb8b8082 Andy McCall ↶ 2774
            // and adopt anonymous ones (zero author, spec §14.4). The seals
2775
            // are stored, so each batch's bundle carries those it needs.
98286406 Andy McCall ↶ 2776
            let mut unsigned = Vec::new();
2777
            for h in &to_send {
cb8b8082 Andy McCall ↶ 2778
                let c = self.store.get_change_meta(h)?;
a2387d54 Andy McCall ↶ 2779
                if self.seals_for_author(&c.author, &key)
75167672 Andy McCall 2780
                    && !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. })
2781
                {
98286406 Andy McCall ↶ 2782
                    unsigned.push(*h);
2783
                }
2784
            }
2785
            for chunk in unsigned.chunks(crate::attest::MAX_SEAL_HASHES) {
cb8b8082 Andy McCall ↶ 2786
                self.sign_attest(&Attest::seal(self.id()?, chunk.to_vec()))?;
98286406 Andy McCall ↶ 2787
            }
cb8b8082 Andy McCall ↶ 2788
        }
2789
        // A long history goes in batches, in dependency order, so neither
2790
        // side holds it whole and no request passes the remote's limit:
2791
        // all but the last carry objects alone; the last moves the bookmark
2792
        // (oxygen#95).
2793
        let batches = self.push_batches(&to_send)?;
2794
        let (last, earlier) = batches.split_last().expect("at least one batch");
2795
        for (i, batch) in earlier.iter().enumerate() {
2796
            progress(&format!(
2797
                "part {} of {}: {} changes",
2798
                i + 1,
2799
                batches.len(),
2800
                batch.len()
2801
            ));
d3b27090 Andy McCall ↶ 2802
            let bundle = self.send_objects_ahead(
2803
                remote.as_ref(),
2804
                remote_name,
2805
                self.bundle_for(batch)?,
2806
                &mut *progress,
2807
            )?;
2808
            let results = remote.push(bundle, vec![], None, vec![])?;
cb8b8082 Andy McCall ↶ 2809
            match results.iter().find(|r| r.name == OBJECTS_RESULT) {
2810
                Some(r) if r.ok => {}
2811
                Some(r) => {
2812
                    return fail(format!(
2813
                        "{remote_name} refused part {} of the push: {}",
2814
                        i + 1,
2815
                        r.reason
2816
                    ))
2817
                }
2818
                None => {
2819
                    return fail(format!(
2820
                        "{remote_name} does not take a push in parts, and this one is too long for one: update the server's oxy"
2821
                    ))
2822
                }
2823
            }
2824
        }
2825
        if batches.len() > 1 {
2826
            progress(&format!(
2827
                "part {} of {}: {} changes",
2828
                batches.len(),
2829
                batches.len(),
2830
                last.len()
2831
            ));
98286406 Andy McCall ↶ 2832
        }
d3b27090 Andy McCall ↶ 2833
        let mut bundle = self.send_objects_ahead(
2834
            remote.as_ref(),
2835
            remote_name,
2836
            self.bundle_for(last)?,
2837
            &mut *progress,
2838
        )?;
75167672 Andy McCall 2839
        bundle.objects.push((
2840
            ObjectType::State,
2841
            StateObject::new(local_heads.clone()).encode(),
2842
        ));
98286406 Andy McCall ↶ 2843
        let attestation = match key {
2844
            Some(_) => Some(self.sign_attest(&Attest {
2845
                repo: self.id()?,
2846
                hlc: Hlc::now(),
75167672 Andy McCall 2847
                body: AttestBody::Ref {
2848
                    name: bookmark.clone(),
2849
                    old_state: remote_state,
2850
                    new_state,
f8c69576 Andy McCall ↶ 2851
                    serial: next_serial(last_serial),
75167672 Andy McCall 2852
                },
98286406 Andy McCall ↶ 2853
            })?),
2854
            None => None,
2855
        };
68a92484 Andy McCall ↶ 2856
        // Whoever signed the move, and whom a delegate acts for, travel with
2857
        // it: a receiver that has never seen a change by this delegate could
2858
        // not otherwise check its signature (oxygenhub#129).
2859
        if attestation.is_some() {
2860
            for k in [self.signing_key(), self.config.author_key]
2861
                .into_iter()
2862
                .flatten()
2863
            {
2864
                if let Ok(id) = self.identity(&k) {
2865
                    let body = id.encode();
2866
                    if !bundle
2867
                        .objects
2868
                        .iter()
2869
                        .any(|(t, b)| *t == ObjectType::Identity && *b == body)
2870
                    {
2871
                        bundle.objects.push((ObjectType::Identity, body));
2872
                    }
2873
                }
2874
            }
f0a517d2 Andy McCall ↶ 2875
            // A short-lived signing key's certificate goes with it (§14.7).
2876
            let signers: Vec<KeyId> = self.signing_key().into_iter().collect();
2877
            self.attach_certificates_for(&mut bundle, &signers)?;
68a92484 Andy McCall ↶ 2878
        }
98286406 Andy McCall ↶ 2879
        let serial = attestation.as_ref().and_then(|a| match a.attest.body {
2880
            AttestBody::Ref { serial, .. } => Some(serial),
2881
            _ => None,
2882
        });
83e0677b Andy McCall ↶ 2883
        let sent_serial = policy_to_send.as_ref().map(|p| p.policy.serial);
132b884b Andy McCall ↶ 2884
        // Tag states must be present on the remote: bundle their changes too.
2885
        for t in &tags_to_send {
2886
            let extra = self.negotiate(&remote_heads, &t.heads)?;
75167672 Andy McCall 2887
            let extra: Vec<ChangeHash> =
2888
                extra.into_iter().filter(|h| !to_send.contains(h)).collect();
132b884b Andy McCall ↶ 2889
            if !extra.is_empty() {
2890
                let more = self.bundle_for(&extra)?;
2891
                bundle.changes.extend(more.changes);
2c75672f Andy McCall ↶ 2892
                bundle.hashes.extend(more.hashes);
132b884b Andy McCall ↶ 2893
                bundle.attests.extend(more.attests);
2894
                bundle.objects.extend(more.objects);
2895
            }
75167672 Andy McCall 2896
            bundle.objects.push((
2897
                ObjectType::State,
2898
                StateObject::new(t.heads.clone()).encode(),
2899
            ));
132b884b Andy McCall ↶ 2900
        }
d8524a9f Andy McCall ↶ 2901
        // A tag's signer travels with it, as the bookmark's does: a tag made
2902
        // by `oxy tag create` under a short-lived key was signed by a key the
2903
        // receiver has never seen, so its identity and certificate go too.
2904
        let tag_signers: Vec<KeyId> = tags_to_send
2905
            .iter()
b63fc7a7 Andy McCall ↶ 2906
            .chain(&withdrawals_to_send)
d8524a9f Andy McCall ↶ 2907
            .filter_map(|t| t.attestation.as_ref())
2908
            .flat_map(|a| a.sigs.iter().map(|s| s.key))
2909
            .collect();
2910
        for k in &tag_signers {
2911
            if let Ok(id) = self.identity(k) {
2912
                let body = id.encode();
2913
                if !bundle
2914
                    .objects
2915
                    .iter()
2916
                    .any(|(t, b)| *t == ObjectType::Identity && *b == body)
2917
                {
2918
                    bundle.objects.push((ObjectType::Identity, body));
2919
                }
2920
            }
2921
        }
2922
        if !tag_signers.is_empty() {
2923
            self.attach_certificates_for(&mut bundle, &tag_signers)?;
2924
        }
28fc23d7 Andy McCall ↶ 2925
        if let Some(l) = &policy_to_send {
2926
            let remote_serial = resp_policy_serial;
75167672 Andy McCall 2927
            bundle.policies = self
2928
                .policy_history()?
2929
                .into_iter()
2930
                .filter(|sp| {
2931
                    remote_serial.is_none_or(|r| sp.policy.serial > r)
2932
                        && sp.policy.serial < l.policy.serial
2933
                })
2934
                .collect();
28fc23d7 Andy McCall ↶ 2935
        }
75167672 Andy McCall 2936
        let results = remote.push(
2937
            bundle,
2938
            vec![RefUpdate {
2939
                name: bookmark.clone(),
2940
                old_state: remote_state,
2941
                new_state,
2942
                force,
2943
                attestation,
2944
            }],
2945
            policy_to_send,
b63fc7a7 Andy McCall ↶ 2946
            tags_to_send
2947
                .into_iter()
2948
                .chain(withdrawals_to_send)
2949
                .collect(),
75167672 Andy McCall 2950
        )?;
2951
        if let (Some(serial), true) = (
2952
            sent_serial,
2953
            results.iter().any(|r| r.ok && r.name == "policy"),
2954
        ) {
83e0677b Andy McCall ↶ 2955
            self.record_policy_serial(remote_name, serial)?;
2956
        }
2957
        if results.iter().any(|r| r.ok && r.name == bookmark) {
98286406 Andy McCall ↶ 2958
            let mut view = op.view.clone();
75167672 Andy McCall 2959
            view.remotes
2960
                .entry(remote_name.to_string())
2961
                .or_default()
2962
                .insert(bookmark.clone(), new_state);
98286406 Andy McCall ↶ 2963
            self.record_op(&[op_ref], &format!("oxy push {remote_name}"), view)?;
2964
            if let Some(s) = serial {
2965
                self.record_serial(remote_name, &bookmark, s)?;
2966
            }
2967
        }
2968
        Ok(results)
2969
    }
2970
2971
    /// Clone a remote into `dest`: same repository ID, `origin` remote,
2972
    /// bookmark set to the remote's, working copy checked out.
2973
    pub fn clone(url: &str, dest: &Path) -> Result<Repo> {
7421309f Andy McCall ↶ 2974
        Self::clone_with(url, dest, None, &[])
b3c5a59a Andy McCall ↶ 2975
    }
2976
7421309f Andy McCall ↶ 2977
    /// Clone, leaving blobs above `max_blob` bytes to be fetched on first
2978
    /// read, and materializing only `sparse` paths (which implies lazy blobs
2979
    /// unless a size is given, so out-of-scope binaries are never fetched).
75167672 Andy McCall 2980
    pub fn clone_with(
2981
        url: &str,
2982
        dest: &Path,
2983
        max_blob: Option<u64>,
2984
        sparse: &[String],
2985
    ) -> Result<Repo> {
2863456a Andy McCall ↶ 2986
        Self::clone_full(url, dest, max_blob, sparse, None)
2987
    }
2988
2989
    /// Shallow clone (spec §17): a checkpoint of the bookmark's heads plus the
2990
    /// changes within `depth` steps of them; the rest is the boundary.
75167672 Andy McCall 2991
    pub fn clone_shallow(
2992
        url: &str,
2993
        dest: &Path,
2994
        depth: u32,
2995
        max_blob: Option<u64>,
2996
        sparse: &[String],
2997
    ) -> Result<Repo> {
2863456a Andy McCall ↶ 2998
        Self::clone_full(url, dest, max_blob, sparse, Some(depth))
2999
    }
3000
75167672 Andy McCall 3001
    fn clone_full(
3002
        url: &str,
3003
        dest: &Path,
3004
        max_blob: Option<u64>,
3005
        sparse: &[String],
3006
        depth: Option<u32>,
3007
    ) -> Result<Repo> {
d36ba2f7 Andy McCall ↶ 3008
        Ok(Self::clone_reporting(url, dest, max_blob, sparse, depth)?.0)
3009
    }
3010
3011
    /// Clone as [`Repo::clone_with`] (or [`Repo::clone_shallow`] with a
3012
    /// `depth`), also returning the bookmarks the fetch left out because
3013
    /// their attestation could not be verified ([`FetchReport::skipped`]),
3014
    /// for the caller to warn about. The bookmark checked out is never one
3015
    /// of them: that fails the clone.
3016
    pub fn clone_reporting(
3017
        url: &str,
3018
        dest: &Path,
3019
        max_blob: Option<u64>,
3020
        sparse: &[String],
3021
        depth: Option<u32>,
3022
    ) -> Result<(Repo, Vec<(String, String)>)> {
ff1f62fe Andy McCall ↶ 3023
        let options = crate::credentials::Credentials::user().options_for(url);
3024
        Self::clone_using(url, dest, max_blob, sparse, depth, &options)
a07f60a3 Andy McCall ↶ 3025
    }
3026
ff1f62fe Andy McCall ↶ 3027
    /// Clone a subrepo with the credentials the parent would use for its
3028
    /// hub (see [`Repo::credentials_for`]). Nothing is recorded in the
3029
    /// subrepo: it finds the same ones again through its parent.
a07f60a3 Andy McCall ↶ 3030
    pub(crate) fn clone_as_subrepo(&self, url: &str, dest: &Path) -> Result<Repo> {
3031
        let options = self.credentials_for(url);
d36ba2f7 Andy McCall ↶ 3032
        Ok(Self::clone_using(url, dest, None, &[], None, &options)?.0)
a07f60a3 Andy McCall ↶ 3033
    }
3034
3035
    /// The name of a remote of this repository on the same hub (scheme,
3036
    /// host and port) as `url`.
3037
    fn remote_on_same_hub(&self, url: &str) -> Option<String> {
3038
        let origin = |u: &str| -> Option<String> {
3039
            let (scheme, rest) = u.split_once("://")?;
9f9e72b9 Andy McCall ↶ 3040
            if !matches!(scheme, "http" | "https") {
3041
                return None;
3042
            }
a07f60a3 Andy McCall ↶ 3043
            let host = rest.split('/').next()?;
3044
            Some(format!(
3045
                "{}://{}",
3046
                scheme.to_ascii_lowercase(),
3047
                host.to_ascii_lowercase()
3048
            ))
3049
        };
3050
        let want = origin(url)?;
3051
        self.config
3052
            .remotes
3053
            .iter()
3054
            .find(|(_, u)| origin(u).as_deref() == Some(want.as_str()))
3055
            .map(|(name, _)| name.clone())
3056
    }
3057
3058
    /// The token and CA to reach `url` with: those of a remote of this
3059
    /// repository on the same hub (scheme, host and port), else the
ff1f62fe Andy McCall ↶ 3060
    /// environment's (`OXY_TOKEN`, `OXY_CA`), the user's credential store
3061
    /// and enclosing checkouts', as a clone would use them.
a07f60a3 Andy McCall ↶ 3062
    pub(crate) fn credentials_for(&self, url: &str) -> RemoteOptions {
3063
        match self.remote_on_same_hub(url) {
3064
            Some(name) => RemoteOptions {
3065
                repo_id: None,
9f9e72b9 Andy McCall ↶ 3066
                cache_dir: None,
a07f60a3 Andy McCall ↶ 3067
                ..self.remote_options(&name)
3068
            },
ff1f62fe Andy McCall ↶ 3069
            None => {
3070
                let mut options = RemoteOptions::default();
3071
                self.fill_credentials(url, &mut options);
3072
                options
3073
            }
a07f60a3 Andy McCall ↶ 3074
        }
3075
    }
3076
3077
    fn clone_using(
3078
        url: &str,
3079
        dest: &Path,
3080
        max_blob: Option<u64>,
3081
        sparse: &[String],
3082
        depth: Option<u32>,
3083
        options: &RemoteOptions,
d36ba2f7 Andy McCall ↶ 3084
    ) -> Result<(Repo, Vec<(String, String)>)> {
a07f60a3 Andy McCall ↶ 3085
        let remote = open_remote(url, options)?;
98286406 Andy McCall ↶ 3086
        let id = remote.repo_id()?;
75167672 Andy McCall 3087
        if id == crate::protocol::UNBOUND_ID {
05e71ed6 Andy McCall ↶ 3088
            return fail(format!(
3089
                "{url} is empty and unbound: push a repository to it first"
3090
            ));
75167672 Andy McCall 3091
        }
98286406 Andy McCall ↶ 3092
        let mut repo = Repo::init_with_id(dest, id)?;
827281ee Andy McCall ↶ 3093
        repo.config
3094
            .set("remote.origin.url", &crate::protocol::stored_url(url))?;
b3c5a59a Andy McCall ↶ 3095
        if let Some(m) = max_blob {
3096
            repo.config.set("filter.max-blob", &m.to_string())?;
7421309f Andy McCall ↶ 3097
        } else if !sparse.is_empty() {
3098
            repo.config.set("filter.max-blob", "0")?;
3099
        }
3100
        if !sparse.is_empty() {
3101
            repo.config.set("filter.paths", &sparse.join(","))?;
b3c5a59a Andy McCall ↶ 3102
        }
98286406 Andy McCall ↶ 3103
        repo.save_config()?;
d36ba2f7 Andy McCall ↶ 3104
        let skipped = if let Some(depth) = depth {
2863456a Andy McCall ↶ 3105
            repo.fetch_shallow("origin", depth)?;
d36ba2f7 Andy McCall ↶ 3106
            Vec::new()
2863456a Andy McCall ↶ 3107
        } else {
d36ba2f7 Andy McCall ↶ 3108
            repo.fetch("origin")?.skipped
3109
        };
98286406 Andy McCall ↶ 3110
        let (op_ref, op) = repo.head_op()?;
d36ba2f7 Andy McCall ↶ 3111
        // The remote's default bookmark, else ours (main), else its only
3112
        // one; a bookmark the fetch skipped counts, so the clone fails
3113
        // rather than check out another (oxygenhub#266).
3114
        let mut theirs = op.view.remotes.get("origin").cloned().unwrap_or_default();
3115
        for (name, _) in &skipped {
3116
            theirs.insert(name.clone(), ObjectRef::ZERO);
3117
        }
9f9e72b9 Andy McCall ↶ 3118
        // Asked of the remote as now configured, so a store-backed one
3119
        // answers from the mirror the fetch just filled.
3120
        let advertised = repo
3121
            .open_named_remote("origin")
3122
            .ok()
3123
            .and_then(|r| r.refs().ok())
3124
            .and_then(|r| r.default_bookmark);
14edbaaf Andy McCall ↶ 3125
        let bookmark = advertised
3126
            .filter(|b| theirs.contains_key(b))
3127
            .or_else(|| {
3128
                theirs
3129
                    .contains_key(&repo.config.bookmark)
3130
                    .then(|| repo.config.bookmark.clone())
3131
            })
3132
            .or_else(|| {
3133
                (theirs.len() == 1)
3134
                    .then(|| theirs.keys().next().cloned())
3135
                    .flatten()
3136
            })
3137
            .unwrap_or_else(|| repo.config.bookmark.clone());
d36ba2f7 Andy McCall ↶ 3138
        if let Some((_, why)) = skipped.iter().find(|(name, _)| *name == bookmark) {
3139
            return fail(format!("cannot check out {bookmark}: {why}"));
3140
        }
14edbaaf Andy McCall ↶ 3141
        let initial = repo.config.bookmark.clone();
3142
        if bookmark != repo.config.bookmark {
3143
            fs::create_dir_all(repo.workspace_bookmark_path().parent().unwrap())?;
3144
            write_atomic(&repo.workspace_bookmark_path(), bookmark.as_bytes())?;
3145
            repo.config.bookmark = bookmark.clone();
3146
        }
3147
        if let Some(state) = theirs.get(&bookmark).copied() {
98286406 Andy McCall ↶ 3148
            let heads = repo.store.get_state(&state)?.heads;
3149
            let mut view = op.view.clone();
14edbaaf Andy McCall ↶ 3150
            // The empty bookmark `init` made goes when another is checked out.
3151
            if initial != bookmark && repo.heads_of(&view, &initial)?.is_empty() {
3152
                view.bookmarks.remove(&initial);
3153
            }
98286406 Andy McCall ↶ 3154
            view.bookmarks.insert(bookmark, state);
3155
            repo.record_op(&[op_ref], &format!("oxy clone {url}"), view)?;
3156
            repo.checkout(&heads, false, "oxy clone (checkout)")?;
3157
        }
d36ba2f7 Andy McCall ↶ 3158
        Ok((repo, skipped))
f8c69576 Andy McCall ↶ 3159
    }
3160
}
3161
3162
/// The serial of a reference's next attestation: one more than the last the
3163
/// remote reports, or, for a bookmark the remote does not have, the clock in
3164
/// milliseconds, so a bookmark created again after a deletion goes past the
3165
/// deletion's serial the remote keeps.
3166
fn next_serial(last: Option<u64>) -> u64 {
3167
    match last {
3168
        Some(l) => l + 1,
3169
        None => Hlc::now().wall_ms.max(1),
98286406 Andy McCall ↶ 3170
    }
3171
}