OXYGENHUB
# 2863456a0f6d oxygen / crates / oxy-core / src / repo / sync.rs
41.3 KiB · Rust · last changed Lazy blobs: clone --max-blob / filter.max-blob, absent-blob registry fetched on first read, objects endpoint (spec 0.6e) 1 month ago source blame history raw
1 
//! Serving and talking to remotes (spec §10.2, §16).
2
3 
use super::*;
4
5 
/// What a fetch brought in.
6 
#[derive(Debug, Clone, PartialEq, Eq, Default)]
7 
pub struct FetchReport {
8 
    /// Bookmark name, its heads, and how many changes were new.
9 
    pub refs: Vec<(String, Vec<ChangeHash>, usize)>,
10 
    /// The remote has never presented a policy, so nothing was verified
11 
    /// against one (spec §14.5: warn loudly).
12 
    pub unverified: bool,
13 
}
14
15 
#[derive(Debug, Clone, PartialEq, Eq)]
16 
pub struct PullReport {
17 
    pub heads: Vec<ChangeHash>,
18 
    pub changed: bool,
19 
    pub checked_out: bool,
20 
}
21
22 
impl Repo {
23 
    // ------------------------------------------------------------ serving (spec §16)
24
25 
    fn ref_attest_path(&self, bookmark: &str) -> PathBuf {
26 
        self.oxy.join("attest").join(bookmark)
27 
    }
28
29 
    /// The latest reference attestation stored for a bookmark.
30 
    pub fn latest_ref_attest(&self, bookmark: &str) -> Result<Option<(ObjectRef, SignedAttest)>> {
31 
        match fs::read_to_string(self.ref_attest_path(bookmark)) {
32 
            Ok(h) => {
33 
                let r = ObjectRef::from_hex(h.trim()).ok_or_else(|| RepoError::Msg("corrupt attestation pointer".into()))?;
34 
                Ok(Some((r, SignedAttest { attest: self.store.get_attest(&r)?, sigs: self.store.signatures(&r)? })))
35 
            }
36 
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
37 
            Err(e) => Err(e.into()),
38 
        }
39 
    }
40
41 
    fn set_ref_attest(&self, bookmark: &str, r: &ObjectRef) -> Result<()> {
42 
        fs::create_dir_all(self.oxy.join("attest"))?;
43 
        write_atomic(&self.ref_attest_path(bookmark), r.to_hex().as_bytes())
44 
    }
45
46 
    pub fn remote_refs(&self) -> Result<RefsResponse> {
47 
        let view = self.view()?;
48 
        let mut out = Vec::new();
49 
        for (name, state) in &view.bookmarks {
50 
            let latest = self.latest_ref_attest(name)?.map(|(_, a)| a);
51 
            let last_serial = latest.as_ref().and_then(|a| match a.attest.body {
52 
                AttestBody::Ref { serial, .. } => Some(serial),
53 
                _ => None,
54 
            });
55 
            let attestation = latest.filter(|a| matches!(&a.attest.body, AttestBody::Ref { new_state, .. } if new_state == state));
56 
            out.push(RemoteRef { name: name.clone(), state: *state, heads: self.store.get_state(state)?.heads, attestation, last_serial });
57 
        }
58 
        Ok(RefsResponse { refs: out, policy: self.current_policy()?.map(|(_, sp)| sp) })
59 
    }
60
61 
    /// Every change reachable from `heads` that is in the store.
62 
    fn closure_in_store(&self, heads: &[ChangeHash]) -> Result<BTreeSet<ChangeHash>> {
63 
        let mut seen = BTreeSet::new();
64 
        let mut stack: Vec<ChangeHash> = heads.to_vec();
65 
        while let Some(h) = stack.pop() {
66 
            if seen.contains(&h) || !self.store.has(&h) {
67 
                continue;
68 
            }
69 
            seen.insert(h);
70 
            stack.extend(self.store.get_change(&h)?.deps);
71 
        }
72 
        Ok(seen)
73 
    }
74
75 
    /// The changes within `depth` dependency steps of `heads` (spec §17),
76 
    /// nearest first; depth 1 is the heads themselves.
77 
    pub fn shallow_changes(&self, heads: &[ChangeHash], depth: u32) -> Result<Vec<ChangeHash>> {
78 
        let mut out = Vec::new();
79 
        let mut seen: BTreeSet<ChangeHash> = BTreeSet::new();
80 
        let mut layer: Vec<ChangeHash> = heads.to_vec();
81 
        for _ in 0..depth.max(1) {
82 
            let mut next = Vec::new();
83 
            for h in layer {
84 
                if !seen.insert(h) || !self.store.has(&h) {
85 
                    continue;
86 
                }
87 
                let c = self.store.get_change(&h)?;
88 
                next.extend(c.deps.iter().copied());
89 
                out.push(h);
90 
            }
91 
            if next.is_empty() {
92 
                break;
93 
            }
94 
            layer = next;
95 
        }
96 
        out.reverse();
97 
        Ok(out)
98 
    }
99
100 
    fn shallow_path(&self) -> PathBuf {
101 
        self.oxy.join("shallow")
102 
    }
103
104 
    /// The shallow boundary: dependencies this repository does not hold (spec §17).
105 
    pub fn boundary(&self) -> BTreeSet<ChangeHash> {
106 
        fs::read_to_string(self.shallow_path()).map(|t| t.lines().filter_map(ObjectRef::from_hex).collect()).unwrap_or_default()
107 
    }
108
109 
    pub fn is_shallow(&self) -> bool {
110 
        !self.boundary().is_empty()
111 
    }
112
113 
    fn write_boundary(&self, boundary: &BTreeSet<ChangeHash>) -> Result<()> {
114 
        if boundary.is_empty() {
115 
            let _ = fs::remove_file(self.shallow_path());
116 
            return Ok(());
117 
        }
118 
        let text: String = boundary.iter().map(|h| format!("{}\n", h.to_hex())).collect();
119 
        write_atomic(&self.shallow_path(), text.as_bytes())
120 
    }
121
122 
    /// Recompute the boundary from what is stored: every dependency of a
123 
    /// stored change that is not itself stored.
124 
    fn refresh_boundary(&self, imported: &[ChangeHash]) -> Result<()> {
125 
        let mut boundary = self.boundary();
126 
        boundary.retain(|h| !self.store.has(h));
127 
        for h in imported {
128 
            for d in self.store.get_change(h)?.deps {
129 
                if !self.store.has(&d) {
130 
                    boundary.insert(d);
131 
                }
132 
            }
133 
        }
134 
        self.write_boundary(&boundary)
135 
    }
136
137 
    /// Fetch `depth` more levels of history below the shallow boundary.
138 
    pub fn deepen(&self, remote_name: &str, depth: u32) -> Result<usize> {
139 
        let boundary = self.boundary();
140 
        if boundary.is_empty() {
141 
            return Ok(0);
142 
        }
143 
        let remote = self.open_named_remote(remote_name)?;
144 
        let wanted = remote.shallow(&boundary.iter().copied().collect::<Vec<_>>(), depth)?;
145 
        let bundle = remote.fetch(&wanted, self.config.max_blob)?;
146 
        let imported = self.import_bundle_shallow(&bundle, false)?;
147 
        if self.config.max_blob.is_some() {
148 
            self.register_missing_blobs(&imported, remote_name)?;
149 
        }
150 
        self.refresh_boundary(&imported)?;
151 
        Ok(imported.len())
152 
    }
153
154 
    /// `closure(want) \ closure(have)`, dependencies first (spec §16.2).
155 
    pub fn negotiate(&self, have: &[ChangeHash], want: &[ChangeHash]) -> Result<Vec<ChangeHash>> {
156 
        let have = self.closure_in_store(have)?;
157 
        // In a shallow repository the walk stops at the boundary, whose
158 
        // changes the remote is assumed to hold.
159 
        let boundary = self.boundary();
160 
        let mut out = Vec::new();
161 
        let mut done: BTreeSet<ChangeHash> = have.clone();
162 
        done.extend(boundary.iter().copied());
163 
        // Iterative post-order DFS.
164 
        let mut stack: Vec<(ChangeHash, bool)> = want.iter().rev().map(|h| (*h, false)).collect();
165 
        while let Some((h, expanded)) = stack.pop() {
166 
            if done.contains(&h) {
167 
                continue;
168 
            }
169 
            if !self.store.has(&h) {
170 
                return fail(format!("change {} is not in this repository", h.short(12)));
171 
            }
172 
            if expanded {
173 
                done.insert(h);
174 
                out.push(h);
175 
                continue;
176 
            }
177 
            stack.push((h, true));
178 
            for d in self.store.get_change(&h)?.deps {
179 
                if !done.contains(&d) {
180 
                    stack.push((d, false));
181 
                }
182 
            }
183 
        }
184 
        Ok(out)
185 
    }
186
187 
    /// Bundle changes (in the given dependency order) with their signatures,
188 
    /// seals, identities, blobs, chunks.
189 
    pub fn bundle_for(&self, changes: &[ChangeHash]) -> Result<Bundle> {
190 
        self.bundle_for_filtered(changes, None)
191 
    }
192
193 
    /// Objects by reference for the `objects` endpoint; a blob brings its chunks.
194 
    pub fn objects_for(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>> {
195 
        let mut out = Vec::new();
196 
        for r in refs {
197 
            let (ty, body) = match self.store.get(r) {
198 
                Ok(x) => x,
199 
                Err(StoreError::NotFound(_)) => continue,
200 
                Err(e) => return Err(e.into()),
201 
            };
202 
            if ty == ObjectType::Blob {
203 
                for c in Blob::decode(&body)?.chunks {
204 
                    if let Ok((cty, cb)) = self.store.get(&c) {
205 
                        out.push((cty, cb));
206 
                    }
207 
                }
208 
            }
209 
            out.push((ty, body));
210 
        }
211 
        Ok(out)
212 
    }
213
214 
    /// Like `bundle_for`, leaving out blobs larger than `max_blob` bytes
215 
    /// (and their chunks) so the receiver fetches them lazily (spec §13.5).
216 
    pub fn bundle_for_filtered(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle> {
217 
        let mut bundle = Bundle::default();
218 
        let mut objects: BTreeSet<ObjectRef> = BTreeSet::new();
219 
        let mut seals: BTreeSet<ObjectRef> = BTreeSet::new();
220 
        let add_object = |r: &ObjectRef, objects: &mut BTreeSet<ObjectRef>| {
221 
            if !r.is_zero() && self.store.has(r) {
222 
                objects.insert(*r);
223 
            }
224 
        };
225 
        for h in changes {
226 
            let c = self.store.get_change(h)?;
227 
            let sigs = self.store.signatures(h)?;
228 
            add_object(&c.author, &mut objects);
229 
            for s in &sigs {
230 
                add_object(&s.key, &mut objects);
231 
            }
232 
            for e in &c.edits {
233 
                if let Edit::Replace { old, new, .. } = e {
234 
                    for b in [old, new].into_iter().flatten() {
235 
                        if self.store.has(b) {
236 
                            if let Ok(blob) = Blob::decode(&self.store.get_typed(b, ObjectType::Blob)?) {
237 
                                if max_blob.is_some_and(|m| blob.length > m) {
238 
                                    continue;
239 
                                }
240 
                                objects.insert(*b);
241 
                                objects.extend(blob.chunks.iter().copied());
242 
                            }
243 
                        }
244 
                    }
245 
                }
246 
            }
247 
            for s in self.store.seals_for(h)? {
248 
                seals.insert(s);
249 
            }
250 
            bundle.changes.push((c, sigs));
251 
        }
252 
        for r in seals {
253 
            let attest = self.store.get_attest(&r)?;
254 
            let sigs = self.store.signatures(&r)?;
255 
            for s in &sigs {
256 
                add_object(&s.key, &mut objects);
257 
            }
258 
            bundle.attests.push(SignedAttest { attest, sigs });
259 
        }
260 
        for r in objects {
261 
            let (ty, body) = self.store.get(&r)?;
262 
            bundle.objects.push((ty, body));
263 
        }
264 
        Ok(bundle)
265 
    }
266
267 
    /// Store a bundle's contents. With `strict`, every change must
268 
    /// authenticate (signature or seal by its author or a delegate).
269 
    pub fn import_bundle(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
270 
        self.import_bundle_inner(bundle, strict, false)
271 
    }
272
273 
    /// Like `import_bundle`, accepting changes whose dependencies are absent
274 
    /// (they join the shallow boundary).
275 
    pub fn import_bundle_shallow(&self, bundle: &Bundle, strict: bool) -> Result<Vec<ChangeHash>> {
276 
        self.import_bundle_inner(bundle, strict, true)
277 
    }
278
279 
    fn import_bundle_inner(&self, bundle: &Bundle, strict: bool, shallow: bool) -> Result<Vec<ChangeHash>> {
280 
        for (ty, body) in &bundle.objects {
281 
            match ty {
282 
                ObjectType::Chunk | ObjectType::Blob | ObjectType::Identity | ObjectType::State | ObjectType::Checkpoint => {
283 
                    self.store.put(*ty, body)?;
284 
                }
285 
                other => return fail(format!("bundle may not carry {} objects", other.name())),
286 
            }
287 
        }
288 
        for sa in &bundle.attests {
289 
            let r = self.store.put_attest(&sa.attest)?;
290 
            for sig in &sa.sigs {
291 
                self.store.put_signature(&r, sig)?;
292 
            }
293 
            if let AttestBody::Seal { hashes } = &sa.attest.body {
294 
                for h in hashes {
295 
                    self.store.index_seal(h, &r)?;
296 
                }
297 
            }
298 
        }
299 
        let incoming: BTreeSet<ChangeHash> = bundle.changes.iter().map(|(c, _)| c.hash()).collect();
300 
        let boundary = self.boundary();
301 
        let mut imported = Vec::new();
302 
        for (c, sigs) in &bundle.changes {
303 
            let h = c.hash();
304 
            for d in &c.deps {
305 
                if !self.store.has(d) && !incoming.contains(d) && !shallow && !boundary.contains(d) {
306 
                    return fail(format!("change {} depends on {}, which is neither stored nor in the bundle", h.short(12), d.short(12)));
307 
                }
308 
            }
309 
            self.store.put_change(c)?;
310 
            for sig in sigs {
311 
                self.store.put_signature(&h, sig)?;
312 
            }
313 
            imported.push(h);
314 
        }
315 
        if strict {
316 
            for h in &imported {
317 
                if !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. }) {
318 
                    return fail(format!("change {} is not authenticated by its author", h.short(12)));
319 
                }
320 
            }
321 
        }
322 
        Ok(imported)
323 
    }
324
325 
    /// Verify a reference attestation against an update it claims to describe.
326 
    /// `allow_equal` lets a client re-read the attestation it already has; a
327 
    /// server receiving a push requires a strictly greater serial.
328 
    fn check_ref_attest(&self, sa: &SignedAttest, name: &str, old_state: &ObjectRef, new_state: &ObjectRef, last_serial: Option<u64>, allow_equal: bool) -> std::result::Result<u64, String> {
329 
        let AttestBody::Ref { name: n, old_state: o, new_state: nw, serial } = &sa.attest.body else {
330 
            return Err("attestation is not a ref attestation".into());
331 
        };
332 
        if n != name || o != old_state || nw != new_state {
333 
            return Err("attestation does not describe this update".into());
334 
        }
335 
        if sa.attest.repo != self.id().map_err(|e| e.to_string())? {
336 
            return Err("attestation is for a different repository".into());
337 
        }
338 
        if let Some(last) = last_serial {
339 
            if *serial < last || (*serial == last && !allow_equal) {
340 
                return Err(format!("attestation serial {serial} is not greater than the last seen {last}"));
341 
            }
342 
        }
343 
        let r = sa.attest.hash();
344 
        let sig = sa.sigs.first().ok_or("attestation carries no signature")?;
345 
        let id = self.store.get_identity(&sig.key).map_err(|_| format!("unknown key {}", sig.key.short(12)))?;
346 
        verify_signature(&id, sig, Domain::Attest, &r).map_err(|e| e.to_string())?;
347 
        Ok(*serial)
348 
    }
349
350 
    /// Accept a push (spec §16.4). Without a trust policy every repository
351 
    /// is unverified, so unsigned changes and unattested updates are accepted;
352 
    /// attestations that are present must still be valid. With a policy,
353 
    /// every change and update is checked against it. A newer policy in the
354 
    /// push is applied first when it is signed by the current roots.
355 
    pub fn receive_push(&self, bundle: Bundle, updates: Vec<RefUpdate>, policy: Option<SignedPolicy>) -> Result<Vec<RefResult>> {
356 
        let mut results = Vec::new();
357 
        if let Some(sp) = policy {
358 
            let current = self.current_policy()?.map(|(_, c)| c.policy);
359 
            match self.check_policy(&sp, current.as_ref()) {
360 
                Ok(()) => {
361 
                    self.set_policy(&sp, &format!("push received: policy serial {}", sp.policy.serial))?;
362 
                    results.push(RefResult { name: "policy".into(), ok: true, reason: format!("serial {}", sp.policy.serial) });
363 
                }
364 
                Err(e) => {
365 
                    results.push(RefResult { name: "policy".into(), ok: false, reason: e });
366 
                    for u in updates {
367 
                        results.push(RefResult { name: u.name, ok: false, reason: "policy update was rejected".into() });
368 
                    }
369 
                    return Ok(results);
370 
                }
371 
            }
372 
        }
373 
        let policy = self.current_policy()?.map(|(_, sp)| sp.policy);
374 
        if let Err(e) = self.import_bundle(&bundle, policy.is_some()) {
375 
            for u in updates {
376 
                results.push(RefResult { name: u.name, ok: false, reason: e.to_string() });
377 
            }
378 
            return Ok(results);
379 
        }
380 
        let (op_ref, op) = self.head_op()?;
381 
        let mut view = op.view.clone();
382 
        let mut names = Vec::new();
383 
        for u in updates {
384 
            let outcome = self.apply_update(&mut view, &u, policy.as_ref());
385 
            let (ok, reason) = match outcome {
386 
                Ok(()) => (true, String::new()),
387 
                Err(e) => (false, e),
388 
            };
389 
            if ok {
390 
                names.push(u.name.clone());
391 
            }
392 
            results.push(RefResult { name: u.name, ok, reason });
393 
        }
394 
        if !names.is_empty() {
395 
            self.record_op(&[op_ref], &format!("push received: {}", names.join(" ")), view)?;
396 
        }
397 
        Ok(results)
398 
    }
399
400 
    fn apply_update(&self, view: &mut View, u: &RefUpdate, policy: Option<&crate::policy::Policy>) -> std::result::Result<(), String> {
401 
        if u.name == "policy" {
402 
            return Err("the policy reference is updated by sending a policy, not a bookmark update".into());
403 
        }
404 
        let empty = self.store.put_state(&StateObject::new(vec![])).map_err(|e| e.to_string())?;
405 
        let current = view.bookmarks.get(&u.name).copied().unwrap_or(empty);
406 
        if current != u.old_state {
407 
            return Err("stale: the remote bookmark has moved; pull first".into());
408 
        }
409 
        let new_state = self.store.get_state(&u.new_state).map_err(|_| "new state object is missing".to_string())?;
410 
        for h in &new_state.heads {
411 
            if !self.store.has(h) {
412 
                return Err(format!("head {} is missing", h.short(12)));
413 
            }
414 
        }
415 
        if !u.force {
416 
            let old_heads = self.store.get_state(&current).map_err(|e| e.to_string())?.heads;
417 
            let closure = self.closure_in_store(&new_state.heads).map_err(|e| e.to_string())?;
418 
            if let Some(h) = old_heads.iter().find(|h| !closure.contains(h)) {
419 
                return Err(format!("not a fast-forward: current head {} would be dropped (use --force)", h.short(12)));
420 
            }
421 
        }
422 
        if let Some(p) = policy {
423 
            let Some(sa) = &u.attestation else {
424 
                return Err(format!("{:?} requires a signed reference attestation under this repository's policy", u.name));
425 
            };
426 
            let signer = sa.sigs.first().map(|s| s.key).ok_or("attestation carries no signature")?;
427 
            if !p.authorizes(&signer, &u.name) {
428 
                return Err(format!("key {} may not update {:?} under this repository's policy", signer.short(12), u.name));
429 
            }
430 
            let old_heads = self.store.get_state(&current).map_err(|e| e.to_string())?.heads;
431 
            self.state_allowed(&old_heads, &new_state.heads, p, &u.name)?;
432 
        }
433 
        if let Some(sa) = &u.attestation {
434 
            let last = self.latest_ref_attest(&u.name).map_err(|e| e.to_string())?.and_then(|(_, a)| match a.attest.body {
435 
                AttestBody::Ref { serial, .. } => Some(serial),
436 
                _ => None,
437 
            });
438 
            self.check_ref_attest(sa, &u.name, &u.old_state, &u.new_state, last, false)?;
439 
            let r = self.store.put_attest(&sa.attest).map_err(|e| e.to_string())?;
440 
            for sig in &sa.sigs {
441 
                self.store.put_signature(&r, sig).map_err(|e| e.to_string())?;
442 
            }
443 
            self.set_ref_attest(&u.name, &r).map_err(|e| e.to_string())?;
444 
        }
445 
        view.bookmarks.insert(u.name.clone(), u.new_state);
446 
        Ok(())
447 
    }
448
449 
    // ------------------------------------------------------------ client side
450
451 
    fn remote_url(&self, name: &str) -> Result<String> {
452 
        self.config.remotes.get(name).cloned().ok_or_else(|| RepoError::Msg(format!("no remote named {name:?}; add one with `oxy remote add`")))
453 
    }
454
455 
    /// Credentials and trust for a remote: `remote.<name>.token` (else
456 
    /// `$OXY_TOKEN`) and `remote.<name>.ca` (else `$OXY_CA`).
457 
    fn remote_options(&self, name: &str) -> RemoteOptions {
458 
        RemoteOptions {
459 
            token: self.config.remote_tokens.get(name).cloned().or_else(|| std::env::var("OXY_TOKEN").ok()),
460 
            ca: self.config.remote_cas.get(name).map(PathBuf::from).or_else(|| std::env::var("OXY_CA").ok().map(PathBuf::from)),
461 
        }
462 
    }
463
464 
    fn open_named_remote(&self, name: &str) -> Result<Box<dyn crate::protocol::Remote>> {
465 
        open_remote(&self.remote_url(name)?, &self.remote_options(name))
466 
    }
467
468 
    fn serial_path(&self, remote: &str, bookmark: &str) -> PathBuf {
469 
        self.oxy.join("remotes").join(remote).join(format!("{bookmark}.serial"))
470 
    }
471
472 
    fn last_seen_serial(&self, remote: &str, bookmark: &str) -> Option<u64> {
473 
        fs::read_to_string(self.serial_path(remote, bookmark)).ok().and_then(|s| s.trim().parse().ok())
474 
    }
475
476 
    fn record_serial(&self, remote: &str, bookmark: &str, serial: u64) -> Result<()> {
477 
        let p = self.serial_path(remote, bookmark);
478 
        fs::create_dir_all(p.parent().unwrap())?;
479 
        write_atomic(&p, serial.to_string().as_bytes())
480 
    }
481
482 
    fn policy_serial_path(&self, remote: &str) -> PathBuf {
483 
        self.oxy.join("remotes").join(remote).join("policy.serial")
484 
    }
485
486 
    /// The highest policy serial `remote` has ever presented.
487 
    fn remote_policy_serial(&self, remote: &str) -> Option<u64> {
488 
        fs::read_to_string(self.policy_serial_path(remote)).ok().and_then(|s| s.trim().parse().ok())
489 
    }
490
491 
    fn record_policy_serial(&self, remote: &str, serial: u64) -> Result<()> {
492 
        let p = self.policy_serial_path(remote);
493 
        fs::create_dir_all(p.parent().unwrap())?;
494 
        write_atomic(&p, serial.to_string().as_bytes())
495 
    }
496
497 
    fn lazy_dir(&self) -> PathBuf {
498 
        self.oxy.join("lazy")
499 
    }
500
501 
    /// Remember that `blob` was withheld by `remote` (spec §13.5).
502 
    fn register_lazy(&self, blob: &ObjectRef, remote: &str) -> Result<()> {
503 
        fs::create_dir_all(self.lazy_dir())?;
504 
        write_atomic(&self.lazy_dir().join(blob.to_hex()), remote.as_bytes())
505 
    }
506
507 
    /// Which remote promised an absent blob, if any.
508 
    pub fn lazy_remote(&self, blob: &ObjectRef) -> Option<String> {
509 
        fs::read_to_string(self.lazy_dir().join(blob.to_hex())).ok().map(|s| s.trim().to_string())
510 
    }
511
512 
    /// Fetch an absent blob (and its chunks) from the remote that promised it.
513 
    pub(crate) fn fetch_lazy_blob(&self, blob: &ObjectRef) -> Result<()> {
514 
        let Some(remote_name) = self.lazy_remote(blob) else {
515 
            return Err(RepoError::Store(StoreError::NotFound(*blob)));
516 
        };
517 
        let remote = self.open_named_remote(&remote_name)?;
518 
        let objects = remote.objects(&[*blob])?;
519 
        for (ty, body) in objects {
520 
            if matches!(ty, ObjectType::Blob | ObjectType::Chunk) {
521 
                self.store.put(ty, &body)?;
522 
            }
523 
        }
524 
        if !self.store.has(blob) {
525 
            return fail(format!("{remote_name} did not send blob {}", blob.short(12)));
526 
        }
527 
        let _ = fs::remove_file(self.lazy_dir().join(blob.to_hex()));
528 
        Ok(())
529 
    }
530
531 
    /// After importing from `remote`, record every blob the changes reference
532 
    /// that the store lacks as lazily available from that remote.
533 
    fn register_missing_blobs(&self, changes: &[ChangeHash], remote: &str) -> Result<usize> {
534 
        let mut n = 0;
535 
        for h in changes {
536 
            let c = self.store.get_change(h)?;
537 
            for e in &c.edits {
538 
                if let Edit::Replace { old, new, .. } = e {
539 
                    for b in [old, new].into_iter().flatten() {
540 
                        if !self.store.has(b) {
541 
                            self.register_lazy(b, remote)?;
542 
                            n += 1;
543 
                        }
544 
                    }
545 
                }
546 
            }
547 
        }
548 
        Ok(n)
549 
    }
550
551 
    /// Every head this repository knows: bookmarks, remote tracking, working copy.
552 
    fn known_heads(&self, view: &View) -> Result<Vec<ChangeHash>> {
553 
        let mut have: BTreeSet<ChangeHash> = BTreeSet::new();
554 
        for st in view.bookmarks.values().chain(view.remotes.values().flat_map(|b| b.values())) {
555 
            if let Ok(s) = self.store.get_state(st) {
556 
                have.extend(s.heads);
557 
            }
558 
        }
559 
        have.extend(view.workspaces.values().copied());
560 
        Ok(have.into_iter().collect())
561 
    }
562
563 
    /// Fetch every bookmark of a remote into `remotes/<name>/bookmarks/*` (spec §10.2).
564 
    pub fn fetch(&self, remote_name: &str) -> Result<FetchReport> {
565 
        let remote = self.open_named_remote(remote_name)?;
566 
        if remote.repo_id()? != self.id()? {
567 
            return fail(format!("remote {remote_name:?} is a different repository (repository IDs differ)"));
568 
        }
569 
        let resp = remote.refs()?;
570 
        // Trust policy (spec §14.5). Adopt the remote's policy when it is newer
571 
        // and signed by roots we trust, or on first contact. A remote that
572 
        // once presented a policy may never present none or an older one:
573 
        // that is a downgrade. A remote that never presented one is
574 
        // unverified, and this fetch is checked only as far as possible.
575 
        let local_policy = self.current_policy()?.map(|(_, sp)| sp.policy);
576 
        let seen = self.remote_policy_serial(remote_name);
577 
        let mut unverified = false;
578 
        match &resp.policy {
579 
            Some(sp) => {
580 
                if seen.is_some_and(|s| sp.policy.serial < s) {
581 
                    return fail(format!("{remote_name} presents policy serial {} but presented {} before: refusing the downgrade", sp.policy.serial, seen.unwrap()));
582 
                }
583 
                match &local_policy {
584 
                    Some(l) if sp.policy.serial < l.serial => {}
585 
                    Some(l) if sp.policy.serial == l.serial => {
586 
                        if sp.policy.hash() != l.hash() {
587 
                            return fail(format!("{remote_name} presents a different policy with the same serial {}", l.serial));
588 
                        }
589 
                    }
590 
                    local => {
591 
                        self.check_policy(sp, local.as_ref()).map_err(|e| RepoError::Msg(format!("policy from {remote_name}: {e}")))?;
592 
                        self.set_policy(sp, &format!("oxy fetch {remote_name} (policy serial {})", sp.policy.serial))?;
593 
                    }
594 
                }
595 
                self.record_policy_serial(remote_name, sp.policy.serial)?;
596 
            }
597 
            None => {
598 
                if seen.is_some() {
599 
                    return fail(format!("{remote_name} presented a policy before but has none now: refusing the downgrade"));
600 
                }
601 
                unverified = true;
602 
            }
603 
        }
604 
        let policy = if unverified { None } else { self.current_policy()?.map(|(_, sp)| sp.policy) };
605 
        let (op_ref, op) = self.head_op()?;
606 
        let mut view = op.view.clone();
607 
        let have = self.known_heads(&view)?;
608 
        let mut report = Vec::new();
609 
        let max_blob = self.config.max_blob;
610 
        for r in resp.refs {
611 
            let needed = remote.negotiate(&have, &r.heads)?;
612 
            let bundle = remote.fetch(&needed, max_blob)?;
613 
            // Changes are not individually re-verified here: the bookmark's
614 
            // attestation (checked below) binds its whole state, and the
615 
            // receiver of the push verified the added changes at that time.
616 
            let imported = self.import_bundle(&bundle, false)?;
617 
            if max_blob.is_some() {
618 
                self.register_missing_blobs(&imported, remote_name)?;
619 
            }
620 
            for h in &r.heads {
621 
                if !self.store.has(h) {
622 
                    return fail(format!("remote did not send head {}", h.short(12)));
623 
                }
624 
            }
625 
            let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
626 
            if state != r.state {
627 
                return fail(format!("remote state {} does not match its heads", r.state.short(12)));
628 
            }
629 
            if let Some(p) = &policy {
630 
                let Some(sa) = &r.attestation else {
631 
                    return fail(format!("{remote_name}/{} has no attestation, which its policy requires", r.name));
632 
                };
633 
                let signer = sa.sigs.first().map(|s| s.key).ok_or_else(|| RepoError::Msg("attestation carries no signature".into()))?;
634 
                if !p.authorizes(&signer, &r.name) {
635 
                    return fail(format!("{remote_name}/{}: attested by {}, which the policy does not authorize", r.name, signer.short(12)));
636 
                }
637 
            }
638 
            if let Some(sa) = &r.attestation {
639 
                let AttestBody::Ref { old_state, new_state, .. } = &sa.attest.body else {
640 
                    return fail("remote sent a non-ref attestation for a bookmark");
641 
                };
642 
                let (old_state, new_state) = (*old_state, *new_state);
643 
                let serial = self
644 
                    .check_ref_attest(sa, &r.name, &old_state, &new_state, self.last_seen_serial(remote_name, &r.name), true)
645 
                    .map_err(|e| RepoError::Msg(format!("attestation for {}/{}: {e}", remote_name, r.name)))?;
646 
                if new_state != r.state {
647 
                    return fail(format!("attestation for {}/{} does not cover its current state", remote_name, r.name));
648 
                }
649 
                let ar = self.store.put_attest(&sa.attest)?;
650 
                for sig in &sa.sigs {
651 
                    self.store.put_signature(&ar, sig)?;
652 
                }
653 
                self.record_serial(remote_name, &r.name, serial)?;
654 
            }
655 
            view.remotes.entry(remote_name.to_string()).or_default().insert(r.name.clone(), state);
656 
            report.push((r.name, r.heads, imported.len()));
657 
        }
658 
        if view != op.view {
659 
            self.record_op(&[op_ref], &format!("oxy fetch {remote_name}"), view)?;
660 
        }
661 
        Ok(FetchReport { refs: report, unverified })
662 
    }
663
664 
    /// Shallow fetch of the remote's bookmark: its checkpoint as our base,
665 
    /// the changes within `depth` steps of its heads, and the boundary below.
666 
    pub fn fetch_shallow(&self, remote_name: &str, depth: u32) -> Result<usize> {
667 
        let remote = self.open_named_remote(remote_name)?;
668 
        if remote.repo_id()? != self.id()? {
669 
            return fail(format!("remote {remote_name:?} is a different repository (repository IDs differ)"));
670 
        }
671 
        let resp = remote.refs()?;
672 
        let bookmark = self.config.bookmark.clone();
673 
        if let Some(sp) = &resp.policy {
674 
            let local = self.current_policy()?.map(|(_, sp)| sp.policy);
675 
            self.check_policy(sp, local.as_ref()).map_err(|e| RepoError::Msg(format!("policy from {remote_name}: {e}")))?;
676 
            self.set_policy(sp, &format!("oxy clone --depth (policy serial {})", sp.policy.serial))?;
677 
            self.record_policy_serial(remote_name, sp.policy.serial)?;
678 
        }
679 
        let Some(r) = resp.refs.into_iter().find(|r| r.name == bookmark) else {
680 
            return fail(format!("{remote_name} has no bookmark {bookmark:?}"));
681 
        };
682 
        if r.heads.is_empty() {
683 
            return Ok(0);
684 
        }
685 
        // Base: the remote's checkpoint of exactly these heads.
686 
        let cp_bundle = remote.checkpoint(&r.heads)?;
687 
        let mut cp_ref = None;
688 
        for (ty, body) in &cp_bundle.objects {
689 
            let put = self.store.put(*ty, body)?;
690 
            if *ty == ObjectType::Checkpoint {
691 
                cp_ref = Some(put);
692 
            }
693 
        }
694 
        let Some(cp_ref) = cp_ref else { return fail(format!("{remote_name} sent no checkpoint")) };
695 
        self.install_checkpoint(&bookmark, &cp_ref)?;
696 
        // History window.
697 
        let wanted = remote.shallow(&r.heads, depth)?;
698 
        let bundle = remote.fetch(&wanted, self.config.max_blob)?;
699 
        let imported = self.import_bundle_shallow(&bundle, false)?;
700 
        if self.config.max_blob.is_some() {
701 
            self.register_missing_blobs(&imported, remote_name)?;
702 
        }
703 
        self.refresh_boundary(&imported)?;
704 
        if let Some(sa) = &r.attestation {
705 
            let ar = self.store.put_attest(&sa.attest)?;
706 
            for sig in &sa.sigs {
707 
                self.store.put_signature(&ar, sig)?;
708 
            }
709 
            if let AttestBody::Ref { serial, .. } = sa.attest.body {
710 
                self.record_serial(remote_name, &r.name, serial)?;
711 
            }
712 
        }
713 
        let state = self.store.put_state(&StateObject::new(r.heads.clone()))?;
714 
        let (op_ref, op) = self.head_op()?;
715 
        let mut view = op.view.clone();
716 
        view.remotes.entry(remote_name.to_string()).or_default().insert(r.name.clone(), state);
717 
        self.record_op(&[op_ref], &format!("oxy fetch {remote_name} --depth {depth}"), view)?;
718 
        Ok(imported.len())
719 
    }
720
721 
    /// Fetch, then take the union of the local bookmark's state with the
722 
    /// remote's: no merge change, conflicts become reified (spec §9.4).
723 
    pub fn pull(&self, remote_name: &str) -> Result<PullReport> {
724 
        self.fetch(remote_name)?;
725 
        let (op_ref, op) = self.head_op()?;
726 
        let bookmark = self.config.bookmark.clone();
727 
        let local = self.heads_of(&op.view, &bookmark)?;
728 
        let Some(remote_state) = op.view.remotes.get(remote_name).and_then(|b| b.get(&bookmark)).copied() else {
729 
            return Ok(PullReport { heads: local, changed: false, checked_out: false });
730 
        };
731 
        let remote_heads = self.store.get_state(&remote_state)?.heads;
732 
        let mut all = local.clone();
733 
        all.extend(remote_heads);
734 
        let st = self.load_state(&all)?;
735 
        let mut heads = st.maximal(&all);
736 
        heads.sort();
737 
        let mut sorted_local = local.clone();
738 
        sorted_local.sort();
739 
        if heads == sorted_local {
740 
            return Ok(PullReport { heads, changed: false, checked_out: false });
741 
        }
742 
        let mut view = op.view.clone();
743 
        view.bookmarks.insert(bookmark.clone(), self.store.put_state(&StateObject::new(heads.clone()))?);
744 
        self.record_op(&[op_ref], &format!("oxy pull {remote_name}"), view)?;
745 
        // Follow along if the working copy was clean and on the old heads.
746 
        let snap = self.snapshot("oxy pull")?;
747 
        let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
748 
        let mut base = snap.base.maximal(&snap.wc.deps);
749 
        base.sort();
750 
        let mut checked_out = false;
751 
        if clean && base == sorted_local {
752 
            self.checkout(&heads, false, "oxy pull (checkout)")?;
753 
            checked_out = true;
754 
        }
755 
        Ok(PullReport { heads, changed: true, checked_out })
756 
    }
757
758 
    /// Push the current bookmark (spec §16.3), sealing this author's unsigned
759 
    /// changes and attesting the reference when a key is available.
760 
    pub fn push(&self, remote_name: &str, force: bool) -> Result<Vec<RefResult>> {
761 
        let remote = self.open_named_remote(remote_name)?;
762 
        if remote.repo_id()? != self.id()? {
763 
            return fail(format!("remote {remote_name:?} is a different repository (repository IDs differ)"));
764 
        }
765 
        let (op_ref, op) = self.head_op()?;
766 
        let bookmark = self.config.bookmark.clone();
767 
        let local_heads = self.heads_of(&op.view, &bookmark)?;
768 
        let resp = remote.refs()?;
769 
        let local_policy = self.current_policy()?.map(|(_, sp)| sp);
770 
        let policy_to_send = match (&local_policy, &resp.policy) {
771 
            (Some(l), Some(r)) if l.policy.serial > r.policy.serial => Some(l.clone()),
772 
            (Some(l), None) => Some(l.clone()),
773 
            (Some(l), Some(r)) if l.policy.serial < r.policy.serial => {
774 
                return fail(format!("{remote_name} has a newer policy (serial {}); fetch first", r.policy.serial));
775 
            }
776 
            _ => None,
777 
        };
778 
        let remote_has_policy = resp.policy.is_some() || policy_to_send.is_some();
779 
        if remote_has_policy && self.secret_key()?.is_none() {
780 
            return fail(format!("{remote_name} enforces a trust policy; pushing needs an identity (`oxy identity create`)"));
781 
        }
782 
        let refs = resp.refs;
783 
        let empty = self.store.put_state(&StateObject::new(vec![]))?;
784 
        let (remote_state, remote_heads, last_serial) = match refs.iter().find(|r| r.name == bookmark) {
785 
            Some(r) => (r.state, r.heads.clone(), r.last_serial),
786 
            None => (empty, vec![], None),
787 
        };
788 
        let new_state = self.store.put_state(&StateObject::new(local_heads.clone()))?;
789 
        // A push that carries a policy re-attests the bookmark even when its
790 
        // state is unchanged, so the current attestation is always by a key
791 
        // the current policy authorizes (a revoked key's attestation would
792 
        // otherwise be refused by every later fetch).
793 
        if new_state == remote_state && policy_to_send.is_none() {
794 
            return Ok(vec![RefResult { name: bookmark, ok: true, reason: "already up to date".into() }]);
795 
        }
796 
        for h in &remote_heads {
797 
            if !self.store.has(h) {
798 
                return fail(format!("the remote has change {} which you do not; pull first", h.short(12)));
799 
            }
800 
        }
801 
        if !force {
802 
            let closure = self.closure_in_store(&local_heads)?;
803 
            if let Some(h) = remote_heads.iter().find(|h| !closure.contains(h)) {
804 
                return fail(format!("not a fast-forward: remote head {} is not in your state; pull first or use --force", h.short(12)));
805 
            }
806 
        }
807 
        let to_send = self.negotiate(&remote_heads, &local_heads)?;
808 
        let mut bundle = self.bundle_for(&to_send)?;
809 
        let key = self.secret_key()?.map(|(k, _)| k);
810 
        if let Some(key) = key {
811 
            // Seal this author's changes that carry no valid authenticator.
812 
            let mut unsigned = Vec::new();
813 
            for h in &to_send {
814 
                let c = self.store.get_change(h)?;
815 
                if c.author == key && !matches!(self.authenticate_change(h)?, SignatureStatus::Valid { .. }) {
816 
                    unsigned.push(*h);
817 
                }
818 
            }
819 
            for chunk in unsigned.chunks(crate::attest::MAX_SEAL_HASHES) {
820 
                let sa = self.sign_attest(&Attest::seal(self.id()?, chunk.to_vec()))?;
821 
                bundle.attests.push(sa);
822 
            }
823 
            bundle = self.bundle_for(&to_send)?;
824 
        }
825 
        bundle.objects.push((ObjectType::State, StateObject::new(local_heads.clone()).encode()));
826 
        let attestation = match key {
827 
            Some(_) => Some(self.sign_attest(&Attest {
828 
                repo: self.id()?,
829 
                hlc: Hlc::now(),
830 
                body: AttestBody::Ref { name: bookmark.clone(), old_state: remote_state, new_state, serial: last_serial.unwrap_or(0) + 1 },
831 
            })?),
832 
            None => None,
833 
        };
834 
        let serial = attestation.as_ref().and_then(|a| match a.attest.body {
835 
            AttestBody::Ref { serial, .. } => Some(serial),
836 
            _ => None,
837 
        });
838 
        let sent_serial = policy_to_send.as_ref().map(|p| p.policy.serial);
839 
        let results = remote.push(bundle, vec![RefUpdate { name: bookmark.clone(), old_state: remote_state, new_state, force, attestation }], policy_to_send)?;
840 
        if let (Some(serial), true) = (sent_serial, results.iter().any(|r| r.ok && r.name == "policy")) {
841 
            self.record_policy_serial(remote_name, serial)?;
842 
        }
843 
        if results.iter().any(|r| r.ok && r.name == bookmark) {
844 
            let mut view = op.view.clone();
845 
            view.remotes.entry(remote_name.to_string()).or_default().insert(bookmark.clone(), new_state);
846 
            self.record_op(&[op_ref], &format!("oxy push {remote_name}"), view)?;
847 
            if let Some(s) = serial {
848 
                self.record_serial(remote_name, &bookmark, s)?;
849 
            }
850 
        }
851 
        Ok(results)
852 
    }
853
854 
    /// Clone a remote into `dest`: same repository ID, `origin` remote,
855 
    /// bookmark set to the remote's, working copy checked out.
856 
    pub fn clone(url: &str, dest: &Path) -> Result<Repo> {
857 
        Self::clone_with(url, dest, None, &[])
858 
    }
859
860 
    /// Clone, leaving blobs above `max_blob` bytes to be fetched on first
861 
    /// read, and materializing only `sparse` paths (which implies lazy blobs
862 
    /// unless a size is given, so out-of-scope binaries are never fetched).
863 
    pub fn clone_with(url: &str, dest: &Path, max_blob: Option<u64>, sparse: &[String]) -> Result<Repo> {
864 
        Self::clone_full(url, dest, max_blob, sparse, None)
865 
    }
866
867 
    /// Shallow clone (spec §17): a checkpoint of the bookmark's heads plus the
868 
    /// changes within `depth` steps of them; the rest is the boundary.
869 
    pub fn clone_shallow(url: &str, dest: &Path, depth: u32, max_blob: Option<u64>, sparse: &[String]) -> Result<Repo> {
870 
        Self::clone_full(url, dest, max_blob, sparse, Some(depth))
871 
    }
872
873 
    fn clone_full(url: &str, dest: &Path, max_blob: Option<u64>, sparse: &[String], depth: Option<u32>) -> Result<Repo> {
874 
        let options = RemoteOptions {
875 
            token: std::env::var("OXY_TOKEN").ok(),
876 
            ca: std::env::var("OXY_CA").ok().map(PathBuf::from),
877 
        };
878 
        let remote = open_remote(url, &options)?;
879 
        let id = remote.repo_id()?;
880 
        let mut repo = Repo::init_with_id(dest, id)?;
881 
        repo.config.set("remote.origin.url", url)?;
882 
        if let Some(m) = max_blob {
883 
            repo.config.set("filter.max-blob", &m.to_string())?;
884 
        } else if !sparse.is_empty() {
885 
            repo.config.set("filter.max-blob", "0")?;
886 
        }
887 
        if !sparse.is_empty() {
888 
            repo.config.set("filter.paths", &sparse.join(","))?;
889 
        }
890 
        repo.save_config()?;
891 
        if let Some(depth) = depth {
892 
            repo.fetch_shallow("origin", depth)?;
893 
        } else {
894 
            repo.fetch("origin")?;
895 
        }
896 
        let (op_ref, op) = repo.head_op()?;
897 
        let bookmark = repo.config.bookmark.clone();
898 
        if let Some(state) = op.view.remotes.get("origin").and_then(|b| b.get(&bookmark)).copied() {
899 
            let heads = repo.store.get_state(&state)?.heads;
900 
            let mut view = op.view.clone();
901 
            view.bookmarks.insert(bookmark, state);
902 
            repo.record_op(&[op_ref], &format!("oxy clone {url}"), view)?;
903 
            repo.checkout(&heads, false, "oxy clone (checkout)")?;
904 
        }
905 
        Ok(repo)
906 
    }
907 
}