Lazy blobs: clone --max-blob / filter.max-blob, absent-blob registry fetched on first read, objects endpoint (spec 0.6e) b3c5a59a875a

    /// The changes with their signatures, seals, identities, and blobs; blobs

    /// larger than `max_blob` bytes are left out for lazy fetching (spec §13.5).

    fn fetch(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle>;

    /// Individual objects by reference (spec §16.1 `objects`); a blob comes

    /// with its chunks.

    fn objects(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>>;

    fn fetch(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle> {

        self.repo.bundle_for_filtered(changes, max_blob)

    }

    fn objects(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>> {

        self.repo.objects_for(refs)

    fn fetch(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle> {

        wire::decode_bundle(&self.call(true, "fetch", Some(&wire::encode_fetch(changes, max_blob)))?)

    fn objects(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>> {

        Ok(wire::decode_bundle(&self.call(true, "objects", Some(&wire::encode_hashes(refs)))?)?.objects)

    }

    /// `fetch` request: `{0: [changes], 1: max blob size or null}`.

    pub fn encode_fetch(hs: &[ChangeHash], max_blob: Option<u64>) -> Vec<u8> {

        cbor::encode(&Value::map([(0, hashes_value(hs)), (1, max_blob.map(Value::Uint).unwrap_or(Value::Null))]))

    }

    pub fn decode_fetch(bytes: &[u8]) -> Result<(Vec<ChangeHash>, Option<u64>)> {

        let v = cbor::decode(bytes).map_err(ObjectError::from)?;

        let m = map(&v, "fetch")?;

        let max = match m.get(&1) {

            Some(x) if !x.is_null() => Some(o::uint(x, "max blob")?),

            _ => None,

        };

        Ok((hashes_from(field(m, 0, "changes")?)?, max))

    }

        assert_eq!(wire::decode_fetch(&wire::encode_fetch(&[c.hash()], Some(4096))).unwrap(), (vec![c.hash()], Some(4096)));

        assert_eq!(wire::decode_fetch(&wire::encode_fetch(&[], None)).unwrap(), (vec![], None));

        // An old-style fetch request without the filter field still decodes.

        assert_eq!(wire::decode_fetch(&wire::encode_hashes(&[c.hash()])).unwrap(), (vec![c.hash()], None));

    /// `filter.max-blob`: blobs larger than this many bytes are fetched lazily (spec §17).

    pub max_blob: Option<u64>,

            max_blob: None,

            "filter.max-blob" => self.max_blob = if value.is_empty() { None } else { Some(parse_size(value)?) },

/// A byte count with an optional `K`, `M`, or `G` suffix (binary units).

pub fn parse_size(v: &str) -> Result<u64> {

    let v = v.trim();

    let (num, mult) = match v.chars().last() {

        Some('K' | 'k') => (&v[..v.len() - 1], 1u64 << 10),

        Some('M' | 'm') => (&v[..v.len() - 1], 1u64 << 20),

        Some('G' | 'g') => (&v[..v.len() - 1], 1u64 << 30),

        _ => (v, 1),

    };

    let n: u64 = num.trim().parse().map_err(|_| RepoError::Msg(format!("expected a size like 1M, got {v:?}")))?;

    Ok(n * mult)

}

pub use config::{parse_size, read_config_file, Config};

    /// Read a blob's content, fetching it from the remote that withheld it

    /// when it is lazily absent (spec §13.5).

        if !self.store.has(r) && self.lazy_remote(r).is_some() {

            self.fetch_lazy_blob(r)?;

        }

        self.bundle_for_filtered(changes, None)

    }

    /// Objects by reference for the `objects` endpoint; a blob brings its chunks.

    pub fn objects_for(&self, refs: &[ObjectRef]) -> Result<Vec<(ObjectType, Vec<u8>)>> {

        let mut out = Vec::new();

        for r in refs {

            let (ty, body) = match self.store.get(r) {

                Ok(x) => x,

                Err(StoreError::NotFound(_)) => continue,

                Err(e) => return Err(e.into()),

            };

            if ty == ObjectType::Blob {

                for c in Blob::decode(&body)?.chunks {

                    if let Ok((cty, cb)) = self.store.get(&c) {

                        out.push((cty, cb));

                    }

                }

            }

            out.push((ty, body));

        }

        Ok(out)

    }

    /// Like `bundle_for`, leaving out blobs larger than `max_blob` bytes

    /// (and their chunks) so the receiver fetches them lazily (spec §13.5).

    pub fn bundle_for_filtered(&self, changes: &[ChangeHash], max_blob: Option<u64>) -> Result<Bundle> {

                                if max_blob.is_some_and(|m| blob.length > m) {

                                    continue;

                                }

                                objects.insert(*b);

    fn lazy_dir(&self) -> PathBuf {

        self.oxy.join("lazy")

    }

    /// Remember that `blob` was withheld by `remote` (spec §13.5).

    fn register_lazy(&self, blob: &ObjectRef, remote: &str) -> Result<()> {

        fs::create_dir_all(self.lazy_dir())?;

        write_atomic(&self.lazy_dir().join(blob.to_hex()), remote.as_bytes())

    }

    /// Which remote promised an absent blob, if any.

    pub fn lazy_remote(&self, blob: &ObjectRef) -> Option<String> {

        fs::read_to_string(self.lazy_dir().join(blob.to_hex())).ok().map(|s| s.trim().to_string())

    }

    /// Fetch an absent blob (and its chunks) from the remote that promised it.

    pub(crate) fn fetch_lazy_blob(&self, blob: &ObjectRef) -> Result<()> {

        let Some(remote_name) = self.lazy_remote(blob) else {

            return Err(RepoError::Store(StoreError::NotFound(*blob)));

        };

        let remote = self.open_named_remote(&remote_name)?;

        let objects = remote.objects(&[*blob])?;

        for (ty, body) in objects {

            if matches!(ty, ObjectType::Blob | ObjectType::Chunk) {

                self.store.put(ty, &body)?;

            }

        }

        if !self.store.has(blob) {

            return fail(format!("{remote_name} did not send blob {}", blob.short(12)));

        }

        let _ = fs::remove_file(self.lazy_dir().join(blob.to_hex()));

        Ok(())

    }

    /// After importing from `remote`, record every blob the changes reference

    /// that the store lacks as lazily available from that remote.

    fn register_missing_blobs(&self, changes: &[ChangeHash], remote: &str) -> Result<usize> {

        let mut n = 0;

        for h in changes {

            let c = self.store.get_change(h)?;

            for e in &c.edits {

                if let Edit::Replace { old, new, .. } = e {

                    for b in [old, new].into_iter().flatten() {

                        if !self.store.has(b) {

                            self.register_lazy(b, remote)?;

                            n += 1;

                        }

                    }

                }

            }

        }

        Ok(n)

    }

        let max_blob = self.config.max_blob;

            let bundle = remote.fetch(&needed, max_blob)?;

            if max_blob.is_some() {

                self.register_missing_blobs(&imported, remote_name)?;

            }

        Self::clone_with(url, dest, None)

    }

    /// Clone, leaving blobs above `max_blob` bytes to be fetched on first read.

    pub fn clone_with(url: &str, dest: &Path, max_blob: Option<u64>) -> Result<Repo> {

        if let Some(m) = max_blob {

            repo.config.set("filter.max-blob", &m.to_string())?;

        }

#[test]

fn lazy_blobs_are_fetched_on_first_read() {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let origin_dir = root.join("origin");

    let origin = Repo::init(&origin_dir).unwrap().with_home(root.join("home"));

    let url = origin_dir.to_string_lossy().to_string();

    let big: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect(); // has NUL bytes: opaque

    write(&origin_dir, "big.bin", &big);

    write(&origin_dir, "small.bin", &[0u8, 1, 2]);

    write(&origin_dir, "t.txt", b"t\n");

    let h = origin.commit("blobs", &[]).unwrap();

    let big_ref = Repo::blob_ref(&big);

    // Fetch only (no checkout) with a filter: the big blob stays absent.

    let a = Repo::init_with_id(&root.join("a"), origin.id().unwrap()).unwrap().with_home(root.join("home"));

    let a = {

        let mut a = a;

        a.config.set("remote.origin.url", &url).unwrap();

        a.config.set("filter.max-blob", "100K").unwrap();

        a.save_config().unwrap();

        a

    };

    assert_eq!(a.config.max_blob, Some(100 * 1024));

    a.fetch("origin").unwrap();

    assert!(a.store().has(&h));

    assert!(!a.store().has(&big_ref), "withheld");

    assert!(a.store().has(&Repo::blob_ref(&[0u8, 1, 2])), "small blob came along");

    assert_eq!(a.lazy_remote(&big_ref).as_deref(), Some("origin"));

    // status and commit do not need it: adopt the state, edit a text file, commit.

    let (op_ref, op) = a.head_op().unwrap();

    let mut view = op.view.clone();

    view.bookmarks.insert("main".into(), a.store().put_state(&StateObject::new(vec![h])).unwrap());

    a.record_op(&[op_ref], "adopt", view).unwrap();

    // Checkout reads the blob lazily, which fetches it.

    a.checkout(&[h], true, "checkout").unwrap();

    assert!(a.store().has(&big_ref), "fetched on first read");

    assert_eq!(fs::read(root.join("a/big.bin")).unwrap(), big);

    assert!(a.lazy_remote(&big_ref).is_none(), "no longer registered");

    write(&root.join("a"), "t.txt", b"t2\n");

    a.commit("text edit", &[]).unwrap();

    // clone --max-blob records the filter; the checkout fetches lazily.

    let b = Repo::clone_with(&url, &root.join("b"), Some(1024)).unwrap();

    assert_eq!(b.config.max_blob, Some(1024));

    assert_eq!(fs::read(root.join("b/big.bin")).unwrap(), big);

    assert_eq!(parse_size("2M").unwrap(), 2 << 20);

    assert!(parse_size("lots").is_err());

}

        let (changes, max_blob) = wire::decode_fetch(&body)?;

        Ok(ok(wire::encode_bundle(&r.bundle_for_filtered(&changes, max_blob)?)))

async fn objects(State(app): State<Shared>, Path(repo): Path<String>, body: Bytes) -> Result<Response, Failure> {

    blocking(move || {

        let r = app.open(&repo)?;

        let refs = wire::decode_hashes(&body)?;

        let bundle = oxy_core::protocol::Bundle { objects: r.objects_for(&refs)?, ..Default::default() };

        Ok(ok(wire::encode_bundle(&bundle)))

    })

    .await

}

        .route("/{repo}/oxy/v1/objects", post(objects))

fn objects_endpoint_serves_lazy_blobs() {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let served = root.join("served");

    let origin = Repo::init(&served.join("origin")).unwrap();

    let big: Vec<u8> = (0..300_000u32).map(|i| (i % 253) as u8).collect(); // has NUL bytes: opaque

    write(&served.join("origin"), "big.bin", &big);

    write(&served.join("origin"), "t.txt", b"t\n");

    origin.commit("big", &[]).unwrap();

    let base = start(&served, vec![]);

    let url = format!("{base}/origin");

    let a = Repo::clone_with(&url, &root.join("a"), Some(1000)).unwrap();

    assert_eq!(fs::read(root.join("a/big.bin")).unwrap(), big, "fetched through the objects endpoint during checkout");

    assert!(!a.snapshot("status").unwrap().changed);

}

#[test]

| **Status** | Draft 0.6e |

A blob or chunk may be **absent** in a partial repository (§17). After a fetch with a blob filter, the repository records every blob its new changes reference but did not receive, together with the remote that withheld it (`.oxy/lazy/<ref>`), and fetches it through `objects` on first read, dropping the record once stored. Changes and checkpoints are never lazy: a repository that has a change has all of its inline data.

| `fetch` request | `{0: [hashes], 1: max blob size or null}`; response a `Bundle` without blobs above the size |

| `objects` request | `{0: [refs]}`; response a `Bundle` carrying only objects, a blob accompanied by its chunks |

The HTTP content type is `application/oxy-cbor`. The `checkpoint` endpoint of §16.1 is not yet specified in this form.

*Changelog.* Draft 0.6e (2026-09-27): lazy blobs: the `fetch` filter, the `objects` message, and the absent-blob record (§13.5, §16.3).

Draft 0.6d (2026-09-27): §11 checkpoint format as implemented, with the loading rule.

    Clone {

        url: String,

        dest: Option<PathBuf>,

        /// Leave blobs larger than this (e.g. 1M) to be fetched on first read

        #[arg(long)]

        max_blob: Option<String>,

    },

    /// Set a configuration key (author.*, core.sign, core.op-retention, core.gc-grace, core.checkpoint-interval, filter.max-blob, workspace.bookmark, remote.<name>.url|token|ca)

        Cmd::Clone { url, dest, max_blob } => sync::clone(&url, dest, max_blob.as_deref()),

pub fn clone(url: &str, dest: Option<PathBuf>, max_blob: Option<&str>) -> Result<i32> {

    let max_blob = max_blob.map(oxy_core::repo::parse_size).transpose()?;

    let repo = Repo::clone_with(url, &dest, max_blob)?;