OXYGENHUB

Sparse checkout: filter.paths, clone --sparse (implies lazy blobs), oxy sparse, scope-aware snapshot/checkout/prompt (spec 0.6f) 7421309f5372

Author
Andy McCall @andymccall <[email protected]>
When
· 2026-09-26 22:26 UTC
Signature
✔ signed by Andy McCall
Hash
7421309f5372005c6ccb9450fd89075bda28c5712214e8c30335609aabc86efc
Depends on
135212de5370 252e8cb23fbc 750251b21a96 83e0677b07fe 982864068a59 b3c5a59a875a fce44ae9ad29 ff0af1ee299d
Depended on by
c397a8c36a9f 767ac6cd0545 d6e0d5b8b339 ff1f62fee654 8b276e0106d1 eaf3e9068180 c5ce0367eddf 958fb27dda7c d42369abf180 05e71ed6cdf1 75167672ebcd b7f102fcbb17 6e723b81f5b9 6d3eefbce2f0 cdb5beba7f05 4e88e774777a 132b884b9a8d 2863456a0f6d 5259a3af8ee6
In bookmarks
agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
In tags
v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
Edits
35 atoms, 0 tree operations, 12 files

Browse the tree at this change

modified crates/oxy-core/src/repo/checkout.rs +4 −1
…
35 unchanged lines
36 36
                }
37 37
            }
38 38
        }
39 -
        // Write target files that differ.
39 +
        // Write target files that differ (only inside the sparse filter).
40 40
        for (path, f) in &target.files {
41 +
            if !self.in_scope(path) {
42 +
                continue;
43 +
            }
41 44
            match &f.content {
42 45
                FileContent::Dir => continue,
43 46
                FileContent::Text { bytes, .. } => {
…
111 unchanged lines
modified crates/oxy-core/src/repo/config.rs +10 −0
…
30 unchanged lines
31 31
    pub checkpoint_interval: u64,
32 32
    /// `filter.max-blob`: blobs larger than this many bytes are fetched lazily (spec §17).
33 33
    pub max_blob: Option<u64>,
34 +
    /// `filter.paths`: only these subtrees are materialized (spec §17); empty means everything.
35 +
    pub sparse_paths: Vec<String>,
34 36
}
35 37
36 38
impl Config {
…
13 unchanged lines
50 52
            gc_grace_days: 14,
51 53
            checkpoint_interval: 256,
52 54
            max_blob: None,
55 +
            sparse_paths: Vec::new(),
53 56
        }
54 57
    }
55 58
…
46 unchanged lines
102 105
            "core.op-retention" => self.op_retention_days = parse_days(value)?,
103 106
            "core.gc-grace" => self.gc_grace_days = parse_days(value)?,
104 107
            "filter.max-blob" => self.max_blob = if value.is_empty() { None } else { Some(parse_size(value)?) },
108 +
            "filter.paths" => {
109 +
                self.sparse_paths = value
110 +
                    .split(',')
111 +
                    .map(|p| p.trim().trim_start_matches("./").trim_matches('/').to_string())
112 +
                    .filter(|p| !p.is_empty())
113 +
                    .collect();
114 +
            }
105 115
            "core.checkpoint-interval" => {
106 116
                self.checkpoint_interval = value.parse().map_err(|_| RepoError::Msg(format!("expected a number, got {value:?}")))?;
107 117
            }
…
97 unchanged lines
modified crates/oxy-core/src/repo/mod.rs +6 −1
…
461 unchanged lines
462 462
463 463
    // ------------------------------------------------------------ working copy
464 464
465 +
    /// Whether a repository path is inside the sparse filter (spec §17).
466 +
    pub fn in_scope(&self, path: &str) -> bool {
467 +
        self.config.sparse_paths.is_empty() || self.config.sparse_paths.iter().any(|p| path == p || path.starts_with(&format!("{p}/")))
468 +
    }
469 +
465 470
    fn walk_working_copy(&self) -> Result<BTreeMap<String, FsEntry>> {
466 471
        let mut wb = ignore::WalkBuilder::new(&self.root);
467 472
        wb.hidden(false)
…
14 unchanged lines
482 487
            let rel = path.strip_prefix(&self.root).unwrap();
483 488
            let rel: String = rel.components().map(|c| c.as_os_str().to_string_lossy().to_string()).collect::<Vec<_>>().join("/");
484 489
            let Some(ft) = entry.file_type() else { continue };
485 -
            if ft.is_dir() {
490 +
            if ft.is_dir() || !self.in_scope(&rel) {
486 491
                continue;
487 492
            }
488 493
            if ft.is_symlink() {
…
106 unchanged lines
modified crates/oxy-core/src/repo/prompt.rs +3 −0
…
156 unchanged lines
157 157
                continue;
158 158
            }
159 159
            let rel: String = entry.path().strip_prefix(&self.root).unwrap().components().map(|c| c.as_os_str().to_string_lossy().to_string()).collect::<Vec<_>>().join("/");
160 +
            if !self.in_scope(&rel) {
161 +
                continue;
162 +
            }
160 163
            let Some((size, mtime_ns, hash)) = manifest.get(&rel) else { return Ok(Some(true)) };
161 164
            seen += 1;
162 165
            let m = fs::symlink_metadata(entry.path())?;
…
45 unchanged lines
modified crates/oxy-core/src/repo/snapshot.rs +5 −2
…
74 unchanged lines
75 75
            }
76 76
        }
77 77
        for (path, expected) in &base_snapshot.files {
78 -
            if fs_entries.contains_key(path) {
78 +
            if fs_entries.contains_key(path) || !self.in_scope(path) {
79 79
                continue;
80 80
            }
81 -
            let still_needed = expected.kind == NodeKind::Dir && fs_entries.keys().any(|p| p.starts_with(&format!("{path}/")));
81 +
            // A directory stays while anything under it exists or is out of scope.
82 +
            let still_needed = expected.kind == NodeKind::Dir
83 +
                && (fs_entries.keys().any(|p| p.starts_with(&format!("{path}/")))
84 +
                    || base_snapshot.files.keys().any(|p| p.starts_with(&format!("{path}/")) && !self.in_scope(p)));
82 85
            if !still_needed {
83 86
                b.remove_path(path, expected);
84 87
            }
…
513 unchanged lines
modified crates/oxy-core/src/repo/sync.rs +10 −3
…
702 unchanged lines
703 703
    /// Clone a remote into `dest`: same repository ID, `origin` remote,
704 704
    /// bookmark set to the remote's, working copy checked out.
705 705
    pub fn clone(url: &str, dest: &Path) -> Result<Repo> {
706 -
        Self::clone_with(url, dest, None)
706 +
        Self::clone_with(url, dest, None, &[])
707 707
    }
708 708
709 -
    /// Clone, leaving blobs above `max_blob` bytes to be fetched on first read.
710 -
    pub fn clone_with(url: &str, dest: &Path, max_blob: Option<u64>) -> Result<Repo> {
709 +
    /// Clone, leaving blobs above `max_blob` bytes to be fetched on first
710 +
    /// read, and materializing only `sparse` paths (which implies lazy blobs
711 +
    /// unless a size is given, so out-of-scope binaries are never fetched).
712 +
    pub fn clone_with(url: &str, dest: &Path, max_blob: Option<u64>, sparse: &[String]) -> Result<Repo> {
711 713
        let options = RemoteOptions {
712 714
            token: std::env::var("OXY_TOKEN").ok(),
713 715
            ca: std::env::var("OXY_CA").ok().map(PathBuf::from),
…
4 unchanged lines
718 720
        repo.config.set("remote.origin.url", url)?;
719 721
        if let Some(m) = max_blob {
720 722
            repo.config.set("filter.max-blob", &m.to_string())?;
723 +
        } else if !sparse.is_empty() {
724 +
            repo.config.set("filter.max-blob", "0")?;
725 +
        }
726 +
        if !sparse.is_empty() {
727 +
            repo.config.set("filter.paths", &sparse.join(","))?;
721 728
        }
722 729
        repo.save_config()?;
723 730
        repo.fetch("origin")?;
…
12 unchanged lines
modified crates/oxy-core/src/repo/tests.rs +52 −1
…
1204 unchanged lines
1205 1205
    a.commit("text edit", &[]).unwrap();
1206 1206
1207 1207
    // clone --max-blob records the filter; the checkout fetches lazily.
1208 -
    let b = Repo::clone_with(&url, &root.join("b"), Some(1024)).unwrap();
1208 +
    let b = Repo::clone_with(&url, &root.join("b"), Some(1024), &[]).unwrap();
1209 1209
    assert_eq!(b.config.max_blob, Some(1024));
1210 1210
    assert_eq!(fs::read(root.join("b/big.bin")).unwrap(), big);
1211 1211
    assert_eq!(parse_size("2M").unwrap(), 2 << 20);
1212 1212
    assert!(parse_size("lots").is_err());
1213 1213
}
1214 +
1215 +
#[test]
1216 +
fn sparse_checkout_materializes_only_chosen_paths() {
1217 +
    hermetic();
1218 +
    let dir = tempfile::tempdir().unwrap();
1219 +
    let root = dir.path();
1220 +
    let origin_dir = root.join("origin");
1221 +
    let origin = Repo::init(&origin_dir).unwrap().with_home(root.join("home"));
1222 +
    write(&origin_dir, "src/main.rs", b"fn main() {}\n");
1223 +
    write(&origin_dir, "src/lib.rs", b"pub fn a() {}\n");
1224 +
    write(&origin_dir, "docs/guide.md", b"# guide\n");
1225 +
    let logo: Vec<u8> = (0..50_000u32).map(|i| (i % 7) as u8).collect();
1226 +
    write(&origin_dir, "docs/logo.png", &logo);
1227 +
    write(&origin_dir, "README.md", b"# top\n");
1228 +
    let h1 = origin.commit("tree", &[]).unwrap();
1229 +
    let url = origin_dir.to_string_lossy().to_string();
1230 +
1231 +
    let a = Repo::clone_with(&url, &root.join("a"), None, &["src".into()]).unwrap().with_home(root.join("home"));
1232 +
    assert_eq!(a.config.sparse_paths, vec!["src"]);
1233 +
    assert_eq!(a.config.max_blob, Some(0), "sparse implies lazy blobs");
1234 +
    assert!(root.join("a/src/main.rs").exists());
1235 +
    assert!(!root.join("a/docs").exists() && !root.join("a/README.md").exists());
1236 +
    assert!(!a.store().has(&Repo::blob_ref(&logo)), "out-of-scope blob never fetched");
1237 +
    assert!(!a.snapshot("status").unwrap().changed, "out-of-scope files are not removals");
1238 +
    assert_eq!(a.quick_status().unwrap().dirty, false);
1239 +
1240 +
    // Edits in scope commit normally and keep the rest of the tree intact.
1241 +
    write(&root.join("a"), "src/lib.rs", b"pub fn a() {}\npub fn b() {}\n");
1242 +
    let h2 = a.commit("edit lib", &[]).unwrap();
1243 +
    let st = a.load_state(&[h2]).unwrap().materialize();
1244 +
    assert!(st.files.contains_key("docs/guide.md"), "the state still has everything");
1245 +
    assert!(a.push("origin", false).unwrap()[0].ok);
1246 +
    // Files created outside the scope are ignored by status.
1247 +
    write(&root.join("a"), "docs/new.md", b"ignored\n");
1248 +
    assert!(a.snapshot("status").unwrap().statuses.is_empty());
1249 +
1250 +
    // Widening the filter checks out the rest; narrowing leaves stray files alone.
1251 +
    let mut a = a;
1252 +
    a.config.set("filter.paths", "src,docs").unwrap();
1253 +
    a.save_config().unwrap();
1254 +
    let r = a.checkout(&a.bookmark_heads().unwrap(), true, "widen").unwrap();
1255 +
    assert!(r.written >= 2);
1256 +
    assert_eq!(fs::read(root.join("a/docs/logo.png")).unwrap(), logo, "now fetched lazily");
1257 +
    assert!(!root.join("a/README.md").exists());
1258 +
    // Origin sees the change made under a sparse checkout with the full tree intact.
1259 +
    origin.pull("origin").err();
1260 +
    let full = origin.load_state(&origin.bookmark_heads().unwrap()).unwrap().materialize();
1261 +
    assert!(matches!(&full.files["src/lib.rs"].content, FileContent::Text { bytes, .. } if bytes == b"pub fn a() {}\npub fn b() {}\n"));
1262 +
    assert!(full.files.contains_key("README.md"));
1263 +
    let _ = h1;
1264 +
}
modified crates/oxy-server/tests/http.rs +1 −1
…
104 unchanged lines
105 105
    origin.commit("big", &[]).unwrap();
106 106
    let base = start(&served, vec![]);
107 107
    let url = format!("{base}/origin");
108 -
    let a = Repo::clone_with(&url, &root.join("a"), Some(1000)).unwrap();
108 +
    let a = Repo::clone_with(&url, &root.join("a"), Some(1000), &[]).unwrap();
109 109
    assert_eq!(fs::read(root.join("a/big.bin")).unwrap(), big, "fetched through the objects endpoint during checkout");
110 110
    assert!(!a.snapshot("status").unwrap().changed);
111 111
}
…
60 unchanged lines
modified docs/spec.md +5 −3
…
1 unchanged lines
2 2
3 3
| | |
4 4
|---|---|
5 -
| **Status** | Draft 0.6e |
5 +
| **Status** | Draft 0.6f |
6 6
| **Format version** | 1 |
7 7
| **Date** | 2026-09-26 |
8 8
| **Audience** | Implementers, security reviewers, and contributors |
…
702 unchanged lines
711 711
depth    = 100                    # shallow: history beyond this many changes from heads is not stored
712 712
```
713 713
714 -
- **Sparse** repositories hold the full change graph but only materialize and fetch blobs under the listed paths. Changes touching other paths are stored and applied to the graph; their content is lazy.
714 +
- **Sparse** repositories hold the full change graph but only materialize the listed paths: the working copy contains nothing else, files outside the filter are neither reported nor removed, and changes touching other paths are stored and applied to the graph. A sparse clone sets `max-blob` to zero unless a size is given, so blobs outside the filter are fetched only if the filter later widens (§13.5). `oxy sparse PATH..` changes the filter of a clean working copy and re-checks it out.
715 715
- **Lazy blobs** are fetched via `objects` on first read. `oxy status` and `oxy commit` MUST NOT need them.
716 716
- **Shallow** repositories store a *shallow boundary*: a set of ChangeHashes whose dependencies are known to be absent. A shallow repository MUST hold a checkpoint at the boundary so that graphs can be reconstructed. Deps into the boundary are accepted as well-formed. Deepening fetches more history and moves the boundary.
717 717
- Thin clients (§22) always use `depth = 0`: a checkpoint plus zero history.
…
122 unchanged lines
840 840
841 841
---
842 842
843 -
*Changelog.* Draft 0.6e (2026-09-27): lazy blobs: the `fetch` filter, the `objects` message, and the absent-blob record (§13.5, §16.3).
843 +
*Changelog.* Draft 0.6f (2026-09-27): sparse checkout semantics (§17).
844 +
845 +
Draft 0.6e (2026-09-27): lazy blobs: the `fetch` filter, the `objects` message, and the absent-blob record (§13.5, §16.3).
844 846
845 847
Draft 0.6d (2026-09-27): §11 checkpoint format as implemented, with the loading rule.
846 848
…
23 unchanged lines
modified src/cli/mod.rs +8 −2
…
31 unchanged lines
32 32
        /// Leave blobs larger than this (e.g. 1M) to be fetched on first read
33 33
        #[arg(long)]
34 34
        max_blob: Option<String>,
35 +
        /// Materialize only these paths (repeatable); implies lazy blobs
36 +
        #[arg(long = "sparse")]
37 +
        sparse: Vec<String>,
35 38
    },
39 +
    /// Show or set the sparse filter (paths to materialize); re-checks out the bookmark
40 +
    Sparse { paths: Vec<String> },
36 41
    /// Show the working copy: changed paths and conflicts
37 42
    Status,
38 43
    /// One-line summary for shell prompts; fast and records nothing
…
104 unchanged lines
143 148
        #[command(subcommand)]
144 149
        cmd: PolicyCmd,
145 150
    },
146 -
    /// Set a configuration key (author.*, core.sign, core.op-retention, core.gc-grace, core.checkpoint-interval, filter.max-blob, workspace.bookmark, remote.<name>.url|token|ca)
151 +
    /// Set a configuration key (author.*, core.sign, core.op-retention, core.gc-grace, core.checkpoint-interval, filter.max-blob, filter.paths, workspace.bookmark, remote.<name>.url|token|ca)
147 152
    Config {
148 153
        key: String,
149 154
        value: String,
…
89 unchanged lines
239 244
        Cmd::Gc { now } => workspace::gc(now),
240 245
        Cmd::Checkpoint => workspace::checkpoint(),
241 246
        Cmd::Op { cmd: OpCmd::Log } => workspace::op_log(),
242 -
        Cmd::Clone { url, dest, max_blob } => sync::clone(&url, dest, max_blob.as_deref()),
247 +
        Cmd::Clone { url, dest, max_blob, sparse } => sync::clone(&url, dest, max_blob.as_deref(), &sparse),
248 +
        Cmd::Sparse { paths } => workspace::sparse(&paths),
243 249
        Cmd::Remote { cmd } => match cmd {
244 250
            RemoteCmd::Add { name, url } => sync::remote_add(&name, &url),
245 251
            RemoteCmd::Remove { name } => sync::remote_remove(&name),
…
23 unchanged lines
modified src/cli/sync.rs +2 −2
…
6 unchanged lines
7 7
use oxy_core::repo::Repo;
8 8
use std::path::PathBuf;
9 9
10 -
pub fn clone(url: &str, dest: Option<PathBuf>, max_blob: Option<&str>) -> Result<i32> {
10 +
pub fn clone(url: &str, dest: Option<PathBuf>, max_blob: Option<&str>, sparse: &[String]) -> Result<i32> {
11 11
    let dest = dest.unwrap_or_else(|| {
12 12
        let base = url.trim_end_matches('/').rsplit('/').next().unwrap_or("repo");
13 13
        PathBuf::from(base)
14 14
    });
15 15
    let max_blob = max_blob.map(oxy_core::repo::parse_size).transpose()?;
16 16
    let sp = ui::spinner(&format!("Cloning {url}…"));
17 -
    let repo = Repo::clone_with(url, &dest, max_blob)?;
17 +
    let repo = Repo::clone_with(url, &dest, max_blob, sparse)?;
18 18
    let heads = repo.bookmark_heads()?;
19 19
    sp.done(&format!("{} into {} ({} on {}: {})", ui::ok("Cloned"), ui::path(repo.root().display()), url, ui::id(&repo.config.bookmark), plural(heads.len(), "head")));
20 20
    Ok(0)
…
146 unchanged lines
modified src/cli/workspace.rs +26 −0
…
274 unchanged lines
275 275
    println!("{}", info.format(format));
276 276
    Ok(0)
277 277
}
278 +
279 +
pub fn sparse(paths: &[String]) -> Result<i32> {
280 +
    let mut repo = open()?;
281 +
    if paths.is_empty() {
282 +
        if repo.config.sparse_paths.is_empty() {
283 +
            println!("No sparse filter: the whole tree is materialized.");
284 +
        } else {
285 +
            println!("{}", ui::heading("Sparse filter"));
286 +
            for p in &repo.config.sparse_paths {
287 +
                println!("  {}", ui::path(p));
288 +
            }
289 +
        }
290 +
        return Ok(0);
291 +
    }
292 +
    let snap = repo.snapshot("oxy sparse")?;
293 +
    if !snap.wc.tree_ops.is_empty() || !snap.wc.edits.is_empty() {
294 +
        anyhow::bail!("the working copy has changes; commit them before changing the sparse filter");
295 +
    }
296 +
    repo.config.set("filter.paths", &paths.join(","))?;
297 +
    repo.save_config()?;
298 +
    let heads = repo.bookmark_heads()?;
299 +
    let sp = ui::spinner("Re-checking out with the new filter…");
300 +
    let r = repo.checkout(&heads, true, "oxy sparse (checkout)")?;
301 +
    sp.done(&format!("{} {}: {} written, {} removed", ui::ok("Sparse filter set to"), ui::path(paths.join(",")), plural(r.written, "file"), r.removed));
302 +
    Ok(0)
303 +
}