Snapshots: large new files left out and named; past working copies expire b689779e9e74

- **Large new files are not snapshotted unseen, and past snapshots expire (#98).** A 1 GB image left untracked in a working tree was taken in by the next `status` and kept by the operation log for good: deleting the file and running `gc --now` freed nothing. Now a new, untracked file over `snapshot.max-new-file-size` (default 100 MiB) is left out and named by `status`, with how to ignore it or raise the limit; tracked files and files already in the working copy are unaffected. Past working-copy snapshots keep their files only for `core.snapshot-retention` (default 7 days), so `gc` prunes what only they held, and `oxy gc --expire-snapshots --now` does it at once. `undo` and `op restore` to an operation whose working copy has expired keep the files as they are.

    /// `core.snapshot-retention`: days for which past operations keep their

    /// working copies reachable (oxygen#98); each workspace's current one is

    /// always kept. A file only ever in a working copy goes with it.

    pub snapshot_retention_days: u64,

    /// `snapshot.max-new-file-size`: a new, untracked file larger than this

    /// is left out of working-copy snapshots and named instead (oxygen#98);

    /// 0 takes in files of any size.

    pub max_new_file: u64,

            max_new_file: 100 << 20,

            snapshot_retention_days: 7,

            "snapshot.max-new-file-size" => self.max_new_file = parse_size(value)?,

            "core.snapshot-retention" => self.snapshot_retention_days = parse_days(value)?,

//! `core.op-retention` (their working copies only within the shorter

//! `core.snapshot-retention`), reference attestations, and the policy; receipts

        // Operations: always kept; views within retention are roots. Past

        // working copies only within the shorter `core.snapshot-retention`

        // (oxygen#98): a file only ever snapshotted, such as a build output

        // a status took in, goes once they expire.

        let now_ms = Hlc::now().wall_ms;

        let cutoff = now_ms.saturating_sub(self.config.op_retention_days * 86_400_000);

        let wc_cutoff = now_ms

            .saturating_sub(self.config.snapshot_retention_days * 86_400_000)

            .max(cutoff);

            }

            if i == 0 || (self.config.snapshot_retention_days > 0 && op.hlc.wall_ms >= wc_cutoff) {

                changes.extend(op.view.workspaces.values().copied());

    /// New untracked files the last walk left out for their size, with

    /// their sizes (oxygen#98).

    pub(crate) skipped_large: std::cell::RefCell<Vec<(String, u64)>>,

            skipped_large: Default::default(),

            skipped_large: Default::default(),

                    skipped_large: Default::default(),

                    skipped_large: Default::default(),

                // Past `core.snapshot-retention` the working copy may be

                // gone: the files then stay as they are.

                let Ok(wc) = self.store.get_change(&then) else {

                    return Ok(Undone {

                        command: target.command,

                        changes,

                        others_kept: others,

                        pushed,

                    });

                };

    }

    /// New untracked files the last snapshot left out for being larger than

    /// `snapshot.max-new-file-size`, with their sizes (oxygen#98).

    pub fn skipped_large(&self) -> Vec<(String, u64)> {

        self.skipped_large.borrow().clone()

        // A new file larger than the limit is left out and named, not taken

        // in unseen (oxygen#98): a build's output or a downloaded image

        // would otherwise be stored by the next status, kept by the

        // operation log. Files tracked, or already in the working copy, stay.

        let limit = self.config.max_new_file;

        let mut skipped = Vec::new();

        let files: Vec<_> = files

            .into_iter()

            .filter(|(rel, _, meta)| {

                let new = !tracked.contains_key(rel)

                    && !cache.is_some_and(|m| m.entries.contains_key(rel));

                if limit > 0 && new && meta.size > limit {

                    skipped.push((rel.clone(), meta.size));

                    return false;

                }

                true

            })

            .collect();

        *self.skipped_large.borrow_mut() = skipped;

        // A working copy past `core.snapshot-retention` may be gone: then

        // the bookmark as it was.

        let base = match target

            .view

            .workspaces

            .get(&self.workspace)

            .map(|w| self.store.get_change(w))

        {

            Some(Ok(wc)) => self.load_state_history(&wc.deps)?.maximal(&wc.deps),

            Some(Err(StoreError::NotFound(_))) | None => {

                match target.view.bookmarks.get(&self.config.bookmark) {

                    Some(st) => self.store.get_state(st)?.heads,

                    None => self.bookmark_heads()?,

                }

            Some(Err(e)) => return Err(e.into()),

            "{{\"bookmark\":{},\"working_copy\":{},\"heads\":{},\"clean\":{},\"conflicted\":{},\"paths\":{paths},\"conflicts\":{},\"untracked_nested\":{},\"skipped_large\":{}}}",

            json_array(snap.untracked_nested.iter().map(|p| json_str(p))),

            json_array(

                self.skipped_large

                    .borrow()

                    .iter()

                    .map(|(p, n)| format!("{{\"path\":{},\"size\":{n}}}", json_str(p)))

            )

/// oxygen#98: a new untracked file over snapshot.max-new-file-size is left

/// out of the snapshot and named; tracked files and files already in the

/// working copy are not.

#[test]

fn large_new_files_are_left_out_of_snapshots() {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let mut repo = Repo::init(root).unwrap();

    repo.config.max_new_file = 1024;

    write(root, "small.txt", b"small\n");

    write(root, "big.bin", &vec![0u8; 4096]);

    let s = repo.snapshot("oxy status").unwrap();

    let paths: Vec<_> = s.statuses.iter().map(|p| p.path.as_str()).collect();

    assert_eq!(paths, ["small.txt"]);

    assert_eq!(repo.skipped_large(), vec![("big.bin".to_string(), 4096)]);

    repo.commit("small", &[]).unwrap();

    // A tracked file may grow past the limit.

    write(root, "small.txt", &vec![b'x'; 4096]);

    let s = repo.snapshot("oxy status").unwrap();

    assert!(s.statuses.iter().any(|p| p.path == "small.txt"));

    // With no limit the big file is taken in, and stays once it is.

    repo.config.max_new_file = 0;

    let s = repo.snapshot("oxy status").unwrap();

    assert!(s.statuses.iter().any(|p| p.path == "big.bin"));

    repo.config.max_new_file = 1024;

    let s = repo.snapshot("oxy status").unwrap();

    assert!(

        s.statuses.iter().any(|p| p.path == "big.bin"),

        "already in the working copy"

    );

    assert!(repo.skipped_large().is_empty());

}

/// oxygen#98: a file only a past working-copy snapshot held is pruned once

/// snapshots expire, and kept while they have not.

#[test]

fn gc_prunes_what_only_expired_snapshots_held() {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let mut repo = Repo::init(root).unwrap();

    repo.config.gc_grace_days = 0;

    write(root, "a.txt", b"a\n");

    repo.commit("a", &[]).unwrap();

    let image: Vec<u8> = (0..200_000u32).map(|i| (i % 7) as u8).collect();

    write(root, "image.tar", &image);

    repo.snapshot("oxy status").unwrap();

    std::fs::remove_file(root.join("image.tar")).unwrap();

    repo.snapshot("oxy status").unwrap();

    let blob = Repo::blob_ref(&image);

    assert!(repo.store().has(&blob));

    repo.gc(true).unwrap();

    assert!(repo.store().has(&blob), "a recent snapshot keeps it");

    repo.config.snapshot_retention_days = 0;

    repo.gc(true).unwrap();

    assert!(!repo.store().has(&blob), "an expired snapshot does not");

    assert_eq!(

        repo.file_at("a.txt", None).unwrap(),

        b"a\n",

        "committed files stay"

    );

}

- A **new, untracked file larger than `snapshot.max-new-file-size`** (default 100 MiB; 0 for no limit) is left out of the snapshot and named by `oxy status` (and `status --json` as `skipped_large`), with how to ignore it or raise the limit (oxygen#98): a build's output or a downloaded image is not stored unseen by the next command. A tracked file, or one already in the working copy, is never left out for its size.

An object is **reachable** if it is referenced, directly or transitively, from any bookmark, tag, policy, the current operation heads, any operation newer than `core.op-retention`, any workspace's current working-copy change, the working-copy changes of operations newer than `core.snapshot-retention` (default 7 days; `oxy gc --expire-snapshots` keeps none but the current ones, oxygen#98), or any checkpoint of a reachable state. A file that was only ever in a working copy, never committed, so goes once the snapshots that held it expire. `oxy gc` deletes unreachable objects that are older than `core.gc-grace` (default 14 days, to protect concurrent writers) and repacks. Tombstone atoms are never collected independently of their change. Divergent changes are reachable if any of their variants is. A change's text blobs and their chunks are reachable with it (§7.4). A seal is reachable if any change it covers is reachable, and reference attestations are reachable while they are the latest for their reference or are within `core.op-retention`.

oxy checkpoint                      oxy gc [--now] [--expire-snapshots]               oxy config [--user] KEY VALUE

        /// Keep no past working-copy snapshots, only each workspace's current one, so files only they held go (core.snapshot-retention, default 7 days)

        #[arg(long)]

        expire_snapshots: bool,

        Cmd::Gc {

            now,

            expire_snapshots,

            remote,

        } => match remote {

            None => workspace::gc(now, expire_snapshots),

/// New untracked files left out of the snapshot for their size (oxygen#98):

/// named, with how to keep them out quietly or take them in.

fn print_skipped_large(repo: &Repo) {

    let skipped = repo.skipped_large();

    if skipped.is_empty() {

        return;

    }

    let size = |n: u64| {

        if n >= 1 << 30 {

            format!("{:.1} GiB", n as f64 / (1u64 << 30) as f64)

        } else {

            format!("{:.0} MiB", n as f64 / (1u64 << 20) as f64)

        }

    };

    println!();

    println!(

        "{} {}",

        ui::section(ui::warn("Not tracked")),

        ui::dim(format!("new files over {}", size(repo.config.max_new_file)))

    );

    for (p, n) in &skipped {

        println!("  {} {}  {}", ui::caution(), ui::path(p), ui::dim(size(*n)));

    }

    println!(

        "  {} {}",

        ui::note(),

        ui::dim(

            "to leave them out quietly, list them in .oxyignore; to track one, \

             raise snapshot.max-new-file-size (oxy config snapshot.max-new-file-size 2G)"

        )

    );

}

        print_skipped_large(&repo);

    print_skipped_large(&repo);

    print_skipped_large(&repo);

pub fn gc(now: bool, expire_snapshots: bool) -> Result<i32> {

    let mut repo = open()?;

    if expire_snapshots {

        repo.config.snapshot_retention_days = 0;

    }