OXYGENHUB

oxy gc says what it is doing 9ae4d15a0e4b

A gc of 1.45 GB of loose objects ran for minutes with no output. The
store's prune and repack now report their phases and, while packing,
the count every tenth; Repo::gc_with passes them to a spinner, which
prints each line on stderr when there is no terminal, so a gc run from
a job or a shell script is never silent either.

closes #66
Author
Andy McCall @andymccall <[email protected]>
When
· 2026-10-04 02:04 UTC
Signature
✔ signed by Andy McCall
Hash
9ae4d15a0e4b6bec06c95b87d01e2fb2f407e3fab663d3613a8cbea7bb916cca
Depends on
135212de5370 75167672ebcd 8f10a07a1dbe 982864068a59 a17fc622ca43 afaba5f1306d ba160f60219e d02e545dc265 da652d6392d8 fce44ae9ad29 ff0af1ee299d
Depended on by
642da2636de2 ef7d4ea6f33b 767ac6cd0545 9d580f2132f3 648189388f1a
In bookmarks
agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
In tags
v0.10.0 v0.11.0 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
Edits
25 atoms, 0 tree operations, 5 files

Browse the tree at this change

modified crates/oxy-core/src/repo/gc.rs +11 −2
…
24 unchanged lines
25 25
    /// Prune unreachable objects older than the grace period (`now`
26 26
    /// ignores it), then repack everything into one pack (spec §13.4).
27 27
    pub fn gc(&self, now: bool) -> Result<GcReport> {
28 +
        self.gc_with(now, &mut |_| {})
29 +
    }
30 +
31 +
    /// [`Repo::gc`], reporting each phase and the packing's progress
32 +
    /// (oxygen#66), for a spinner or a log.
33 +
    pub fn gc_with(&self, now: bool, progress: crate::store::Progress) -> Result<GcReport> {
34 +
        progress("finding what is reachable");
28 35
        let keep = self.reachable()?;
29 36
        let grace = if now {
30 37
            std::time::Duration::ZERO
31 38
        } else {
32 39
            std::time::Duration::from_secs(self.config.gc_grace_days * 86_400)
33 40
        };
34 -
        let pruned = self.store.prune(&keep, grace)?;
41 +
        let pruned = self.store.prune_with(&keep, grace, progress)?;
35 42
        // Then everything left in one pack, compressed harder than a
36 43
        // commit's (spec §13.4), as `git gc` repacks.
37 -
        let (packed, merged_packs) = self.store.repack_all(crate::store::REPACK_ZSTD_LEVEL)?;
44 +
        let (packed, merged_packs) = self
45 +
            .store
46 +
            .repack_all_with(crate::store::REPACK_ZSTD_LEVEL, progress)?;
38 47
        Ok(GcReport {
39 48
            packed,
40 49
            pruned,
…
132 unchanged lines
modified crates/oxy-core/src/repo/tests.rs +34 −0
…
1096 unchanged lines
1097 1097
    assert!(!c.snapshot("status").unwrap().changed);
1098 1098
}
1099 1099
1100 +
/// oxygen#66: a gc says what it is doing, phase by phase.
1100 1101
#[test]
1102 +
fn gc_reports_its_phases() {
1103 +
    hermetic();
1104 +
    let dir = tempfile::tempdir().unwrap();
1105 +
    let repo = Repo::init(dir.path()).unwrap();
1106 +
    for i in 0..3 {
1107 +
        write(
1108 +
            dir.path(),
1109 +
            &format!("f{i}.txt"),
1110 +
            format!("{i}\n").as_bytes(),
1111 +
        );
1112 +
        repo.commit(&format!("c{i}"), &[]).unwrap();
1113 +
    }
1114 +
    let mut phases = Vec::new();
1115 +
    repo.gc_with(true, &mut |p| phases.push(p.to_string()))
1116 +
        .unwrap();
1117 +
    assert_eq!(phases[0], "finding what is reachable", "{phases:?}");
1118 +
    assert!(
1119 +
        phases.iter().any(|p| p.starts_with("pruning")),
1120 +
        "{phases:?}"
1121 +
    );
1122 +
    assert!(
1123 +
        phases.iter().any(|p| p.starts_with("packing 0 of ")),
1124 +
        "{phases:?}"
1125 +
    );
1126 +
    assert!(
1127 +
        phases
1128 +
            .iter()
1129 +
            .any(|p| p.starts_with("writing the index of ")),
1130 +
        "{phases:?}"
1131 +
    );
1132 +
}
1133 +
1134 +
#[test]
1101 1135
fn gc_packs_objects_and_history_still_works() {
1102 1136
    hermetic();
1103 1137
    let dir = tempfile::tempdir().unwrap();
…
5072 unchanged lines
modified crates/oxy-core/src/store.rs +40 −3
…
42 unchanged lines
43 43
/// An object read from the store: its type and body.
44 44
pub type Object = (ObjectType, Vec<u8>);
45 45
46 +
/// Where a long operation reports what it is doing (oxygen#66): a phase,
47 +
/// or a count within one, as a line for a spinner or a log.
48 +
pub type Progress<'a> = &'a mut dyn FnMut(&str);
49 +
50 +
fn silent() -> impl FnMut(&str) {
51 +
    |_| {}
52 +
}
53 +
46 54
#[derive(Clone, Copy, Debug)]
47 55
struct PackEntry {
48 56
    offset: u64,
…
340 unchanged lines
389 397
            &loose.iter().map(|(_, p)| p.clone()).collect::<Vec<_>>(),
390 398
            &[],
391 399
            ZSTD_LEVEL,
400 +
            &mut silent(),
392 401
        )
393 402
    }
394 403
…
1 unchanged lines
396 405
    /// `level` (spec §13.4): what an explicit `oxy gc` does, as `git gc`
397 406
    /// repacks. Returns the report and how many packs were folded in.
398 407
    pub fn repack_all(&self, level: i32) -> Result<(PackReport, usize)> {
408 +
        self.repack_all_with(level, &mut silent())
409 +
    }
410 +
411 +
    /// [`Store::repack_all`], reporting progress.
412 +
    pub fn repack_all_with(&self, level: i32, progress: Progress) -> Result<(PackReport, usize)> {
399 413
        let loose = self.loose_objects()?;
400 414
        let (mut refs, paths): (Vec<ObjectRef>, Vec<PathBuf>) = {
401 415
            let packs = self.packs.borrow();
…
8 unchanged lines
410 424
        refs.extend(loose.iter().map(|(r, _)| *r));
411 425
        let loose_paths: Vec<PathBuf> = loose.into_iter().map(|(_, p)| p).collect();
412 426
        let n = paths.len();
413 -
        let report = self.write_pack(&refs, &loose_paths, &paths, level)?;
427 +
        let report = self.write_pack(&refs, &loose_paths, &paths, level, progress)?;
414 428
        Ok((report, n))
415 429
    }
416 430
…
11 unchanged lines
428 442
            )
429 443
        };
430 444
        let n = paths.len();
431 -
        self.write_pack(&refs, &[], &paths, ZSTD_LEVEL)?;
445 +
        self.write_pack(&refs, &[], &paths, ZSTD_LEVEL, &mut silent())?;
432 446
        Ok(n)
433 447
    }
434 448
…
3 unchanged lines
438 452
    /// are rewritten without them. Signature sidecars and seal-index entries
439 453
    /// of deleted objects go too.
440 454
    pub fn prune(
455 +
        &self,
456 +
        keep: &std::collections::HashSet<ObjectRef>,
457 +
        grace: std::time::Duration,
458 +
    ) -> Result<PruneReport> {
459 +
        self.prune_with(keep, grace, &mut silent())
460 +
    }
461 +
462 +
    /// [`Store::prune`], reporting progress.
463 +
    pub fn prune_with(
441 464
        &self,
442 465
        keep: &std::collections::HashSet<ObjectRef>,
443 466
        grace: std::time::Duration,
467 +
        progress: Progress,
444 468
    ) -> Result<PruneReport> {
469 +
        progress("pruning loose objects");
445 470
        let now = std::time::SystemTime::now();
446 471
        let old_enough = |m: &fs::Metadata| {
447 472
            m.modified()
…
21 unchanged lines
469 494
        let mut keep_refs: Vec<ObjectRef> = Vec::new();
470 495
        let mut old_paths: Vec<PathBuf> = Vec::new();
471 496
        let mut rewrite = false;
497 +
        progress("pruning packs");
472 498
        for p in self.packs.borrow().iter() {
473 499
            let meta = fs::metadata(&p.path)?;
474 500
            let unreachable = p.refs.iter().filter(|r| !keep.contains(r)).count();
…
25 unchanged lines
500 526
                    .filter(|r| !keep.contains(r))
501 527
                    .collect()
502 528
            };
503 -
            self.write_pack(&keep_refs, &[], &old_paths, ZSTD_LEVEL)?;
529 +
            self.write_pack(&keep_refs, &[], &old_paths, ZSTD_LEVEL, progress)?;
504 530
            for r in dropped {
505 531
                self.remove_sidecars(&r)?;
506 532
            }
…
57 unchanged lines
564 590
        loose_paths: &[PathBuf],
565 591
        old_packs: &[PathBuf],
566 592
        level: i32,
593 +
        progress: Progress,
567 594
    ) -> Result<PackReport> {
568 595
        if refs.is_empty() {
569 596
            for p in old_packs {
…
14 unchanged lines
584 611
        // A full repack of enough objects trains a dictionary for them.
585 612
        let dict = if level == REPACK_ZSTD_LEVEL && refs.len() >= DICT_MIN_OBJECTS {
586 613
            let _t = crate::timing::phase("pack: train a dictionary");
614 +
            progress(&format!("training a dictionary on {} objects", refs.len()));
587 615
            self.train_dictionary(&refs)
588 616
        } else {
589 617
            None
…
9 unchanged lines
599 627
            // compress it on every core, write it in order.
600 628
            let mut batches = refs.chunks(1024);
601 629
            let mut compressed_batch = Vec::new();
630 +
            let total = refs.len();
631 +
            let mut reported = usize::MAX;
602 632
            loop {
603 633
                if compressed_batch.is_empty() {
604 634
                    let Some(batch) = batches.next() else { break };
635 +
                    // Every tenth, and the first batch.
636 +
                    let tenth = report.objects * 10 / total.max(1);
637 +
                    if tenth != reported {
638 +
                        reported = tenth;
639 +
                        progress(&format!("packing {} of {total} objects", report.objects));
640 +
                    }
605 641
                    let mut read: Vec<(ObjectRef, ObjectType, Vec<u8>, bool)> =
606 642
                        Vec::with_capacity(batch.len());
607 643
                    for r in batch {
…
70 unchanged lines
678 714
        if let Some(d) = &dict {
679 715
            crate::durable::write_atomic(&self.dir.join(format!("pack-{id}.dict")), d)?;
680 716
        }
717 +
        progress(&format!("writing the index of {} objects", index.len()));
681 718
        self.finish_pack_files(&id, &tmp_pack, &index)?;
682 719
        for p in old_packs {
683 720
            remove_pack_files(p)?;
…
1354 unchanged lines
modified src/cli/ui.rs +8 −0
…
221 unchanged lines
222 222
            None => println!("{msg}"),
223 223
        }
224 224
    }
225 +
    /// Say what is happening now: the spinner's message, or, with no
226 +
    /// terminal, a line on stderr, so a long command is never silent.
227 +
    pub fn set(&self, msg: &str) {
228 +
        match &self.bar {
229 +
            Some(b) => b.set_message(msg.to_string()),
230 +
            None => eprintln!("{} {msg}", dim(glyph("›", ">"))),
231 +
        }
232 +
    }
225 233
    /// Clear the spinner without printing.
226 234
    pub fn clear(self) {
227 235
        if let Some(b) = &self.bar {
…
25 unchanged lines
modified src/cli/workspace.rs +1 −1
…
1036 unchanged lines
1037 1037
pub fn gc(now: bool) -> Result<i32> {
1038 1038
    let repo = open()?;
1039 1039
    let sp = ui::spinner("Packing and pruning…");
1040 -
    let r = repo.gc(now)?;
1040 +
    let r = repo.gc_with(now, &mut |phase| sp.set(&format!("{phase}…")))?;
1041 1041
    sp.clear();
1042 1042
    if r.packed.objects > 0 {
1043 1043
        println!(
…
833 unchanged lines