oxy gc says what it is doing 9ae4d15a0e4b
A gc of 1.45 GB of loose objects ran for minutes with no output. The store's prune and repack now report their phases and, while packing, the count every tenth; Repo::gc_with passes them to a spinner, which prints each line on stderr when there is no terminal, so a gc run from a job or a shell script is never silent either. closes #66
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-10-04 02:04 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
9ae4d15a0e4b6bec06c95b87d01e2fb2f407e3fab663d3613a8cbea7bb916cca- Depends on
- 135212de5370 75167672ebcd 8f10a07a1dbe 982864068a59 a17fc622ca43 afaba5f1306d ba160f60219e d02e545dc265 da652d6392d8 fce44ae9ad29 ff0af1ee299d
- Depended on by
- 642da2636de2 ef7d4ea6f33b 767ac6cd0545 9d580f2132f3 648189388f1a
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.10.0 v0.11.0 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 25 atoms, 0 tree operations, 5 files
Browse the tree at this change
modified crates/oxy-core/src/repo/gc.rs +11 −2
| … | 24 unchanged lines |
||
| 25 | 25 | /// Prune unreachable objects older than the grace period (`now` |
|
| 26 | 26 | /// ignores it), then repack everything into one pack (spec §13.4). |
|
| 27 | 27 | pub fn gc(&self, now: bool) -> Result<GcReport> { |
|
| 28 | + | self.gc_with(now, &mut |_| {}) |
|
| 29 | + | } |
|
| 30 | + | ||
| 31 | + | /// [`Repo::gc`], reporting each phase and the packing's progress |
|
| 32 | + | /// (oxygen#66), for a spinner or a log. |
|
| 33 | + | pub fn gc_with(&self, now: bool, progress: crate::store::Progress) -> Result<GcReport> { |
|
| 34 | + | progress("finding what is reachable"); |
|
| 28 | 35 | let keep = self.reachable()?; |
|
| 29 | 36 | let grace = if now { |
|
| 30 | 37 | std::time::Duration::ZERO |
|
| 31 | 38 | } else { |
|
| 32 | 39 | std::time::Duration::from_secs(self.config.gc_grace_days * 86_400) |
|
| 33 | 40 | }; |
|
| 34 | - | let pruned = self.store.prune(&keep, grace)?; |
|
| 41 | + | let pruned = self.store.prune_with(&keep, grace, progress)?; |
|
| 35 | 42 | // Then everything left in one pack, compressed harder than a |
|
| 36 | 43 | // commit's (spec §13.4), as `git gc` repacks. |
|
| 37 | - | let (packed, merged_packs) = self.store.repack_all(crate::store::REPACK_ZSTD_LEVEL)?; |
|
| 44 | + | let (packed, merged_packs) = self |
|
| 45 | + | .store |
|
| 46 | + | .repack_all_with(crate::store::REPACK_ZSTD_LEVEL, progress)?; |
|
| 38 | 47 | Ok(GcReport { |
|
| 39 | 48 | packed, |
|
| 40 | 49 | pruned, |
|
| … | 132 unchanged lines |
modified crates/oxy-core/src/repo/tests.rs +34 −0
| … | 1096 unchanged lines |
||
| 1097 | 1097 | assert!(!c.snapshot("status").unwrap().changed); |
|
| 1098 | 1098 | } |
|
| 1099 | 1099 | ||
| 1100 | + | /// oxygen#66: a gc says what it is doing, phase by phase. |
|
| 1100 | 1101 | #[test] |
|
| 1102 | + | fn gc_reports_its_phases() { |
|
| 1103 | + | hermetic(); |
|
| 1104 | + | let dir = tempfile::tempdir().unwrap(); |
|
| 1105 | + | let repo = Repo::init(dir.path()).unwrap(); |
|
| 1106 | + | for i in 0..3 { |
|
| 1107 | + | write( |
|
| 1108 | + | dir.path(), |
|
| 1109 | + | &format!("f{i}.txt"), |
|
| 1110 | + | format!("{i}\n").as_bytes(), |
|
| 1111 | + | ); |
|
| 1112 | + | repo.commit(&format!("c{i}"), &[]).unwrap(); |
|
| 1113 | + | } |
|
| 1114 | + | let mut phases = Vec::new(); |
|
| 1115 | + | repo.gc_with(true, &mut |p| phases.push(p.to_string())) |
|
| 1116 | + | .unwrap(); |
|
| 1117 | + | assert_eq!(phases[0], "finding what is reachable", "{phases:?}"); |
|
| 1118 | + | assert!( |
|
| 1119 | + | phases.iter().any(|p| p.starts_with("pruning")), |
|
| 1120 | + | "{phases:?}" |
|
| 1121 | + | ); |
|
| 1122 | + | assert!( |
|
| 1123 | + | phases.iter().any(|p| p.starts_with("packing 0 of ")), |
|
| 1124 | + | "{phases:?}" |
|
| 1125 | + | ); |
|
| 1126 | + | assert!( |
|
| 1127 | + | phases |
|
| 1128 | + | .iter() |
|
| 1129 | + | .any(|p| p.starts_with("writing the index of ")), |
|
| 1130 | + | "{phases:?}" |
|
| 1131 | + | ); |
|
| 1132 | + | } |
|
| 1133 | + | ||
| 1134 | + | #[test] |
|
| 1101 | 1135 | fn gc_packs_objects_and_history_still_works() { |
|
| 1102 | 1136 | hermetic(); |
|
| 1103 | 1137 | let dir = tempfile::tempdir().unwrap(); |
|
| … | 5072 unchanged lines |
modified crates/oxy-core/src/store.rs +40 −3
| … | 42 unchanged lines |
||
| 43 | 43 | /// An object read from the store: its type and body. |
|
| 44 | 44 | pub type Object = (ObjectType, Vec<u8>); |
|
| 45 | 45 | ||
| 46 | + | /// Where a long operation reports what it is doing (oxygen#66): a phase, |
|
| 47 | + | /// or a count within one, as a line for a spinner or a log. |
|
| 48 | + | pub type Progress<'a> = &'a mut dyn FnMut(&str); |
|
| 49 | + | ||
| 50 | + | fn silent() -> impl FnMut(&str) { |
|
| 51 | + | |_| {} |
|
| 52 | + | } |
|
| 53 | + | ||
| 46 | 54 | #[derive(Clone, Copy, Debug)] |
|
| 47 | 55 | struct PackEntry { |
|
| 48 | 56 | offset: u64, |
|
| … | 340 unchanged lines |
||
| 389 | 397 | &loose.iter().map(|(_, p)| p.clone()).collect::<Vec<_>>(), |
|
| 390 | 398 | &[], |
|
| 391 | 399 | ZSTD_LEVEL, |
|
| 400 | + | &mut silent(), |
|
| 392 | 401 | ) |
|
| 393 | 402 | } |
|
| 394 | 403 | ||
| … | 1 unchanged lines |
||
| 396 | 405 | /// `level` (spec §13.4): what an explicit `oxy gc` does, as `git gc` |
|
| 397 | 406 | /// repacks. Returns the report and how many packs were folded in. |
|
| 398 | 407 | pub fn repack_all(&self, level: i32) -> Result<(PackReport, usize)> { |
|
| 408 | + | self.repack_all_with(level, &mut silent()) |
|
| 409 | + | } |
|
| 410 | + | ||
| 411 | + | /// [`Store::repack_all`], reporting progress. |
|
| 412 | + | pub fn repack_all_with(&self, level: i32, progress: Progress) -> Result<(PackReport, usize)> { |
|
| 399 | 413 | let loose = self.loose_objects()?; |
|
| 400 | 414 | let (mut refs, paths): (Vec<ObjectRef>, Vec<PathBuf>) = { |
|
| 401 | 415 | let packs = self.packs.borrow(); |
|
| … | 8 unchanged lines |
||
| 410 | 424 | refs.extend(loose.iter().map(|(r, _)| *r)); |
|
| 411 | 425 | let loose_paths: Vec<PathBuf> = loose.into_iter().map(|(_, p)| p).collect(); |
|
| 412 | 426 | let n = paths.len(); |
|
| 413 | - | let report = self.write_pack(&refs, &loose_paths, &paths, level)?; |
|
| 427 | + | let report = self.write_pack(&refs, &loose_paths, &paths, level, progress)?; |
|
| 414 | 428 | Ok((report, n)) |
|
| 415 | 429 | } |
|
| 416 | 430 | ||
| … | 11 unchanged lines |
||
| 428 | 442 | ) |
|
| 429 | 443 | }; |
|
| 430 | 444 | let n = paths.len(); |
|
| 431 | - | self.write_pack(&refs, &[], &paths, ZSTD_LEVEL)?; |
|
| 445 | + | self.write_pack(&refs, &[], &paths, ZSTD_LEVEL, &mut silent())?; |
|
| 432 | 446 | Ok(n) |
|
| 433 | 447 | } |
|
| 434 | 448 | ||
| … | 3 unchanged lines |
||
| 438 | 452 | /// are rewritten without them. Signature sidecars and seal-index entries |
|
| 439 | 453 | /// of deleted objects go too. |
|
| 440 | 454 | pub fn prune( |
|
| 455 | + | &self, |
|
| 456 | + | keep: &std::collections::HashSet<ObjectRef>, |
|
| 457 | + | grace: std::time::Duration, |
|
| 458 | + | ) -> Result<PruneReport> { |
|
| 459 | + | self.prune_with(keep, grace, &mut silent()) |
|
| 460 | + | } |
|
| 461 | + | ||
| 462 | + | /// [`Store::prune`], reporting progress. |
|
| 463 | + | pub fn prune_with( |
|
| 441 | 464 | &self, |
|
| 442 | 465 | keep: &std::collections::HashSet<ObjectRef>, |
|
| 443 | 466 | grace: std::time::Duration, |
|
| 467 | + | progress: Progress, |
|
| 444 | 468 | ) -> Result<PruneReport> { |
|
| 469 | + | progress("pruning loose objects"); |
|
| 445 | 470 | let now = std::time::SystemTime::now(); |
|
| 446 | 471 | let old_enough = |m: &fs::Metadata| { |
|
| 447 | 472 | m.modified() |
|
| … | 21 unchanged lines |
||
| 469 | 494 | let mut keep_refs: Vec<ObjectRef> = Vec::new(); |
|
| 470 | 495 | let mut old_paths: Vec<PathBuf> = Vec::new(); |
|
| 471 | 496 | let mut rewrite = false; |
|
| 497 | + | progress("pruning packs"); |
|
| 472 | 498 | for p in self.packs.borrow().iter() { |
|
| 473 | 499 | let meta = fs::metadata(&p.path)?; |
|
| 474 | 500 | let unreachable = p.refs.iter().filter(|r| !keep.contains(r)).count(); |
|
| … | 25 unchanged lines |
||
| 500 | 526 | .filter(|r| !keep.contains(r)) |
|
| 501 | 527 | .collect() |
|
| 502 | 528 | }; |
|
| 503 | - | self.write_pack(&keep_refs, &[], &old_paths, ZSTD_LEVEL)?; |
|
| 529 | + | self.write_pack(&keep_refs, &[], &old_paths, ZSTD_LEVEL, progress)?; |
|
| 504 | 530 | for r in dropped { |
|
| 505 | 531 | self.remove_sidecars(&r)?; |
|
| 506 | 532 | } |
|
| … | 57 unchanged lines |
||
| 564 | 590 | loose_paths: &[PathBuf], |
|
| 565 | 591 | old_packs: &[PathBuf], |
|
| 566 | 592 | level: i32, |
|
| 593 | + | progress: Progress, |
|
| 567 | 594 | ) -> Result<PackReport> { |
|
| 568 | 595 | if refs.is_empty() { |
|
| 569 | 596 | for p in old_packs { |
|
| … | 14 unchanged lines |
||
| 584 | 611 | // A full repack of enough objects trains a dictionary for them. |
|
| 585 | 612 | let dict = if level == REPACK_ZSTD_LEVEL && refs.len() >= DICT_MIN_OBJECTS { |
|
| 586 | 613 | let _t = crate::timing::phase("pack: train a dictionary"); |
|
| 614 | + | progress(&format!("training a dictionary on {} objects", refs.len())); |
|
| 587 | 615 | self.train_dictionary(&refs) |
|
| 588 | 616 | } else { |
|
| 589 | 617 | None |
|
| … | 9 unchanged lines |
||
| 599 | 627 | // compress it on every core, write it in order. |
|
| 600 | 628 | let mut batches = refs.chunks(1024); |
|
| 601 | 629 | let mut compressed_batch = Vec::new(); |
|
| 630 | + | let total = refs.len(); |
|
| 631 | + | let mut reported = usize::MAX; |
|
| 602 | 632 | loop { |
|
| 603 | 633 | if compressed_batch.is_empty() { |
|
| 604 | 634 | let Some(batch) = batches.next() else { break }; |
|
| 635 | + | // Every tenth, and the first batch. |
|
| 636 | + | let tenth = report.objects * 10 / total.max(1); |
|
| 637 | + | if tenth != reported { |
|
| 638 | + | reported = tenth; |
|
| 639 | + | progress(&format!("packing {} of {total} objects", report.objects)); |
|
| 640 | + | } |
|
| 605 | 641 | let mut read: Vec<(ObjectRef, ObjectType, Vec<u8>, bool)> = |
|
| 606 | 642 | Vec::with_capacity(batch.len()); |
|
| 607 | 643 | for r in batch { |
|
| … | 70 unchanged lines |
||
| 678 | 714 | if let Some(d) = &dict { |
|
| 679 | 715 | crate::durable::write_atomic(&self.dir.join(format!("pack-{id}.dict")), d)?; |
|
| 680 | 716 | } |
|
| 717 | + | progress(&format!("writing the index of {} objects", index.len())); |
|
| 681 | 718 | self.finish_pack_files(&id, &tmp_pack, &index)?; |
|
| 682 | 719 | for p in old_packs { |
|
| 683 | 720 | remove_pack_files(p)?; |
|
| … | 1354 unchanged lines |
modified src/cli/ui.rs +8 −0
| … | 221 unchanged lines |
||
| 222 | 222 | None => println!("{msg}"), |
|
| 223 | 223 | } |
|
| 224 | 224 | } |
|
| 225 | + | /// Say what is happening now: the spinner's message, or, with no |
|
| 226 | + | /// terminal, a line on stderr, so a long command is never silent. |
|
| 227 | + | pub fn set(&self, msg: &str) { |
|
| 228 | + | match &self.bar { |
|
| 229 | + | Some(b) => b.set_message(msg.to_string()), |
|
| 230 | + | None => eprintln!("{} {msg}", dim(glyph("›", ">"))), |
|
| 231 | + | } |
|
| 232 | + | } |
|
| 225 | 233 | /// Clear the spinner without printing. |
|
| 226 | 234 | pub fn clear(self) { |
|
| 227 | 235 | if let Some(b) = &self.bar { |
|
| … | 25 unchanged lines |
modified src/cli/workspace.rs +1 −1
| … | 1036 unchanged lines |
||
| 1037 | 1037 | pub fn gc(now: bool) -> Result<i32> { |
|
| 1038 | 1038 | let repo = open()?; |
|
| 1039 | 1039 | let sp = ui::spinner("Packing and pruning…"); |
|
| 1040 | - | let r = repo.gc(now)?; |
|
| 1040 | + | let r = repo.gc_with(now, &mut |phase| sp.set(&format!("{phase}…")))?; |
|
| 1041 | 1041 | sp.clear(); |
|
| 1042 | 1042 | if r.packed.objects > 0 { |
|
| 1043 | 1043 | println!( |
|
| … | 833 unchanged lines |