A repository kept in a bucket lists its packs once a while (closes #116) 87c948720343
Every Store::open lists the bucket's pack- keys, and a forge opens the store once per request: about 33 LISTs a minute on oxygenhub.io now. The shared cloud client keeps the pack- listing for 30 s, the window of its missing-key memory, and forgets it the moment this process writes or deletes a key under it, so its own packs are never hidden; another writer's new pack is seen within 30 s, as a missing key already is. Other prefixes are never kept (gc lists loose objects fresh). Test: kept, other prefixes not, a loose write leaves it, a pack write or age forgets it; the cloud conformance runs pass. Closes #116
- Author
- Andy McCall @andymccall <[email protected]>
- via
- agent
agent:claude@andymccall <[email protected]> · keyless, under your grant for changes,refs,seal · key 68113d48434d
- When
- · 2026-10-18 01:22 UTC
- Signature
- ◈ sealed by agent:claude@andymccall
- Hash
87c948720343629fc6727793e820b56557c7fe7ec603b45bdd36a8e07d04a693- Depends on
- 1e0fbb47ab90 8b53b935dd03 8d80356371c5 c9369ad8b1b1
- Depended on by
- 767ac6cd0545 30644f1ecb7b
- In bookmarks
- carried-not-removed doctor-trust identical-sides keyless-tags main merge-remote-bookmark paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state
- In tags
- v0.10.0 v0.11.0 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 9 atoms, 0 tree operations, 1 file
Browse the tree at this change
modified crates/oxy-strata/src/lib.rs +75 −0
| … | 398 unchanged lines |
||
| 399 | 399 | runtime: Option<tokio::runtime::Runtime>, |
|
| 400 | 400 | /// Keys `has` found missing lately (see [`Misses`]). |
|
| 401 | 401 | misses: Misses, |
|
| 402 | + | /// Pack listings, kept a moment (see [`Listings`]). |
|
| 403 | + | listings: Listings, |
|
| 402 | 404 | } |
|
| 403 | 405 | ||
| 404 | 406 | /// How long a key found missing is taken as missing without asking again. |
|
| … | 33 unchanged lines |
||
| 438 | 440 | } |
|
| 439 | 441 | } |
|
| 440 | 442 | ||
| 443 | + | /// Listings kept for [`MISS_FOR`] (oxygen#116): the `pack-` keys, which |
|
| 444 | + | /// every open of a repository's store lists, so a forge serving a |
|
| 445 | + | /// repository from a bucket lists them once a while rather than once a |
|
| 446 | + | /// request. Like [`Misses`], this process forgets a listing the moment it |
|
| 447 | + | /// writes or deletes a key under it; another writer's pack is seen within |
|
| 448 | + | /// `MISS_FOR`, as a missing key is. |
|
| 449 | + | #[derive(Default)] |
|
| 450 | + | struct Listings(Mutex<std::collections::HashMap<String, (std::time::Instant, Vec<Entry>)>>); |
|
| 451 | + | ||
| 452 | + | impl Listings { |
|
| 453 | + | /// The prefixes kept: those every open lists. |
|
| 454 | + | fn kept(prefix: &str) -> bool { |
|
| 455 | + | prefix == "pack-" |
|
| 456 | + | } |
|
| 457 | + | ||
| 458 | + | fn get(&self, prefix: &str) -> Option<Vec<Entry>> { |
|
| 459 | + | if !Self::kept(prefix) { |
|
| 460 | + | return None; |
|
| 461 | + | } |
|
| 462 | + | let map = self.0.lock().unwrap_or_else(|p| p.into_inner()); |
|
| 463 | + | map.get(prefix) |
|
| 464 | + | .filter(|(t, _)| t.elapsed() < MISS_FOR) |
|
| 465 | + | .map(|(_, e)| e.clone()) |
|
| 466 | + | } |
|
| 467 | + | ||
| 468 | + | fn put(&self, prefix: &str, entries: &[Entry]) { |
|
| 469 | + | if Self::kept(prefix) { |
|
| 470 | + | self.0.lock().unwrap_or_else(|p| p.into_inner()).insert( |
|
| 471 | + | prefix.to_string(), |
|
| 472 | + | (std::time::Instant::now(), entries.to_vec()), |
|
| 473 | + | ); |
|
| 474 | + | } |
|
| 475 | + | } |
|
| 476 | + | ||
| 477 | + | fn written(&self, key: &str) { |
|
| 478 | + | self.0 |
|
| 479 | + | .lock() |
|
| 480 | + | .unwrap_or_else(|p| p.into_inner()) |
|
| 481 | + | .retain(|prefix, _| !key.starts_with(prefix.as_str())); |
|
| 482 | + | } |
|
| 483 | + | } |
|
| 484 | + | ||
| 441 | 485 | impl Drop for CloudStore { |
|
| 442 | 486 | /// Without waiting, so a store can be dropped inside a runtime too. |
|
| 443 | 487 | fn drop(&mut self) { |
|
| … | 87 unchanged lines |
||
| 531 | 575 | hint, |
|
| 532 | 576 | runtime: Some(runtime), |
|
| 533 | 577 | misses: Misses::default(), |
|
| 578 | + | listings: Listings::default(), |
|
| 534 | 579 | }) |
|
| 535 | 580 | } |
|
| 536 | 581 | ||
| … | 162 unchanged lines |
||
| 699 | 744 | fn put_new(&self, key: &str, bytes: &[u8]) -> Result<()> { |
|
| 700 | 745 | let path = self.path(key)?; |
|
| 701 | 746 | self.misses.written(key); |
|
| 747 | + | self.listings.written(key); |
|
| 702 | 748 | let opts = PutOptions { |
|
| 703 | 749 | mode: PutMode::Create, |
|
| 704 | 750 | ..Default::default() |
|
| … | 8 unchanged lines |
||
| 713 | 759 | } |
|
| 714 | 760 | ||
| 715 | 761 | fn list(&self, prefix: &str) -> Result<Vec<Entry>> { |
|
| 762 | + | if let Some(kept) = self.listings.get(prefix) { |
|
| 763 | + | return Ok(kept); |
|
| 764 | + | } |
|
| 716 | 765 | // Providers list by whole path segments; the asked prefix may end |
|
| 717 | 766 | // mid-segment, so list its directory and filter here. |
|
| 718 | 767 | let dir = prefix.rsplit_once('/').map(|(d, _)| d).unwrap_or(""); |
|
| … | 28 unchanged lines |
||
| 747 | 796 | }) |
|
| 748 | 797 | .collect(); |
|
| 749 | 798 | out.sort_by(|a, b| a.key.cmp(&b.key)); |
|
| 799 | + | self.listings.put(prefix, &out); |
|
| 750 | 800 | Ok(out) |
|
| 751 | 801 | } |
|
| 752 | 802 | ||
| 753 | 803 | fn delete(&self, key: &str) -> Result<()> { |
|
| 754 | 804 | let path = self.path(key)?; |
|
| 805 | + | self.listings.written(key); |
|
| 755 | 806 | counted(Op::Delete); |
|
| 756 | 807 | match self.block(self.inner.delete(&path)) { |
|
| 757 | 808 | Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()), |
|
| … | 23 unchanged lines |
||
| 781 | 832 | fn write_ref(&self, key: &str, bytes: &[u8], expected: Option<&Version>) -> Result<Version> { |
|
| 782 | 833 | let path = self.path(key)?; |
|
| 783 | 834 | self.misses.written(key); |
|
| 835 | + | self.listings.written(key); |
|
| 784 | 836 | let mode = match expected { |
|
| 785 | 837 | None => PutMode::Create, |
|
| 786 | 838 | Some(v) => PutMode::Update(Self::update_version(v)), |
|
| … | 38 unchanged lines |
||
| 825 | 877 | } |
|
| 826 | 878 | ||
| 827 | 879 | /// Every kind of request is reported, once and by its own name. |
|
| 880 | + | /// The pack listing is kept until a pack key is written or deleted |
|
| 881 | + | /// here, or for MISS_FOR; other prefixes are never kept (oxygen#116). |
|
| 882 | + | #[test] |
|
| 883 | + | fn the_pack_listing_is_kept_until_a_pack_is_written() { |
|
| 884 | + | let l = super::Listings::default(); |
|
| 885 | + | let e = |k: &str| oxy_core::strata::Entry { |
|
| 886 | + | key: k.into(), |
|
| 887 | + | size: 1, |
|
| 888 | + | modified_ms: None, |
|
| 889 | + | }; |
|
| 890 | + | l.put("pack-", &[e("pack-1.idx")]); |
|
| 891 | + | assert_eq!(l.get("pack-").map(|v| v.len()), Some(1)); |
|
| 892 | + | l.put("loose/", &[e("loose/ab")]); |
|
| 893 | + | assert!(l.get("loose/").is_none(), "only the pack listing is kept"); |
|
| 894 | + | l.written("loose/cd"); |
|
| 895 | + | assert!(l.get("pack-").is_some(), "a loose object leaves it"); |
|
| 896 | + | l.written("pack-2.pack"); |
|
| 897 | + | assert!(l.get("pack-").is_none(), "a new pack forgets it"); |
|
| 898 | + | l.put("pack-", &[e("pack-1.idx")]); |
|
| 899 | + | l.0.lock().unwrap().get_mut("pack-").unwrap().0 -= super::MISS_FOR; |
|
| 900 | + | assert!(l.get("pack-").is_none(), "and it ages out"); |
|
| 901 | + | } |
|
| 902 | + | ||
| 828 | 903 | #[test] |
|
| 829 | 904 | fn stats_name_every_kind_of_request() { |
|
| 830 | 905 | let names: Vec<&str> = super::stats().ops().map(|(n, _)| n).collect(); |
|
| … | 119 unchanged lines |