Read many objects in pack and offset order 795b8b502ded
Store::get_many reads packed objects by pack then offset, each thread a run of them, so a spinning disk sweeps rather than seeks; results keep the asked order. Not measured cold (dropping Earth's page cache needs root). refs #137
- Author
- Andy McCall @andymccall <[email protected]>
- via
- agent
agent:claude@andymccall <[email protected]> · keyless, under your grant for changes,refs,seal · key 6263c1a9ffcb
- When
- · 2026-10-26 03:56 UTC
- Signature
- ◈ sealed by agent:claude@andymccall
- Hash
795b8b502dedd45bc3959c80eec48b43082faf9596e1133f71df0b70724abf1f- Depends on
- 092457020692 544d0541a7e5 648189388f1a 767ac6cd0545 8fd78eb52ad2 afaba5f1306d e4595038502a ff0af1ee299d
- Depended on by
- e02edccce98c
- In bookmarks
- identical-sides main push-notes subrepo-pin-state
- In tags
- v0.10.0 v0.11.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 7 atoms, 0 tree operations, 2 files
Browse the tree at this change
modified CHANGELOG.md +1 −0
| … | 1 unchanged lines |
||
| 2 | 2 | ||
| 3 | 3 | ## Unreleased |
|
| 4 | 4 | ||
| 5 | + | - **Many objects are read in pack order (#137):** `Store::get_many` (a clone's checkpoint, a fetch's objects) reads packed objects by pack and offset, each thread a run of them, so a spinning disk sweeps instead of seeking across the packs; results keep the asked order. The server's gathering of a VS Code-sized checkpoint (tens of thousands of objects in ~95 packs) was the slow part of a depth-0 clone on Earth. |
|
| 5 | 6 | - **Windows paths print plainly (#143):** a repository's root showed as `\\?\C:\work\demo` in every message, Windows' verbatim form from `canonicalize`; oxy now drops the prefix where the plain path says the same (`oxy_core::portable::canonical`). Found running the shipped Windows build on a Windows 11 VM. |
|
| 6 | 7 | - **Oxygen's CI writes a test report beside its coverage (#139):** the test job runs under cargo-nextest (`.config/nextest.toml`, profile `ci`) as well as cargo-llvm-cov, keeping a JUnit report whether or not the tests pass, for the hub's Tests tab. |
|
| 7 | 8 | - **Pushing a new bookmark sends only what the remote lacks (#142).** A push was worked out against the remote's heads for that bookmark alone, so a bookmark new to the remote sent its whole history again: slow on a big repository, and OxygenHub's push protection scanned all of it (and refused a proposal's bookmark for an example key in a change main had had for weeks). The heads of every bookmark the remote lists now count as already there. |
|
| … | 270 unchanged lines |
modified crates/oxy-core/src/store.rs +49 −9
| … | 1887 unchanged lines |
||
| 1888 | 1888 | }) |
|
| 1889 | 1889 | .collect(); |
|
| 1890 | 1890 | let indexes: &[Arc<PackIndex>] = &packs; |
|
| 1891 | - | let found: Vec<Option<Result<Object>>> = refs |
|
| 1891 | + | // Read in (pack, offset) order, each thread a run of it, so a |
|
| 1892 | + | // spinning disk sweeps rather than seeks (oxygen#137: a clone's |
|
| 1893 | + | // checkpoint reads tens of thousands of objects across ~95 packs). |
|
| 1894 | + | let order = read_order(&place); |
|
| 1895 | + | let fetched: Vec<(usize, Option<Result<Object>>)> = order |
|
| 1892 | 1896 | .par_iter() |
|
| 1893 | - | .zip(place.par_iter()) |
|
| 1894 | - | .map(|(r, at)| match at { |
|
| 1895 | - | Some((p, e)) => Some(indexes[*p].fetch(backend, r, *e)), |
|
| 1896 | - | None => match get_optional(backend, &loose_key(r)) { |
|
| 1897 | - | Ok(Some(bytes)) => Some(parse_loose(&bytes, r)), |
|
| 1898 | - | Ok(None) => None, |
|
| 1899 | - | Err(e) => Some(Err(e)), |
|
| 1900 | - | }, |
|
| 1897 | + | .map(|&i| { |
|
| 1898 | + | let r = &refs[i]; |
|
| 1899 | + | let got = match &place[i] { |
|
| 1900 | + | Some((p, e)) => Some(indexes[*p].fetch(backend, r, *e)), |
|
| 1901 | + | None => match get_optional(backend, &loose_key(r)) { |
|
| 1902 | + | Ok(Some(bytes)) => Some(parse_loose(&bytes, r)), |
|
| 1903 | + | Ok(None) => None, |
|
| 1904 | + | Err(e) => Some(Err(e)), |
|
| 1905 | + | }, |
|
| 1906 | + | }; |
|
| 1907 | + | (i, got) |
|
| 1901 | 1908 | }) |
|
| 1902 | 1909 | .collect(); |
|
| 1903 | 1910 | drop(packs); |
|
| 1911 | + | let mut found: Vec<Option<Result<Object>>> = (0..refs.len()).map(|_| None).collect(); |
|
| 1912 | + | for (i, got) in fetched { |
|
| 1913 | + | found[i] = got; |
|
| 1914 | + | } |
|
| 1904 | 1915 | found |
|
| 1905 | 1916 | .into_iter() |
|
| 1906 | 1917 | .zip(refs) |
|
| … | 900 unchanged lines |
||
| 2807 | 2818 | } |
|
| 2808 | 2819 | ||
| 2809 | 2820 | /// A loose object's bytes, verified against `r`. |
|
| 2821 | + | /// The order to read objects in: packed ones by pack then offset, loose |
|
| 2822 | + | /// ones after, each group keeping the asked order otherwise. |
|
| 2823 | + | fn read_order(place: &[Option<(usize, PackEntry)>]) -> Vec<usize> { |
|
| 2824 | + | let mut order: Vec<usize> = (0..place.len()).collect(); |
|
| 2825 | + | order.sort_by_key(|&i| match &place[i] { |
|
| 2826 | + | Some((p, e)) => (0, *p, e.offset), |
|
| 2827 | + | None => (1, 0, 0), |
|
| 2828 | + | }); |
|
| 2829 | + | order |
|
| 2830 | + | } |
|
| 2831 | + | ||
| 2810 | 2832 | fn parse_loose(data: &[u8], r: &ObjectRef) -> Result<Object> { |
|
| 2811 | 2833 | let nul = data |
|
| 2812 | 2834 | .iter() |
|
| … | 603 unchanged lines |
||
| 3416 | 3438 | b"dropped" |
|
| 3417 | 3439 | ); |
|
| 3418 | 3440 | }); |
|
| 3441 | + | } |
|
| 3442 | + | ||
| 3443 | + | #[test] |
|
| 3444 | + | fn packed_objects_are_read_by_pack_then_offset() { |
|
| 3445 | + | let at = |p: usize, offset: u64| { |
|
| 3446 | + | Some(( |
|
| 3447 | + | p, |
|
| 3448 | + | PackEntry { |
|
| 3449 | + | offset, |
|
| 3450 | + | stored_len: 1, |
|
| 3451 | + | raw_len: 1, |
|
| 3452 | + | ty: ObjectType::Chunk, |
|
| 3453 | + | flags: 0, |
|
| 3454 | + | }, |
|
| 3455 | + | )) |
|
| 3456 | + | }; |
|
| 3457 | + | let place = vec![at(1, 50), None, at(0, 900), at(1, 10), None, at(0, 5)]; |
|
| 3458 | + | assert_eq!(read_order(&place), [5, 2, 3, 0, 1, 4]); |
|
| 3419 | 3459 | } |
|
| 3420 | 3460 | ||
| 3421 | 3461 | #[test] |
|
| … | 1246 unchanged lines |