mimalloc as the global allocator (refs #41) e0d56dfca51d
Status, walks and index loads allocate many small values from every thread; glibc's arenas made that the largest single cost. On the Linux tree: one-file status 0.47 s (from 0.78 s), clean status 0.18 s (Git 0.22 s), first commit 24.4 s (from 33.3 s) at the same 3.2 GB peak, gc 29.9 s (Git 70 s), fsck 14.0 s (from 24.4 s). The glibc arena limit now applies only to the counting allocator of alloc-stats builds.
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-30 21:51 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
e0d56dfca51d391b5cdf8acd3bc3ce489c307dfac17b865a48462d93380114e6- Depends on
- 19527875ebdc 5b1114a04905 d42369abf180 d443639bcf35 ff0af1ee299d
- Depended on by
- c397a8c36a9f
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 9 atoms, 0 tree operations, 3 files
Browse the tree at this change
modified Cargo.lock +19 −0
| … | 1083 unchanged lines |
||
| 1084 | 1084 | checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" |
|
| 1085 | 1085 | ||
| 1086 | 1086 | [[package]] |
|
| 1087 | + | name = "libmimalloc-sys" |
|
| 1088 | + | version = "0.1.49" |
|
| 1089 | + | source = "registry+https://github.com/rust-lang/crates.io-index" |
|
| 1090 | + | checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" |
|
| 1091 | + | dependencies = [ |
|
| 1092 | + | "cc", |
|
| 1093 | + | ] |
|
| 1094 | + | ||
| 1095 | + | [[package]] |
|
| 1087 | 1096 | name = "linux-raw-sys" |
|
| 1088 | 1097 | version = "0.12.1" |
|
| 1089 | 1098 | source = "registry+https://github.com/rust-lang/crates.io-index" |
|
| … | 22 unchanged lines |
||
| 1112 | 1121 | version = "2.8.3" |
|
| 1113 | 1122 | source = "registry+https://github.com/rust-lang/crates.io-index" |
|
| 1114 | 1123 | checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" |
|
| 1124 | + | ||
| 1125 | + | [[package]] |
|
| 1126 | + | name = "mimalloc" |
|
| 1127 | + | version = "0.1.52" |
|
| 1128 | + | source = "registry+https://github.com/rust-lang/crates.io-index" |
|
| 1129 | + | checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" |
|
| 1130 | + | dependencies = [ |
|
| 1131 | + | "libmimalloc-sys", |
|
| 1132 | + | ] |
|
| 1115 | 1133 | ||
| 1116 | 1134 | [[package]] |
|
| 1117 | 1135 | name = "mime" |
|
| … | 137 unchanged lines |
||
| 1255 | 1273 | "console", |
|
| 1256 | 1274 | "indicatif", |
|
| 1257 | 1275 | "libc", |
|
| 1276 | + | "mimalloc", |
|
| 1258 | 1277 | "oxy-core", |
|
| 1259 | 1278 | "oxy-server", |
|
| 1260 | 1279 | "serde_json", |
|
| … | 1278 unchanged lines |
modified Cargo.toml +1 −0
| … | 29 unchanged lines |
||
| 30 | 30 | serde_json = "1" |
|
| 31 | 31 | clap_complete = "4" |
|
| 32 | 32 | clap_mangen = "0.3" |
|
| 33 | + | mimalloc = { version = "0.1", default-features = false } |
|
| 33 | 34 | ||
| 34 | 35 | [features] |
|
| 35 | 36 | # A counting global allocator: OXY_TIMINGS=1 then reports memory per phase. |
|
| … | 4 unchanged lines |
modified src/main.rs +14 −5
| … | 8 unchanged lines |
||
| 9 | 9 | #[global_allocator] |
|
| 10 | 10 | static ALLOC: oxy_core::alloc_stats::Counting = oxy_core::alloc_stats::Counting; |
|
| 11 | 11 | ||
| 12 | + | // Many small allocations from every thread (paths, entries, atoms): mimalloc |
|
| 13 | + | // serves them from per-thread pages without the arena locks of glibc's. |
|
| 14 | + | #[cfg(not(feature = "alloc-stats"))] |
|
| 15 | + | #[global_allocator] |
|
| 16 | + | static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; |
|
| 17 | + | ||
| 12 | 18 | fn main() { |
|
| 13 | 19 | // Behave like other CLI tools when output is piped into `head`: die |
|
| 14 | 20 | // quietly on SIGPIPE instead of panicking on a failed write. |
|
| … | 1 unchanged lines |
||
| 16 | 22 | unsafe { |
|
| 17 | 23 | libc::signal(libc::SIGPIPE, libc::SIG_DFL); |
|
| 18 | 24 | } |
|
| 19 | - | // glibc gives each thread that allocates an arena of its own and keeps |
|
| 20 | - | // what is freed there: the first commit of the Linux tree, read and |
|
| 21 | - | // hashed across threads, held 4.3 GB for 2.7 GB in use. Two arenas |
|
| 22 | - | // hold 3.1 GB, no slower. MALLOC_ARENA_MAX still has the last word. |
|
| 23 | - | #[cfg(all(target_os = "linux", target_env = "gnu"))] |
|
| 25 | + | // With the counting allocator (over glibc's): glibc gives each thread |
|
| 26 | + | // that allocates an arena of its own and keeps what is freed there: the |
|
| 27 | + | // first commit of the Linux tree, read and hashed across threads, held |
|
| 28 | + | // 4.3 GB for 2.7 GB in use. Two arenas hold 3.1 GB, no slower. |
|
| 29 | + | // MALLOC_ARENA_MAX still has the last word. |
|
| 30 | + | #[cfg(all(target_os = "linux", target_env = "gnu", feature = "alloc-stats"))] |
|
| 24 | 31 | if std::env::var_os("MALLOC_ARENA_MAX").is_none() { |
|
| 25 | 32 | unsafe { |
|
| 26 | 33 | libc::mallopt(libc::M_ARENA_MAX, 2); |
|
| 27 | 34 | } |
|
| 28 | 35 | } |
|
| 36 | + | let command = oxy_core::timing::phase("command"); |
|
| 29 | 37 | let result = cli::run(); |
|
| 38 | + | drop(command); |
|
| 30 | 39 | oxy_core::timing::memory_summary(); |
|
| 31 | 40 | match result { |
|
| 32 | 41 | Ok(code) => std::process::exit(code), |
|
| … | 7 unchanged lines |