Walk the working copy in parallel, and keep its stats 8b276e0106d1
A snapshot found and statted the working copy's files on one thread, then wrote the manifest by statting every file again. The walk now runs on every core, and the stat taken during the walk (before the file is read, so a file changed meanwhile is read again next time) goes into the manifest. Snapshot phases for the tree view, the manifest read and the recording are now timed too. A status with one file changed on the Linux tip takes 2.5 s (from 4.1 s). refs #35
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-09-29 23:02 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
8b276e0106d174e76b338cd7da55a3d717121c59622c663afe43aaf16655d7cb- Depends on
- 0bdc3f91a39f 135212de5370 1d360c91a289 69816d688300 7421309f5372 75167672ebcd 982864068a59 cc7a00565161 e7ddd552b619
- Depended on by
- e96a430c78f3 76740263089c 992467d31d26 d0990c2bff74 d0d52107b074 17f94b13ae54 192fad1e46a2 206c76cad4bf
- In bookmarks
- agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 11 atoms, 0 tree operations, 3 files
Browse the tree at this change
modified crates/oxy-core/src/repo/mod.rs +77 −34
| … | 1144 unchanged lines |
||
| 1145 | 1145 | let nested = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); |
|
| 1146 | 1146 | let wb = self.walker(&nested); |
|
| 1147 | 1147 | let mut out = BTreeMap::new(); |
|
| 1148 | + | // The walk itself runs on every core: each entry is found and |
|
| 1149 | + | // statted on its own. |
|
| 1150 | + | enum Found { |
|
| 1151 | + | File(String, PathBuf, fs::Metadata), |
|
| 1152 | + | Symlink(String, Vec<u8>), |
|
| 1153 | + | } |
|
| 1154 | + | let found: std::sync::Mutex<Vec<Found>> = std::sync::Mutex::new(Vec::new()); |
|
| 1155 | + | let failed: std::sync::Mutex<Option<String>> = std::sync::Mutex::new(None); |
|
| 1156 | + | let (root, sparse) = (&self.root, &self.config.sparse_paths); |
|
| 1157 | + | let in_scope = |p: &str| { |
|
| 1158 | + | sparse.is_empty() |
|
| 1159 | + | || sparse |
|
| 1160 | + | .iter() |
|
| 1161 | + | .any(|s| p == s || p.starts_with(&format!("{s}/"))) |
|
| 1162 | + | }; |
|
| 1163 | + | wb.build_parallel().run(|| { |
|
| 1164 | + | let (found, failed) = (&found, &failed); |
|
| 1165 | + | Box::new(move |entry| { |
|
| 1166 | + | use ignore::WalkState; |
|
| 1167 | + | let step = || -> std::result::Result<Option<Found>, String> { |
|
| 1168 | + | let entry = entry.map_err(|e| e.to_string())?; |
|
| 1169 | + | let path = entry.path(); |
|
| 1170 | + | if path == root { |
|
| 1171 | + | return Ok(None); |
|
| 1172 | + | } |
|
| 1173 | + | let rel: String = path |
|
| 1174 | + | .strip_prefix(root) |
|
| 1175 | + | .unwrap() |
|
| 1176 | + | .components() |
|
| 1177 | + | .map(|c| c.as_os_str().to_string_lossy().to_string()) |
|
| 1178 | + | .collect::<Vec<_>>() |
|
| 1179 | + | .join("/"); |
|
| 1180 | + | let Some(ft) = entry.file_type() else { |
|
| 1181 | + | return Ok(None); |
|
| 1182 | + | }; |
|
| 1183 | + | if ft.is_dir() || !in_scope(&rel) { |
|
| 1184 | + | return Ok(None); |
|
| 1185 | + | } |
|
| 1186 | + | if ft.is_symlink() { |
|
| 1187 | + | let target = fs::read_link(path).map_err(|e| e.to_string())?; |
|
| 1188 | + | return Ok(Some(Found::Symlink( |
|
| 1189 | + | rel, |
|
| 1190 | + | target.to_string_lossy().as_bytes().to_vec(), |
|
| 1191 | + | ))); |
|
| 1192 | + | } |
|
| 1193 | + | if ft.is_file() { |
|
| 1194 | + | let meta = entry.metadata().map_err(|e| e.to_string())?; |
|
| 1195 | + | return Ok(Some(Found::File(rel, path.to_path_buf(), meta))); |
|
| 1196 | + | } |
|
| 1197 | + | Ok(None) |
|
| 1198 | + | }; |
|
| 1199 | + | match step() { |
|
| 1200 | + | Ok(Some(f)) => found.lock().unwrap().push(f), |
|
| 1201 | + | Ok(None) => {} |
|
| 1202 | + | Err(e) => { |
|
| 1203 | + | failed.lock().unwrap().get_or_insert(e); |
|
| 1204 | + | return WalkState::Quit; |
|
| 1205 | + | } |
|
| 1206 | + | } |
|
| 1207 | + | WalkState::Continue |
|
| 1208 | + | }) |
|
| 1209 | + | }); |
|
| 1210 | + | if let Some(e) = failed.into_inner().unwrap() { |
|
| 1211 | + | return fail(e); |
|
| 1212 | + | } |
|
| 1148 | 1213 | let mut files: Vec<(String, PathBuf, fs::Metadata)> = Vec::new(); |
|
| 1149 | - | for entry in wb.build() { |
|
| 1150 | - | let entry = entry.map_err(|e| RepoError::Msg(e.to_string()))?; |
|
| 1151 | - | let path = entry.path(); |
|
| 1152 | - | if path == self.root { |
|
| 1153 | - | continue; |
|
| 1214 | + | for f in found.into_inner().unwrap() { |
|
| 1215 | + | match f { |
|
| 1216 | + | Found::File(rel, path, meta) => files.push((rel, path, meta)), |
|
| 1217 | + | Found::Symlink(rel, target) => { |
|
| 1218 | + | out.insert(rel, FsEntry::new(NodeKind::Symlink, false, target)); |
|
| 1154 | 1219 | } |
|
| 1155 | - | let rel = path.strip_prefix(&self.root).unwrap(); |
|
| 1156 | - | let rel: String = rel |
|
| 1157 | - | .components() |
|
| 1158 | - | .map(|c| c.as_os_str().to_string_lossy().to_string()) |
|
| 1159 | - | .collect::<Vec<_>>() |
|
| 1160 | - | .join("/"); |
|
| 1161 | - | let Some(ft) = entry.file_type() else { |
|
| 1162 | - | continue; |
|
| 1163 | - | }; |
|
| 1164 | - | if ft.is_dir() || !self.in_scope(&rel) { |
|
| 1165 | - | continue; |
|
| 1166 | - | } |
|
| 1167 | - | if ft.is_symlink() { |
|
| 1168 | - | let target = fs::read_link(path)?; |
|
| 1169 | - | out.insert( |
|
| 1170 | - | rel, |
|
| 1171 | - | FsEntry::new( |
|
| 1172 | - | NodeKind::Symlink, |
|
| 1173 | - | false, |
|
| 1174 | - | target.to_string_lossy().as_bytes().to_vec(), |
|
| 1175 | - | ), |
|
| 1176 | - | ); |
|
| 1177 | - | } else if ft.is_file() { |
|
| 1178 | - | let meta = entry |
|
| 1179 | - | .metadata() |
|
| 1180 | - | .map_err(|e| RepoError::Msg(e.to_string()))?; |
|
| 1181 | - | files.push((rel, path.to_path_buf(), meta)); |
|
| 1182 | 1220 | } |
|
| 1183 | 1221 | } |
|
| 1184 | 1222 | let keys = keys.filter(|k| { |
|
| … | 448 unchanged lines |
||
| 1633 | 1671 | key: Option<&str>, |
|
| 1634 | 1672 | ) -> Result<FsEntry> { |
|
| 1635 | 1673 | let exec = is_executable(meta); |
|
| 1636 | - | Ok(match stat_cached(rel, meta, cache) { |
|
| 1674 | + | let stat = Some((meta.len(), prompt::mtime_ns(meta))); |
|
| 1675 | + | let mut e = match stat_cached(rel, meta, cache) { |
|
| 1637 | 1676 | Some(e) if e.blob.is_some() => FsEntry { |
|
| 1638 | 1677 | blob: e.blob, |
|
| 1639 | 1678 | content_hash: Some(e.hash.clone()), |
|
| … | 10 unchanged lines |
||
| 1650 | 1689 | ..FsEntry::new(NodeKind::File, exec, fs::read(path)?) |
|
| 1651 | 1690 | }, |
|
| 1652 | 1691 | None => FsEntry::new(NodeKind::File, exec, fs::read(path)?), |
|
| 1653 | - | }) |
|
| 1692 | + | }; |
|
| 1693 | + | // The stat the manifest records: taken before the read, so a file that |
|
| 1694 | + | // changes meanwhile is read again next time. |
|
| 1695 | + | e.stat = stat; |
|
| 1696 | + | Ok(e) |
|
| 1654 | 1697 | } |
modified crates/oxy-core/src/repo/snapshot.rs +8 −1
| … | 287 unchanged lines |
||
| 288 | 288 | let _t = crate::timing::phase("snapshot: load base history"); |
|
| 289 | 289 | self.load_state_history(&old_wc.deps)? |
|
| 290 | 290 | }; |
|
| 291 | + | let keys_phase = crate::timing::phase("snapshot: tree view and graph keys"); |
|
| 291 | 292 | let view = base.tree().view(); |
|
| 292 | 293 | let keys: HashMap<String, String> = view |
|
| 293 | 294 | .entries |
|
| … | 4 unchanged lines |
||
| 298 | 299 | Some((p.clone(), crate::hash::ObjectRef(k).to_hex())) |
|
| 299 | 300 | }) |
|
| 300 | 301 | .collect(); |
|
| 301 | - | let manifest = self.read_manifest()?.map(|(_, _, m)| m); |
|
| 302 | + | drop(keys_phase); |
|
| 303 | + | let manifest = { |
|
| 304 | + | let _t = crate::timing::phase("snapshot: read manifest"); |
|
| 305 | + | self.read_manifest()?.map(|(_, _, m)| m) |
|
| 306 | + | }; |
|
| 302 | 307 | let tracked: BTreeMap<String, (NodeKind, Option<ObjectRef>)> = { |
|
| 303 | 308 | let _t = crate::timing::phase("snapshot: tracked paths"); |
|
| 304 | 309 | let mut t = BTreeMap::new(); |
|
| … | 85 unchanged lines |
||
| 390 | 395 | let contributors = built.contributors; |
|
| 391 | 396 | let statuses = built.statuses; |
|
| 392 | 397 | // Empty snapshot: hash the candidate before storing; only record an op on change. |
|
| 398 | + | let record = crate::timing::phase("snapshot: hash and record"); |
|
| 393 | 399 | let new_hash = wc.hash(); |
|
| 394 | 400 | let changed = new_hash != old_hash; |
|
| 395 | 401 | if changed { |
|
| … | 2 unchanged lines |
||
| 398 | 404 | view.workspaces.insert(self.workspace, new_hash); |
|
| 399 | 405 | self.record_op(&[op_ref], &format!("{command}{SNAPSHOT_SUFFIX}"), view)?; |
|
| 400 | 406 | } |
|
| 407 | + | drop(record); |
|
| 401 | 408 | // The prompt compares against this without recording anything. |
|
| 402 | 409 | let _t = crate::timing::phase("snapshot: write manifest"); |
|
| 403 | 410 | self.write_manifest_entries( |
|
| … | 1937 unchanged lines |
modified docs/benchmarks.md +10 −0
| … | 343 unchanged lines |
||
| 344 | 344 | | `blame MAINTAINERS` | killed at 14 GB | 1.1 s, 315 MB | 0.31 s | |
|
| 345 | 345 | | `fsck` | killed at 14 GB | 123 s, 4.5 GB | to measure again | |
|
| 346 | 346 | ||
| 347 | + | ### One file changed (2026-09-30, oxygen#35) |
|
| 348 | + | ||
| 349 | + | A status with one file changed (MAINTAINERS) took 4.1 s: the walk ran |
|
| 350 | + | on one thread (0.83 s), and the manifest was written by statting all |
|
| 351 | + | 96 000 files again (0.57 s). The walk now runs in parallel and each |
|
| 352 | + | file's stat is carried from the walk into the manifest. It now takes |
|
| 353 | + | 2.5 s: the base's history 0.5 s, its tree view and graph keys 0.38 s, |
|
| 354 | + | the walk 0.35 s, rendering the changed file 0.47 s, the diff 0.18 s, |
|
| 355 | + | and the manifest 0.14 s. (Git: 0.23 s.) |
|
| 356 | + |