OXYGENHUB

Walk the working copy in parallel, and keep its stats 8b276e0106d1

A snapshot found and statted the working copy's files on one thread,
then wrote the manifest by statting every file again. The walk now runs
on every core, and the stat taken during the walk (before the file is
read, so a file changed meanwhile is read again next time) goes into
the manifest. Snapshot phases for the tree view, the manifest read and
the recording are now timed too. A status with one file changed on the
Linux tip takes 2.5 s (from 4.1 s).

refs #35
Author
Andy McCall @andymccall <[email protected]>
When
· 2026-09-29 23:02 UTC
Signature
✔ signed by Andy McCall
Hash
8b276e0106d174e76b338cd7da55a3d717121c59622c663afe43aaf16655d7cb
Depends on
0bdc3f91a39f 135212de5370 1d360c91a289 69816d688300 7421309f5372 75167672ebcd 982864068a59 cc7a00565161 e7ddd552b619
Depended on by
e96a430c78f3 76740263089c 992467d31d26 d0990c2bff74 d0d52107b074 17f94b13ae54 192fad1e46a2 206c76cad4bf
In bookmarks
agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
In tags
v0.1.0 v0.10.0 v0.11.0 v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
Edits
11 atoms, 0 tree operations, 3 files

Browse the tree at this change

modified crates/oxy-core/src/repo/mod.rs +77 −34
…
1144 unchanged lines
1145 1145
        let nested = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
1146 1146
        let wb = self.walker(&nested);
1147 1147
        let mut out = BTreeMap::new();
1148 +
        // The walk itself runs on every core: each entry is found and
1149 +
        // statted on its own.
1150 +
        enum Found {
1151 +
            File(String, PathBuf, fs::Metadata),
1152 +
            Symlink(String, Vec<u8>),
1153 +
        }
1154 +
        let found: std::sync::Mutex<Vec<Found>> = std::sync::Mutex::new(Vec::new());
1155 +
        let failed: std::sync::Mutex<Option<String>> = std::sync::Mutex::new(None);
1156 +
        let (root, sparse) = (&self.root, &self.config.sparse_paths);
1157 +
        let in_scope = |p: &str| {
1158 +
            sparse.is_empty()
1159 +
                || sparse
1160 +
                    .iter()
1161 +
                    .any(|s| p == s || p.starts_with(&format!("{s}/")))
1162 +
        };
1163 +
        wb.build_parallel().run(|| {
1164 +
            let (found, failed) = (&found, &failed);
1165 +
            Box::new(move |entry| {
1166 +
                use ignore::WalkState;
1167 +
                let step = || -> std::result::Result<Option<Found>, String> {
1168 +
                    let entry = entry.map_err(|e| e.to_string())?;
1169 +
                    let path = entry.path();
1170 +
                    if path == root {
1171 +
                        return Ok(None);
1172 +
                    }
1173 +
                    let rel: String = path
1174 +
                        .strip_prefix(root)
1175 +
                        .unwrap()
1176 +
                        .components()
1177 +
                        .map(|c| c.as_os_str().to_string_lossy().to_string())
1178 +
                        .collect::<Vec<_>>()
1179 +
                        .join("/");
1180 +
                    let Some(ft) = entry.file_type() else {
1181 +
                        return Ok(None);
1182 +
                    };
1183 +
                    if ft.is_dir() || !in_scope(&rel) {
1184 +
                        return Ok(None);
1185 +
                    }
1186 +
                    if ft.is_symlink() {
1187 +
                        let target = fs::read_link(path).map_err(|e| e.to_string())?;
1188 +
                        return Ok(Some(Found::Symlink(
1189 +
                            rel,
1190 +
                            target.to_string_lossy().as_bytes().to_vec(),
1191 +
                        )));
1192 +
                    }
1193 +
                    if ft.is_file() {
1194 +
                        let meta = entry.metadata().map_err(|e| e.to_string())?;
1195 +
                        return Ok(Some(Found::File(rel, path.to_path_buf(), meta)));
1196 +
                    }
1197 +
                    Ok(None)
1198 +
                };
1199 +
                match step() {
1200 +
                    Ok(Some(f)) => found.lock().unwrap().push(f),
1201 +
                    Ok(None) => {}
1202 +
                    Err(e) => {
1203 +
                        failed.lock().unwrap().get_or_insert(e);
1204 +
                        return WalkState::Quit;
1205 +
                    }
1206 +
                }
1207 +
                WalkState::Continue
1208 +
            })
1209 +
        });
1210 +
        if let Some(e) = failed.into_inner().unwrap() {
1211 +
            return fail(e);
1212 +
        }
1148 1213
        let mut files: Vec<(String, PathBuf, fs::Metadata)> = Vec::new();
1149 -
        for entry in wb.build() {
1150 -
            let entry = entry.map_err(|e| RepoError::Msg(e.to_string()))?;
1151 -
            let path = entry.path();
1152 -
            if path == self.root {
1153 -
                continue;
1214 +
        for f in found.into_inner().unwrap() {
1215 +
            match f {
1216 +
                Found::File(rel, path, meta) => files.push((rel, path, meta)),
1217 +
                Found::Symlink(rel, target) => {
1218 +
                    out.insert(rel, FsEntry::new(NodeKind::Symlink, false, target));
1154 1219
                }
1155 -
            let rel = path.strip_prefix(&self.root).unwrap();
1156 -
            let rel: String = rel
1157 -
                .components()
1158 -
                .map(|c| c.as_os_str().to_string_lossy().to_string())
1159 -
                .collect::<Vec<_>>()
1160 -
                .join("/");
1161 -
            let Some(ft) = entry.file_type() else {
1162 -
                continue;
1163 -
            };
1164 -
            if ft.is_dir() || !self.in_scope(&rel) {
1165 -
                continue;
1166 -
            }
1167 -
            if ft.is_symlink() {
1168 -
                let target = fs::read_link(path)?;
1169 -
                out.insert(
1170 -
                    rel,
1171 -
                    FsEntry::new(
1172 -
                        NodeKind::Symlink,
1173 -
                        false,
1174 -
                        target.to_string_lossy().as_bytes().to_vec(),
1175 -
                    ),
1176 -
                );
1177 -
            } else if ft.is_file() {
1178 -
                let meta = entry
1179 -
                    .metadata()
1180 -
                    .map_err(|e| RepoError::Msg(e.to_string()))?;
1181 -
                files.push((rel, path.to_path_buf(), meta));
1182 1220
            }
1183 1221
        }
1184 1222
        let keys = keys.filter(|k| {
…
448 unchanged lines
1633 1671
    key: Option<&str>,
1634 1672
) -> Result<FsEntry> {
1635 1673
    let exec = is_executable(meta);
1636 -
    Ok(match stat_cached(rel, meta, cache) {
1674 +
    let stat = Some((meta.len(), prompt::mtime_ns(meta)));
1675 +
    let mut e = match stat_cached(rel, meta, cache) {
1637 1676
        Some(e) if e.blob.is_some() => FsEntry {
1638 1677
            blob: e.blob,
1639 1678
            content_hash: Some(e.hash.clone()),
…
10 unchanged lines
1650 1689
            ..FsEntry::new(NodeKind::File, exec, fs::read(path)?)
1651 1690
        },
1652 1691
        None => FsEntry::new(NodeKind::File, exec, fs::read(path)?),
1653 -
    })
1692 +
    };
1693 +
    // The stat the manifest records: taken before the read, so a file that
1694 +
    // changes meanwhile is read again next time.
1695 +
    e.stat = stat;
1696 +
    Ok(e)
1654 1697
}
modified crates/oxy-core/src/repo/snapshot.rs +8 −1
…
287 unchanged lines
288 288
            let _t = crate::timing::phase("snapshot: load base history");
289 289
            self.load_state_history(&old_wc.deps)?
290 290
        };
291 +
        let keys_phase = crate::timing::phase("snapshot: tree view and graph keys");
291 292
        let view = base.tree().view();
292 293
        let keys: HashMap<String, String> = view
293 294
            .entries
…
4 unchanged lines
298 299
                Some((p.clone(), crate::hash::ObjectRef(k).to_hex()))
299 300
            })
300 301
            .collect();
301 -
        let manifest = self.read_manifest()?.map(|(_, _, m)| m);
302 +
        drop(keys_phase);
303 +
        let manifest = {
304 +
            let _t = crate::timing::phase("snapshot: read manifest");
305 +
            self.read_manifest()?.map(|(_, _, m)| m)
306 +
        };
302 307
        let tracked: BTreeMap<String, (NodeKind, Option<ObjectRef>)> = {
303 308
            let _t = crate::timing::phase("snapshot: tracked paths");
304 309
            let mut t = BTreeMap::new();
…
85 unchanged lines
390 395
        let contributors = built.contributors;
391 396
        let statuses = built.statuses;
392 397
        // Empty snapshot: hash the candidate before storing; only record an op on change.
398 +
        let record = crate::timing::phase("snapshot: hash and record");
393 399
        let new_hash = wc.hash();
394 400
        let changed = new_hash != old_hash;
395 401
        if changed {
…
2 unchanged lines
398 404
            view.workspaces.insert(self.workspace, new_hash);
399 405
            self.record_op(&[op_ref], &format!("{command}{SNAPSHOT_SUFFIX}"), view)?;
400 406
        }
407 +
        drop(record);
401 408
        // The prompt compares against this without recording anything.
402 409
        let _t = crate::timing::phase("snapshot: write manifest");
403 410
        self.write_manifest_entries(
…
1937 unchanged lines
modified docs/benchmarks.md +10 −0
…
343 unchanged lines
344 344
| `blame MAINTAINERS` | killed at 14 GB | 1.1 s, 315 MB | 0.31 s |
345 345
| `fsck` | killed at 14 GB | 123 s, 4.5 GB | to measure again |
346 346
347 +
### One file changed (2026-09-30, oxygen#35)
348 +
349 +
A status with one file changed (MAINTAINERS) took 4.1 s: the walk ran
350 +
on one thread (0.83 s), and the manifest was written by statting all
351 +
96 000 files again (0.57 s). The walk now runs in parallel and each
352 +
file's stat is carried from the walk into the manifest. It now takes
353 +
2.5 s: the base's history 0.5 s, its tree view and graph keys 0.38 s,
354 +
the walk 0.35 s, rendering the changed file 0.47 s, the diff 0.18 s,
355 +
and the manifest 0.14 s. (Git: 0.23 s.)
356 +