Edited files are diffed in parallel (refs #41) 360aa74a6277

The change builder diffs the edited text files against their base
renderings on every core before it updates them one by one, taking
the diffs it made. A 1 000-file commit on the Linux tree builds its
change in 72 ms (from 243); the commit takes 0.79-0.86 s (from 0.99,
Git 1.1-1.2 s). Graph loading reports how many graphs, changes and
blobs it read.

Author: Andy McCall @andymccall <[email protected]>
When: 1 month ago · 2026-10-02 02:24 UTC
Signature: ✔ signed by Andy McCall
Hash: 360aa74a6277f6f8fbf738776cf8e02e7dc741ef5576924f04b4fe861509aa0d
Depends on: 2306a4d24779 57d8dc05c9d4 75167672ebcd 982864068a59 a87c7b9d563a c3069537d7c5 cdb5beba7f05 fd280228bdc2 ff0af1ee299d
Depended on by: b35df36b7dc5
In bookmarks: agents-in-readme bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
In tags: v0.10.0 v0.11.0 v0.3.0 v0.3.1 v0.4.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
Edits: 9 atoms, 0 tree operations, 3 files

Browse the tree at this change

show whitespace changes

modified crates/oxy-core/src/repo/snapshot.rs +68 −6

modified crates/oxy-core/src/repo/tests.rs +45 −0

modified crates/oxy-core/src/state.rs +10 −0

		…	196 unchanged lines
197	197		.map(\|(old, new)\| (old.as_str(), new.as_str()))
198	198		.collect();
199	199		laps.lap("build: renames");
	200	+	b.diff_in_parallel(fs_entries);
	201	+	laps.lap("build: diffs");
200	202		for (path, fs_entry) in fs_entries {
201	203		match base_snapshot.files.get(path) {
202	204		None => match renamed_from.get(path.as_str()) {
		…	1448 unchanged lines
1651	1653		/// For a Git import, the base's dead directories by where they were
1652	1654		/// (see `ensure_dir`), worked out when first needed.
1653	1655		dead_dirs: Option<HashMap<(NodeId, String), (NodeId, crate::tree::OpKey)>>,
	1656	+	/// Text edits of edited files, worked out in parallel beforehand
	1657	+	/// (`diff_in_parallel`); a file not here is diffed when it is updated.
	1658	+	diffs: HashMap<String, Vec<Edit>>,
1654	1659		}
1655	1660
1656	1661		impl<'a> ChangeBuilder<'a> {
		…	24 unchanged lines
1681	1686		resolve: HashSet::new(),
1682	1687		node_seed: None,
1683	1688		dead_dirs: None,
	1689	+	diffs: HashMap::new(),
1684	1690		}
1685	1691		}
1686	1692
	1693	+	/// Diff the edited text files among `entries` against their base
	1694	+	/// renderings at once, on every core, for `update_path` to take: the
	1695	+	/// same diff it would make, only not one file after another.
	1696	+	fn diff_in_parallel(&mut self, entries: &BTreeMap<String, FsEntry>) {
	1697	+	use rayon::prelude::*;
	1698	+	let files = &self.base_snapshot.files;
	1699	+	let edited: Vec<(&String, &FsEntry, &crate::state::FileState)> = entries
	1700	+	.iter()
	1701	+	.filter_map(\|(p, e)\| {
	1702	+	let x = files.get(p)?;
	1703	+	(x.kind == e.kind && !e.unchanged && !self.opaque_for(p, e)).then_some((p, e, x))
	1704	+	})
	1705	+	.collect();
	1706	+	// Below this, starting the threads costs more than it saves.
	1707	+	if edited.len() < 16 {
	1708	+	return;
	1709	+	}
	1710	+	let this = &*self;
	1711	+	let attrs = &this.attrs;
	1712	+	let diffs: Vec<(String, Vec<Edit>)> = edited
	1713	+	.par_iter()
	1714	+	.filter_map(\|(p, e, x)\| {
	1715	+	let FileContent::Text {
	1716	+	bytes,
	1717	+	materialized,
	1718	+	} = &x.content
	1719	+	else {
	1720	+	return None;
	1721	+	};
	1722	+	if materialized.is_conflicted() {
	1723	+	return None;
	1724	+	}
	1725	+	let disk_text = if attrs.lookup(p).eol.is_some() {
	1726	+	std::sync::Arc::new(crate::attributes::normalize_eol(&e.content))
	1727	+	} else {
	1728	+	e.content.clone()
	1729	+	};
	1730	+	if bytes == disk_text {
	1731	+	return None;
	1732	+	}
	1733	+	let keys = materialized.live_atoms();
	1734	+	let lines = crate::diff::split_lines(bytes);
	1735	+	(keys.len() == lines.len()).then(\|\| {
	1736	+	let edits = crate::diff::text_edits_of(x.node, &keys, &lines, &disk_text);
	1737	+	((*p).clone(), edits)
	1738	+	})
	1739	+	})
	1740	+	.collect();
	1741	+	self.diffs = diffs.into_iter().collect();
	1742	+	}
	1743	+
1687	1744		fn with_attributes(mut self, attrs: crate::attributes::Attributes) -> Self {
1688	1745		self.attrs = attrs;
1689	1746		self
		…	421 unchanged lines
2111	2168		let mut laps = crate::timing::Laps::start();
2112	2169		// The base's rendering holds its live atoms in order,
2113	2170		// one per line (it is not conflicted here).
2114		-	let keys = materialized.live_atoms();
2115		-	let lines = crate::diff::split_lines(bytes);
2116		-	let edits = if keys.len() == lines.len() {
2117		-	crate::diff::text_edits_of(node, &keys, &lines, &disk_text)
2118		-	} else {
2119		-	text_edits(node, &self.base.live_lines(&node), &disk_text)
	2171	+	let edits = match self.diffs.remove(path) {
	2172	+	Some(edits) => edits,
	2173	+	None => {
	2174	+	let keys = materialized.live_atoms();
	2175	+	let lines = crate::diff::split_lines(bytes);
	2176	+	if keys.len() == lines.len() {
	2177	+	crate::diff::text_edits_of(node, &keys, &lines, &disk_text)
	2178	+	} else {
	2179	+	text_edits(node, &self.base.live_lines(&node), &disk_text)
	2180	+	}
	2181	+	}
2120	2182		};
2121	2183		laps.lap("build: text diff");
2122	2184		if !edits.is_empty() {
		…	473 unchanged lines

		…	6452 unchanged lines
6453	6453		assert_eq!(after[1], before[1]);
6454	6454		assert_eq!(after[0], b"1\ntwo\n3\n4\n5\n");
6455	6455		}
	6456	+
	6457	+	#[test]
	6458	+	fn many_edited_files_diff_in_parallel() {
	6459	+	// Enough edited files that the builder diffs them in parallel; the
	6460	+	// commit must hold exactly what is on disk, and a second snapshot
	6461	+	// must find nothing left to commit.
	6462	+	hermetic();
	6463	+	let dir = tempfile::tempdir().unwrap();
	6464	+	let root = dir.path();
	6465	+	let repo = Repo::init(root).unwrap();
	6466	+	let text = \|i: usize, round: usize\| {
	6467	+	let mut s = String::new();
	6468	+	for line in 0..50 {
	6469	+	if round > 0 && line % 7 == i % 7 {
	6470	+	s.push_str(&format!("edited {round} {i} {line}\n"));
	6471	+	} else {
	6472	+	s.push_str(&format!("file {i} line {line}\n"));
	6473	+	}
	6474	+	}
	6475	+	if round > 0 && i.is_multiple_of(3) {
	6476	+	s.push_str("appended\n");
	6477	+	}
	6478	+	s
	6479	+	};
	6480	+	for i in 0..40 {
	6481	+	write(root, &format!("d/f{i}.txt"), text(i, 0).as_bytes());
	6482	+	}
	6483	+	repo.commit("first", &[]).unwrap();
	6484	+	for i in 0..40 {
	6485	+	write(root, &format!("d/f{i}.txt"), text(i, 1).as_bytes());
	6486	+	}
	6487	+	let h = repo.commit("edit every file", &[]).unwrap();
	6488	+	let snap = repo.load_state(&[h]).unwrap().materialize();
	6489	+	for i in 0..40 {
	6490	+	let want = text(i, 1);
	6491	+	assert!(
	6492	+	matches!(&snap.files[&format!("d/f{i}.txt")].content, FileContent::Text { bytes, .. } if bytes == want.as_bytes()),
	6493	+	"d/f{i}.txt as committed"
	6494	+	);
	6495	+	}
	6496	+	assert!(
	6497	+	repo.commit("nothing", &[]).is_err(),
	6498	+	"nothing left to commit"
	6499	+	);
	6500	+	}

		…	673 unchanged lines
674	674		}
675	675		}
676	676		}
	677	+	let n_changes = changes.len();
	678	+	drop(reading);
	679	+	let reading = crate::timing::phase("graphs: of which blobs");
677	680		let mut blobs = if refs.is_empty() {
678	681		Vec::new()
679	682		} else {
680	683		load(&refs)?
681	684		};
	685	+	if crate::timing::enabled() {
	686	+	eprintln!(
	687	+	"timing graphs: {} graphs, {n_changes} changes, {} blobs",
	688	+	taken.len(),
	689	+	refs.len()
	690	+	);
	691	+	}
682	692		if blobs.len() != refs.len() {
683	693		return Err(ObjectError::Schema("graph blobs: wrong count".into()));
684	694		}
		…	1831 unchanged lines

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

Edited files are diffed in parallel (refs #41) 360aa74a6277

Keyboard