crates/oxy-core/src/repo/git.rs at v0.4.1 · andymccall/oxygen

◆ v0.4.1 oxygen / crates / oxy-core / src / repo / git.rs

89.6 KiB · Rust · last changed Release 0.4.1 27 days ago source blame history raw

79ab8528 Andy McCall 1 month ago ↶	1	//! Git bridge (spec §23), export side: each state the current bookmark
	2	//! passed through becomes a Git commit through `git fast-import`, with the
	3	//! state's full tree and the previous state's commit as parent. A state
	4	//! that added exactly one change carries that change's message, author,
	5	//! and `Change-Id`/`Oxy-Hash` trailers; every commit carries `Oxy-State`.
	6	//!
	7	//! Marks make re-exports incremental: `.oxy/git/<target>/git-marks` is
	8	//! git's own marks file and `oxy-marks` maps state refs to marks.
	9
	10	use super::*;
	11	use std::process::{Command, Stdio};
62794ecc Andy McCall 1 month ago ↶	12	use std::sync::Arc;
79ab8528 Andy McCall 1 month ago ↶	13
79ab8528 Andy McCall 1 month ago ↶	14	#[derive(Debug, Clone, Default, PartialEq, Eq)]
c3069537 Andy McCall 1 month ago ↶	15	pub struct GitImportReport {
	16	pub commits: usize,
	17	pub merges: usize,
4dee277c Andy McCall 1 month ago ↶	18	/// Gitlink entries imported as subrepo nodes with Git pins.
4dee277c Andy McCall 1 month ago ↶	19	pub submodules: usize,
c3069537 Andy McCall 1 month ago ↶	20	pub heads: usize,
	21	}
	22
	23	/// A running `git cat-file --batch`, for reading many blobs cheaply.
	24	struct CatFile {
	25	child: std::process::Child,
	26	stdin: std::process::ChildStdin,
	27	stdout: std::io::BufReader<std::process::ChildStdout>,
	28	}
	29
	30	impl CatFile {
	31	fn start(git_dir: &Path) -> Result<CatFile> {
	32	let mut child = Command::new("git")
	33	.arg("-C")
	34	.arg(git_dir)
	35	.args(["cat-file", "--batch"])
	36	.stdin(Stdio::piped())
	37	.stdout(Stdio::piped())
	38	.stderr(Stdio::null())
	39	.spawn()
	40	.map_err(\|e\| RepoError::Msg(format!("running git cat-file: {e}")))?;
	41	let stdin = child.stdin.take().unwrap();
	42	let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
75167672 Andy McCall 1 month ago	43	Ok(CatFile {
	44	child,
	45	stdin,
	46	stdout,
	47	})
c3069537 Andy McCall 1 month ago ↶	48	}
	49
	50	fn read(&mut self, sha: &str) -> Result<Vec<u8>> {
	51	use std::io::{BufRead, Read};
	52	writeln!(self.stdin, "{sha}")?;
	53	self.stdin.flush()?;
	54	let mut header = String::new();
	55	self.stdout.read_line(&mut header)?;
	56	let parts: Vec<&str> = header.trim_end().split(' ').collect();
	57	if parts.len() != 3 {
	58	return fail(format!("git cat-file: object {sha} not found"));
	59	}
75167672 Andy McCall 1 month ago	60	let size: usize = parts[2]
	61	.parse()
	62	.map_err(\|_\| RepoError::Msg("git cat-file: bad size".into()))?;
c3069537 Andy McCall 1 month ago ↶	63	let mut body = vec![0u8; size];
	64	self.stdout.read_exact(&mut body)?;
	65	let mut nl = [0u8; 1];
	66	self.stdout.read_exact(&mut nl)?;
	67	Ok(body)
	68	}
	69	}
	70
	71	impl Drop for CatFile {
	72	fn drop(&mut self) {
	73	let _ = self.child.kill();
	74	let _ = self.child.wait();
	75	}
	76	}
	77
2306a4d2 Andy McCall 1 month ago ↶	78	/// One path a commit changes against its first parent: its new mode and
	79	/// blob, or `None` when the commit deletes it.
	80	type TreeDiff = Vec<(String, Option<(String, String)>)>;
	81
	82	/// A running `git diff-tree --stdin`, fed every `commit parent` pair up
	83	/// front, whose diffs are read in the same order as they are needed: a
	84	/// commit's tree is its first parent's plus its diff, at a cost that
	85	/// follows what it changes rather than the whole tree (oxygen#25).
	86	struct DiffTree {
	87	child: std::process::Child,
	88	stdout: std::io::BufReader<std::process::ChildStdout>,
	89	feeder: Option<std::thread::JoinHandle<()>>,
	90	/// The header read at the end of the previous diff.
	91	next: Option<String>,
	92	}
	93
	94	impl DiffTree {
	95	fn start(git_dir: &Path, pairs: Vec<(String, String)>) -> Result<DiffTree> {
	96	let mut child = Command::new("git")
	97	.arg("-C")
	98	.arg(git_dir)
	99	.args([
	100	"diff-tree",
	101	"--stdin",
	102	"--always",
	103	"-r",
	104	"-z",
	105	"--no-renames",
	106	])
	107	.stdin(Stdio::piped())
	108	.stdout(Stdio::piped())
	109	.stderr(Stdio::null())
	110	.spawn()
	111	.map_err(\|e\| RepoError::Msg(format!("running git diff-tree: {e}")))?;
	112	let mut stdin = child.stdin.take().unwrap();
	113	let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
	114	// A thread writes, so git never waits on a full pipe for us to read.
	115	let feeder = std::thread::spawn(move \|\| {
f0406c3b Andy McCall 27 days ago ↶	116	// An import that stops before reading every diff kills git
	117	// while this thread may still be writing. oxy restores
	118	// SIGPIPE's default (to end quietly under `\| head`), so that
	119	// write would kill the whole process (signal 13, oxygen#81);
	120	// blocked here, it fails with EPIPE and the thread ends.
	121	#[cfg(unix)]
	122	unsafe {
	123	let mut set: libc::sigset_t = std::mem::zeroed();
	124	libc::sigemptyset(&mut set);
	125	libc::sigaddset(&mut set, libc::SIGPIPE);
	126	libc::pthread_sigmask(libc::SIG_BLOCK, &set, std::ptr::null_mut());
	127	}
2306a4d2 Andy McCall 1 month ago ↶	128	let mut w = std::io::BufWriter::new(&mut stdin);
	129	for (c, p) in pairs {
	130	if writeln!(w, "{c} {p}").is_err() {
	131	return;
	132	}
	133	}
	134	let _ = w.flush();
	135	});
	136	Ok(DiffTree {
	137	child,
	138	stdout,
	139	feeder: Some(feeder),
	140	next: None,
	141	})
	142	}
	143
	144	fn field(&mut self) -> Result<Option<String>> {
	145	use std::io::BufRead;
	146	let mut buf = Vec::new();
	147	if self.stdout.read_until(0, &mut buf)? == 0 {
	148	return Ok(None);
	149	}
	150	if buf.last() == Some(&0) {
	151	buf.pop();
	152	}
	153	Ok(Some(String::from_utf8_lossy(&buf).into_owned()))
	154	}
	155
	156	/// The diff of `sha` against the parent it was fed with.
	157	fn diff(&mut self, sha: &str) -> Result<TreeDiff> {
	158	let header = match self.next.take() {
	159	Some(h) => Some(h),
	160	None => self.field()?,
	161	};
	162	if header.as_deref() != Some(sha) {
	163	return fail(format!("git diff-tree: expected {sha}, got {header:?}"));
	164	}
	165	let mut out = Vec::new();
	166	while let Some(f) = self.field()? {
	167	let Some(meta) = f.strip_prefix(':') else {
	168	self.next = Some(f);
	169	break;
	170	};
	171	let Some(path) = self.field()? else {
	172	return fail("git diff-tree: record without a path");
	173	};
	174	// `oldmode newmode oldsha newsha status`
	175	let m: Vec<&str> = meta.split(' ').collect();
	176	if m.len() != 5 {
	177	return fail(format!("git diff-tree: bad record {f:?}"));
	178	}
	179	let entry = if m[4].starts_with('D') {
	180	None
	181	} else {
	182	Some((m[1].to_string(), m[3].to_string()))
	183	};
	184	out.push((path, entry));
	185	}
	186	Ok(out)
	187	}
	188	}
	189
	190	impl Drop for DiffTree {
	191	fn drop(&mut self) {
192	let _ = self.child.kill();
193	let _ = self.child.wait();
194	if let Some(f) = self.feeder.take() {
195	let _ = f.join();
196	}
197	}
198	}
199
c3069537 Andy McCall 1 month ago ↶	200	/// One commit as `git log` describes it.
	201	struct GitCommit {
	202	sha: String,
	203	parents: Vec<String>,
	204	author_secs: u64,
	205	author: String,
	206	message: String,
	207	}
	208
	209	fn list_commits(git_dir: &Path, branch: &str) -> Result<Vec<GitCommit>> {
	210	let out = Command::new("git")
	211	.arg("-C")
	212	.arg(git_dir)
75167672 Andy McCall 1 month ago	213	.args([
	214	"log",
	215	"--reverse",
	216	"--topo-order",
	217	"--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e",
	218	branch,
	219	])
c3069537 Andy McCall 1 month ago ↶	220	.output()
	221	.map_err(\|e\| RepoError::Msg(format!("running git log: {e}")))?;
	222	if !out.status.success() {
75167672 Andy McCall 1 month ago	223	return fail(format!(
	224	"git log {branch} failed: {}",
	225	String::from_utf8_lossy(&out.stderr).trim()
	226	));
c3069537 Andy McCall 1 month ago ↶	227	}
	228	let text = String::from_utf8_lossy(&out.stdout);
	229	let mut commits = Vec::new();
	230	for record in text.split('\x1e') {
	231	let record = record.trim_start_matches('\n');
	232	if record.trim().is_empty() {
	233	continue;
	234	}
	235	let f: Vec<&str> = record.splitn(5, '\0').collect();
	236	if f.len() != 5 {
	237	return fail("unexpected git log record");
	238	}
	239	commits.push(GitCommit {
	240	sha: f[0].to_string(),
	241	parents: f[1].split_whitespace().map(str::to_string).collect(),
	242	author_secs: f[2].parse().unwrap_or(0),
	243	author: f[3].to_string(),
	244	message: f[4].trim_end().to_string(),
	245	});
	246	}
	247	Ok(commits)
	248	}
	249
	250	/// `path → (mode, blob sha)` for a commit's tree.
	251	fn list_tree(git_dir: &Path, sha: &str) -> Result<BTreeMap<String, (String, String)>> {
	252	let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?;
	253	let mut out = BTreeMap::new();
	254	for entry in text.split('\0') {
	255	if entry.is_empty() {
	256	continue;
	257	}
75167672 Andy McCall 1 month ago	258	let Some((meta, path)) = entry.split_once('\t') else {
	259	continue;
	260	};
c3069537 Andy McCall 1 month ago ↶	261	let m: Vec<&str> = meta.split(' ').collect();
	262	if m.len() == 3 {
	263	out.insert(path.to_string(), (m[0].to_string(), m[2].to_string()));
	264	}
	265	}
	266	Ok(out)
	267	}
	268
	269	/// The ChangeId for a Git commit: its `Change-Id` trailer if it has a
	270	/// 32-hex one, else BLAKE3-128 of the commit hash (spec §23).
	271	fn change_id_for(sha: &str, message: &str) -> ChangeId {
	272	for line in message.lines().rev() {
	273	if let Some(v) = line.strip_prefix("Change-Id:") {
	274	if let Some(id) = ChangeId::from_hex(v.trim()) {
	275	return id;
	276	}
	277	}
	278	}
	279	let h = blake3::hash(sha.as_bytes());
	280	ChangeId(h.as_bytes()[..16].try_into().unwrap())
	281	}
	282
30ea9c1e Andy McCall 1 month ago ↶	283	/// Git commits with a known Oxygen counterpart.
	284	struct GitKnown {
	285	/// Imported commit → its change.
	286	imported: HashMap<String, ChangeHash>,
	287	/// Exported commit (or a fetched tip) → the bookmark state it equals.
	288	exported: HashMap<String, ObjectRef>,
	289	}
	290
c3069537 Andy McCall 1 month ago ↶	291	#[derive(Debug, Clone, Default, PartialEq, Eq)]
79ab8528 Andy McCall 1 month ago ↶	292	pub struct GitExportReport {
	293	pub exported: usize,
	294	pub skipped: usize,
4dee277c Andy McCall 1 month ago ↶	295	/// Subrepo nodes with Oxygen pins, which Git cannot represent.
4dee277c Andy McCall 1 month ago ↶	296	pub skipped_subrepos: usize,
79ab8528 Andy McCall 1 month ago ↶	297	pub branch: String,
	298	pub heads: usize,
	299	}
	300
fd280228 Andy McCall 1 month ago ↶	301	/// A merge bringing more commits than this sheds its base's graphs first.
fd280228 Andy McCall 1 month ago ↶	302	const MERGE_SHED_COMMITS: usize = 100;
d45dab04 Andy McCall 1 month ago ↶	303
	304	/// The atoms a kept state's built graphs may hold before they go back into
	305	/// its history (#25): about a few hundred megabytes.
	306	const KEPT_ATOMS: usize = 1_500_000;
fd280228 Andy McCall 1 month ago ↶	307	/// A kept state with more built graphs than this sheds them.
fd280228 Andy McCall 1 month ago ↶	308	const KEPT_GRAPHS: usize = 2000;
9935d490 Andy McCall 29 days ago ↶	309	/// An import moves its bookmark to the commits done so far every this
	310	/// many commits, so one that stops part way resumes from there: imported
	311	/// commits are found from the bookmark's history (`imported_commits`).
	312	const CHECKPOINT_COMMITS: usize = 2000;
fd280228 Andy McCall 1 month ago ↶	313
c3d2ef61 Andy McCall 1 month ago ↶	314	/// Text files of a merge rendered this many at a time (oxygen#25): a
	315	/// batch's graphs are built beside the state and dropped once rendered,
	316	/// so what is held for the whole tree is the renderings, not the graphs
	317	/// with every deleted line of their history.
	318	const MERGE_BATCH: usize = 500;
	319
	320	thread_local! {
	321	/// A smaller batch for this thread's imports: tests set 1, so a small
	322	/// merge takes the batched path.
	323	static MERGE_BATCH_OVERRIDE: std::cell::Cell<Option<usize>> = const { std::cell::Cell::new(None) };
	324	}
	325
	326	#[cfg(test)]
	327	pub(crate) fn set_merge_batch_for_tests(n: Option<usize>) {
	328	MERGE_BATCH_OVERRIDE.with(\|c\| c.set(n));
	329	}
	330
	331	fn merge_batch() -> usize {
	332	MERGE_BATCH_OVERRIDE
	333	.with(\|c\| c.get())
	334	.filter(\|n\| *n > 0)
33be2409 Andy McCall 1 month ago ↶	335	.or_else(\|\| {
	336	std::env::var("OXY_MERGE_BATCH")
	337	.ok()
	338	.and_then(\|v\| v.parse().ok())
	339	})
c3d2ef61 Andy McCall 1 month ago ↶	340	.unwrap_or(MERGE_BATCH)
	341	}
	342
	343	impl Repo {
	344	/// As `State::materialize_except`, for a tree whose graphs would not
	345	/// fit in memory at once. The few files the change builder reads
	346	/// graphs for (conflicted ones, and renderings whose lines are not
	347	/// one per live atom) are built in the state afterwards.
	348	fn materialize_batched(
	349	&self,
	350	base: &mut State,
	351	need: &[crate::ids::NodeId],
	352	skip: &HashSet<crate::ids::NodeId>,
	353	) -> Result<crate::state::Snapshot> {
	354	use crate::graph::Materialized;
	355	use crate::state::{FileContent, FileState};
	356	use rayon::prelude::*;
	357	let view = base.tree().view();
	358	let mut rendered: HashMap<crate::ids::NodeId, (Vec<u8>, Materialized)> =
	359	HashMap::with_capacity(need.len());
	360	let mut keep_built: Vec<crate::ids::NodeId> = Vec::new();
	361	for batch in need.chunks(merge_batch()) {
	362	let t = std::time::Instant::now();
	363	let graphs = self.build_graphs_detached(base, batch)?;
	364	crate::timing::add("render: build graphs, batch", t.elapsed());
	365	let t = std::time::Instant::now();
	366	let batch_rendered: Vec<(crate::ids::NodeId, Vec<u8>, Materialized)> = graphs
	367	.par_iter()
	368	.map(\|(node, g)\| {
	369	let (bytes, m) = base.render_graph(g);
	370	(*node, bytes, m)
	371	})
	372	.collect();
	373	drop(graphs);
	374	crate::timing::add("render: render, batch", t.elapsed());
	375	for (node, bytes, m) in batch_rendered {
	376	if m.is_conflicted()
	377	\|\| m.live_atoms().len() != crate::diff::split_lines(&bytes).len()
	378	{
	379	keep_built.push(node);
	380	}
	381	rendered.insert(node, (bytes, m));
	382	}
	383	}
	384	if !keep_built.is_empty() {
	385	self.load_graphs_of(base, &keep_built)?;
	386	}
	387	let mut files = BTreeMap::new();
	388	let mut unrendered = BTreeSet::new();
	389	for (path, e) in view.entries.iter() {
	390	let f = if e.kind == NodeKind::File && !e.mode.opaque {
	391	match rendered.remove(&e.node) {
	392	Some((bytes, materialized)) => FileState {
	393	node: e.node,
	394	kind: e.kind,
	395	mode: e.mode,
	396	content: FileContent::Text {
	397	bytes,
	398	materialized,
	399	},
	400	},
	401	None if skip.contains(&e.node) => {
	402	unrendered.insert(path.clone());
	403	FileState {
404	node: e.node,
405	kind: e.kind,
406	mode: e.mode,
407	content: FileContent::Text {
408	bytes: Vec::new(),
409	materialized: Materialized::default(),
410	},
411	}
412	}
413	None => {
414	// Neither asked for nor known unchanged: render it
415	// from a built graph, as the whole-tree path does.
416	self.load_graphs_of(base, &[e.node])?;
417	base.entry_state(e)
418	}
419	}
420	} else {
421	base.entry_state(e)
422	};
423	files.insert(path.clone(), f);
424	}
425	Ok(crate::state::Snapshot {
426	tree: view,
427	files,
428	unrendered,
429	})
430	}
431	}
432
79ab8528 Andy McCall 1 month ago ↶	433	fn git(dir: &Path, args: &[&str]) -> Result<String> {
75167672 Andy McCall 1 month ago	434	let out = Command::new("git")
	435	.arg("-C")
	436	.arg(dir)
	437	.args(args)
	438	.output()
	439	.map_err(\|e\| RepoError::Msg(format!("running git: {e}")))?;
79ab8528 Andy McCall 1 month ago ↶	440	if !out.status.success() {
75167672 Andy McCall 1 month ago	441	return fail(format!(
	442	"git {} failed: {}",
	443	args.join(" "),
	444	String::from_utf8_lossy(&out.stderr).trim()
	445	));
79ab8528 Andy McCall 1 month ago ↶	446	}
	447	Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
	448	}
	449
	450	impl Repo {
	451	fn git_marks_dir(&self, git_dir: &Path) -> Result<PathBuf> {
75167672 Andy McCall 1 month ago	452	let canonical = git_dir
	453	.canonicalize()
	454	.unwrap_or_else(\|_\| git_dir.to_path_buf());
79ab8528 Andy McCall 1 month ago ↶	455	let key = hash_object(ObjectType::Chunk, canonical.to_string_lossy().as_bytes()).short(16);
	456	let dir = self.oxy.join("git").join(key);
	457	fs::create_dir_all(&dir)?;
	458	Ok(dir)
	459	}
	460
30ea9c1e Andy McCall 1 month ago ↶	461	/// `state ref → Git commit sha` for bookmark states that equal a Git
	462	/// commit's tree (recorded by import and fetch).
	463	fn read_git_states(path: &Path) -> Result<BTreeMap<ObjectRef, String>> {
	464	let mut out = BTreeMap::new();
	465	if let Ok(text) = fs::read_to_string(path) {
	466	for line in text.lines() {
	467	if let Some((h, sha)) = line.split_once(' ') {
	468	if let Some(h) = ObjectRef::from_hex(h) {
	469	out.insert(h, sha.trim().to_string());
	470	}
	471	}
	472	}
	473	}
	474	Ok(out)
	475	}
	476
	477	fn record_git_state(&self, git_dir: &Path, state: &ObjectRef, sha: &str) -> Result<()> {
	478	let path = self.git_marks_dir(git_dir)?.join("git-states");
	479	let mut states = Self::read_git_states(&path)?;
	480	states.insert(*state, sha.to_string());
75167672 Andy McCall 1 month ago	481	let text: String = states
	482	.iter()
	483	.map(\|(h, s)\| format!("{} {s}\n", h.to_hex()))
	484	.collect();
30ea9c1e Andy McCall 1 month ago ↶	485	write_atomic(&path, text.as_bytes())
	486	}
	487
	488	/// The bare mirror this repository keeps for the Git remote `url`.
	489	fn git_mirror(&self, url: &str) -> Result<PathBuf> {
	490	let key = hash_object(ObjectType::Chunk, url.as_bytes()).short(16);
	491	let dir = self.oxy.join("git").join(key).join("mirror.git");
	492	if !dir.join("HEAD").exists() {
	493	fs::create_dir_all(&dir)?;
	494	git(&dir, &["init", "--bare", "-q"])?;
	495	}
	496	Ok(dir)
	497	}
	498
	499	/// Git commits already imported into the bookmark, by sha.
7891e21c Andy McCall 1 month ago ↶	500	pub(crate) fn imported_commits(&self) -> Result<HashMap<String, ChangeHash>> {
30ea9c1e Andy McCall 1 month ago ↶	501	let mut map = HashMap::new();
	502	for e in self.log()? {
	503	if e.is_working_copy {
	504	continue;
	505	}
75167672 Andy McCall 1 month ago	506	if let Some(sha) = e
	507	.change
	508	.message
	509	.lines()
	510	.rev()
	511	.find_map(\|l\| l.strip_prefix("Git-Commit: "))
	512	{
30ea9c1e Andy McCall 1 month ago ↶	513	map.insert(sha.trim().to_string(), e.hash);
	514	}
	515	}
	516	Ok(map)
	517	}
	518
	519	/// Bring the mirror's `branch` up to date with the Git remote `url`.
	520	/// A remote without that branch yet leaves the mirror as it is.
	521	fn git_mirror_fetch(&self, mirror: &Path, url: &str, branch: &str) -> Result<()> {
	522	let spec = format!("+refs/heads/{branch}:refs/heads/{branch}");
	523	match git(mirror, &["fetch", "-q", url, &spec]) {
	524	Ok(_) => Ok(()),
	525	Err(e) => {
	526	let ls = git(mirror, &["ls-remote", "--heads", url, branch]).unwrap_or_default();
75167672 Andy McCall 1 month ago	527	if ls.trim().is_empty() {
	528	Ok(())
	529	} else {
	530	Err(e)
	531	}
30ea9c1e Andy McCall 1 month ago ↶	532	}
	533	}
	534	}
	535
	536	/// Fetch `branch` of the Git repository at `url` (a path or a Git URL)
	537	/// through a mirror and import the commits not yet in the bookmark
	538	/// (spec §23). Works on a repository that already has history: the
	539	/// new changes join the bookmark like a pull.
	540	pub fn git_fetch(&self, url: &str, branch: &str) -> Result<GitImportReport> {
9935d490 Andy McCall 29 days ago ↶	541	// A local path, made absolute: git runs it from inside the mirror.
	542	let local = Path::new(url)
	543	.canonicalize()
	544	.ok()
	545	.filter(\|_\| !url.contains("://"));
	546	let absolute = local.as_ref().map(\|p\| p.to_string_lossy().into_owned());
	547	let url = absolute.as_deref().unwrap_or(url);
30ea9c1e Andy McCall 1 month ago ↶	548	let mirror = self.git_mirror(url)?;
30ea9c1e Andy McCall 1 month ago ↶	549	self.git_mirror_fetch(&mirror, url, branch)?;
75167672 Andy McCall 1 month ago	550	if git(
	551	&mirror,
	552	&[
	553	"rev-parse",
	554	"--verify",
	555	"-q",
	556	&format!("refs/heads/{branch}"),
	557	],
	558	)
	559	.is_err()
	560	{
30ea9c1e Andy McCall 1 month ago ↶	561	return fail(format!("{url} has no branch {branch:?}"));
30ea9c1e Andy McCall 1 month ago ↶	562	}
58dfe0e1 Andy McCall 1 month ago ↶	563	self.import_git_commits(
	564	&mirror,
	565	branch,
	566	&format!("oxy git fetch {url} {branch}"),
	567	&mut \|_\| {},
	568	)
30ea9c1e Andy McCall 1 month ago ↶	569	}
	570
	571	/// Export the bookmark to `branch` of the Git repository at `url`
	572	/// through the mirror and push it. The remote's tip must already have
	573	/// been fetched (`git_fetch`), so the push fast-forwards.
	574	pub fn git_push(&self, url: &str, branch: &str) -> Result<GitExportReport> {
	575	let mirror = self.git_mirror(url)?;
	576	self.git_mirror_fetch(&mirror, url, branch)?;
75167672 Andy McCall 1 month ago	577	if let Ok(tip) = git(
	578	&mirror,
	579	&[
	580	"rev-parse",
	581	"--verify",
	582	"-q",
	583	&format!("refs/heads/{branch}"),
	584	],
	585	) {
30ea9c1e Andy McCall 1 month ago ↶	586	let known = self.git_known(&mirror)?;
30ea9c1e Andy McCall 1 month ago ↶	587	if !known.imported.contains_key(&tip) && !known.exported.contains_key(&tip) {
75167672 Andy McCall 1 month ago	588	return fail(format!(
	589	"{url} {branch} has commits not yet fetched; run `oxy git fetch {url}` first"
	590	));
30ea9c1e Andy McCall 1 month ago ↶	591	}
	592	}
	593	let report = self.git_export_as(&mirror, branch)?;
	594	let spec = format!("refs/heads/{branch}:refs/heads/{branch}");
	595	git(&mirror, &["push", "-q", url, &spec])?;
	596	Ok(report)
	597	}
	598
79ab8528 Andy McCall 1 month ago ↶	599	fn read_oxy_marks(path: &Path) -> Result<BTreeMap<ObjectRef, u64>> {
	600	let mut out = BTreeMap::new();
	601	if let Ok(text) = fs::read_to_string(path) {
	602	for line in text.lines() {
	603	if let Some((h, m)) = line.split_once(' ') {
	604	if let (Some(h), Ok(m)) = (ObjectRef::from_hex(h), m.parse()) {
	605	out.insert(h, m);
	606	}
	607	}
	608	}
	609	}
	610	Ok(out)
	611	}
	612
	613	/// Export the current bookmark to the Git repository at `git_dir`,
	614	/// creating a bare one if nothing is there. Each state the bookmark
	615	/// passed through (per the operation log) becomes one commit whose tree
	616	/// is that state, so the Git branch is a linear history of full trees.
	617	pub fn git_export(&self, git_dir: &Path) -> Result<GitExportReport> {
30ea9c1e Andy McCall 1 month ago ↶	618	let bookmark = self.config.bookmark.clone();
	619	self.git_export_as(git_dir, &bookmark)
	620	}
	621
	622	/// `git_export` to the Git branch `branch`. A state that came from a
	623	/// Git commit (`oxy git fetch`) is not re-exported: later states chain
	624	/// onto that commit, so a push back to the Git remote fast-forwards.
	625	pub fn git_export_as(&self, git_dir: &Path, branch: &str) -> Result<GitExportReport> {
79ab8528 Andy McCall 1 month ago ↶	626	if !git_dir.join("HEAD").exists() && !git_dir.join(".git").exists() {
	627	fs::create_dir_all(git_dir)?;
	628	git(git_dir, &["init", "--bare", "-q"])?;
	629	}
	630	let marks_dir = self.git_marks_dir(git_dir)?;
	631	let git_marks = marks_dir.join("git-marks");
	632	let oxy_marks_path = marks_dir.join("oxy-marks");
	633	let mut marks = Self::read_oxy_marks(&oxy_marks_path)?;
30ea9c1e Andy McCall 1 month ago ↶	634	let git_states = Self::read_git_states(&marks_dir.join("git-states"))?;
79ab8528 Andy McCall 1 month ago ↶	635	let mut next_mark = marks.values().copied().max().unwrap_or(0) + 1;
	636
	637	let bookmark = self.config.bookmark.clone();
	638	let mut stream: Vec<u8> = Vec::new();
75167672 Andy McCall 1 month ago	639	let mut report = GitExportReport {
	640	branch: branch.to_string(),
	641	heads: self.bookmark_heads()?.len(),
	642	..Default::default()
	643	};
79ab8528 Andy McCall 1 month ago ↶	644
	645	// States along the bookmark, oldest first, skipping unchanged and pruned ones.
	646	let mut ops = self.op_log()?;
	647	ops.reverse();
	648	let mut prev: Option<(ObjectRef, Vec<ChangeHash>)> = None;
30ea9c1e Andy McCall 1 month ago ↶	649	// The fast-import parent of the next commit: `:mark` or a commit sha.
30ea9c1e Andy McCall 1 month ago ↶	650	let mut last_parent: Option<String> = None;
79ab8528 Andy McCall 1 month ago ↶	651	for (_, op) in &ops {
75167672 Andy McCall 1 month ago	652	let Some(state_ref) = op.view.bookmarks.get(&bookmark).copied() else {
	653	continue;
	654	};
79ab8528 Andy McCall 1 month ago ↶	655	if prev.as_ref().is_some_and(\|(p, _)\| *p == state_ref) {
	656	continue;
	657	}
75167672 Andy McCall 1 month ago	658	let Ok(state) = self.store.get_state(&state_ref) else {
	659	continue;
	660	};
79ab8528 Andy McCall 1 month ago ↶	661	if state.heads.is_empty() {
	662	prev = Some((state_ref, vec![]));
	663	continue;
	664	}
30ea9c1e Andy McCall 1 month ago ↶	665	if let Some(sha) = git_states.get(&state_ref) {
	666	report.skipped += 1;
	667	last_parent = Some(sha.clone());
	668	prev = Some((state_ref, state.heads.clone()));
	669	continue;
	670	}
79ab8528 Andy McCall 1 month ago ↶	671	if let Some(m) = marks.get(&state_ref) {
79ab8528 Andy McCall 1 month ago ↶	672	report.skipped += 1;
30ea9c1e Andy McCall 1 month ago ↶	673	last_parent = Some(format!(":{m}"));
79ab8528 Andy McCall 1 month ago ↶	674	prev = Some((state_ref, state.heads.clone()));
	675	continue;
	676	}
	677	let prev_heads = prev.as_ref().map(\|(_, h)\| h.clone()).unwrap_or_default();
75167672 Andy McCall 1 month ago	678	let added = self
	679	.negotiate(&prev_heads, &state.heads)
	680	.unwrap_or_default();
79ab8528 Andy McCall 1 month ago ↶	681	let loaded = self.load_state(&state.heads)?;
	682	let snapshot = loaded.materialize();
	683	let (author, secs, mut message) = match added.as_slice() {
	684	[h] => {
	685	let c = self.store.get_change(h)?;
	686	let mut m = c.message.clone();
	687	if !m.ends_with('\n') {
	688	m.push('\n');
	689	}
75167672 Andy McCall 1 month ago	690	m.push_str(&format!(
	691	"\nChange-Id: {}\nOxy-Hash: {}\n",
	692	c.change_id.to_hex(),
	693	h.to_hex()
	694	));
79ab8528 Andy McCall 1 month ago ↶	695	(self.git_author(&c), c.hlc.wall_ms / 1000, m)
79ab8528 Andy McCall 1 month ago ↶	696	}
75167672 Andy McCall 1 month ago	697	[] => (
	698	"oxy <oxy@localhost>".to_string(),
	699	op.hlc.wall_ms / 1000,
	700	format!("oxy: {}\n\n", op.command),
	701	),
79ab8528 Andy McCall 1 month ago ↶	702	many => {
	703	let mut m = format!("oxy: {} ({} changes)\n\n", op.command, many.len());
	704	for h in many {
	705	let c = self.store.get_change(h)?;
75167672 Andy McCall 1 month ago	706	m.push_str(&format!(
	707	"- {} {}\n",
	708	c.change_id.short(8),
	709	c.message.lines().next().unwrap_or("")
	710	));
79ab8528 Andy McCall 1 month ago ↶	711	}
75167672 Andy McCall 1 month ago	712	m.push_str(&format!(
	713	"\nOxy-Changes: {}\n",
	714	many.iter()
	715	.map(\|h\| h.to_hex())
	716	.collect::<Vec<_>>()
	717	.join(" ")
	718	));
79ab8528 Andy McCall 1 month ago ↶	719	("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, m)
	720	}
	721	};
	722	message.push_str(&format!("Oxy-State: {}\n", state_ref.to_hex()));
	723	if snapshot.is_conflicted() {
	724	message.push_str("Oxy-Conflicted: true\n");
	725	}
	726	let mark = next_mark;
	727	next_mark += 1;
30ea9c1e Andy McCall 1 month ago ↶	728	let parents: Vec<String> = last_parent.clone().into_iter().collect();
75167672 Andy McCall 1 month ago	729	self.write_commit(
	730	&mut stream,
	731	"refs/oxy/export",
	732	mark,
	733	&author,
	734	secs,
	735	&message,
	736	&parents,
	737	&snapshot,
	738	)?;
79ab8528 Andy McCall 1 month ago ↶	739	marks.insert(state_ref, mark);
30ea9c1e Andy McCall 1 month ago ↶	740	last_parent = Some(format!(":{mark}"));
79ab8528 Andy McCall 1 month ago ↶	741	prev = Some((state_ref, state.heads.clone()));
	742	report.exported += 1;
	743	}
30ea9c1e Andy McCall 1 month ago ↶	744	if let Some(p) = &last_parent {
30ea9c1e Andy McCall 1 month ago ↶	745	stream.extend_from_slice(format!("reset refs/heads/{branch}\nfrom {p}\n\n").as_bytes());
79ab8528 Andy McCall 1 month ago ↶	746	}
132b884b Andy McCall 1 month ago ↶	747	// Tags: a tag whose state was exported as a bookmark state points at
	748	// that commit; otherwise its state gets a commit of its own.
	749	for t in self.tags()? {
30ea9c1e Andy McCall 1 month ago ↶	750	let target = match (git_states.get(&t.state), marks.get(&t.state)) {
	751	(Some(sha), _) => sha.clone(),
	752	(None, Some(m)) => format!(":{m}"),
	753	(None, None) => {
132b884b Andy McCall 1 month ago ↶	754	let loaded = self.load_state(&t.heads)?;
	755	let snapshot = loaded.materialize();
	756	let mark = next_mark;
	757	next_mark += 1;
75167672 Andy McCall 1 month ago	758	let message = format!(
	759	"oxy: tag {}\n\n{}Oxy-State: {}\n",
	760	t.name,
	761	if t.message.is_empty() {
	762	String::new()
	763	} else {
	764	format!("{}\n\n", t.message)
	765	},
	766	t.state.to_hex()
	767	);
	768	self.write_commit(
	769	&mut stream,
	770	"refs/oxy/export",
	771	mark,
	772	"oxy <oxy@localhost>",
	773	Hlc::now().wall_ms / 1000,
	774	&message,
	775	&[],
	776	&snapshot,
	777	)?;
132b884b Andy McCall 1 month ago ↶	778	marks.insert(t.state, mark);
132b884b Andy McCall 1 month ago ↶	779	report.exported += 1;
30ea9c1e Andy McCall 1 month ago ↶	780	format!(":{mark}")
132b884b Andy McCall 1 month ago ↶	781	}
132b884b Andy McCall 1 month ago ↶	782	};
75167672 Andy McCall 1 month ago	783	stream.extend_from_slice(
	784	format!("reset refs/tags/{}\nfrom {target}\n\n", t.name).as_bytes(),
	785	);
132b884b Andy McCall 1 month ago ↶	786	}
132b884b Andy McCall 1 month ago ↶	787	stream.extend_from_slice(b"done\n");
79ab8528 Andy McCall 1 month ago ↶	788
	789	let mut cmd = Command::new("git");
	790	// `--force`: the scratch ref may jump between unrelated states.
75167672 Andy McCall 1 month ago	791	cmd.arg("-C")
	792	.arg(git_dir)
	793	.args(["fast-import", "--quiet", "--done", "--force"])
	794	.arg(format!("--export-marks={}", git_marks.display()));
79ab8528 Andy McCall 1 month ago ↶	795	if git_marks.exists() {
	796	cmd.arg(format!("--import-marks={}", git_marks.display()));
	797	}
75167672 Andy McCall 1 month ago	798	let mut child = cmd
	799	.stdin(Stdio::piped())
	800	.stdout(Stdio::null())
	801	.stderr(Stdio::piped())
	802	.spawn()
	803	.map_err(\|e\| RepoError::Msg(format!("running git fast-import: {e}")))?;
79ab8528 Andy McCall 1 month ago ↶	804	child.stdin.take().unwrap().write_all(&stream)?;
	805	let out = child.wait_with_output()?;
	806	if !out.status.success() {
75167672 Andy McCall 1 month ago	807	return fail(format!(
	808	"git fast-import failed: {}",
	809	String::from_utf8_lossy(&out.stderr).trim()
	810	));
79ab8528 Andy McCall 1 month ago ↶	811	}
75167672 Andy McCall 1 month ago	812	let text: String = marks
	813	.iter()
	814	.map(\|(h, m)\| format!("{} {m}\n", h.to_hex()))
	815	.collect();
79ab8528 Andy McCall 1 month ago ↶	816	write_atomic(&oxy_marks_path, text.as_bytes())?;
79ab8528 Andy McCall 1 month ago ↶	817	let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]);
4dee277c Andy McCall 1 month ago ↶	818	report.skipped_subrepos = self.git_export_skipped.replace(0);
79ab8528 Andy McCall 1 month ago ↶	819	Ok(report)
	820	}
	821
c3069537 Andy McCall 1 month ago ↶	822	/// Import branch `branch` of the Git repository at `git_dir` into this
	823	/// (empty) repository's bookmark (spec §23): one change per commit, its
	824	/// edits the difference between the union of its parents' states and its
	825	/// tree, so merge commits become resolutions. Submodules are skipped.
	826	pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result<GitImportReport> {
58dfe0e1 Andy McCall 1 month ago ↶	827	self.import_git_with(git_dir, branch, &mut \|_\| {})
	828	}
	829
	830	/// `import_git`, telling `progress` where it is every few hundred
	831	/// commits: the count so far, the merges, and the commit in hand, so a
	832	/// long import is never silent and a run that dies says where.
	833	pub fn import_git_with(
	834	&self,
	835	git_dir: &Path,
	836	branch: &str,
	837	progress: crate::store::Progress<'_>,
	838	) -> Result<GitImportReport> {
c3069537 Andy McCall 1 month ago ↶	839	if !self.bookmark_heads()?.is_empty() {
	840	return fail("git import needs an empty bookmark; import into a fresh repository");
	841	}
75167672 Andy McCall 1 month ago	842	self.import_git_commits(
	843	git_dir,
	844	branch,
	845	&format!("oxy git import {} {branch}", git_dir.display()),
58dfe0e1 Andy McCall 1 month ago ↶	846	progress,
75167672 Andy McCall 1 month ago	847	)
30ea9c1e Andy McCall 1 month ago ↶	848	}
	849
	850	/// Import the commits of `branch` that the bookmark does not have yet,
	851	/// then move the bookmark to the union of its heads and theirs.
75167672 Andy McCall 1 month ago	852	fn import_git_commits(
	853	&self,
	854	git_dir: &Path,
	855	branch: &str,
	856	command: &str,
58dfe0e1 Andy McCall 1 month ago ↶	857	progress: crate::store::Progress<'_>,
75167672 Andy McCall 1 month ago	858	) -> Result<GitImportReport> {
8f10a07a Andy McCall 1 month ago ↶	859	let batch = self.store.begin_batch()?;
58dfe0e1 Andy McCall 1 month ago ↶	860	let result = self.import_git_commits_batched(git_dir, branch, command, progress);
8f10a07a Andy McCall 1 month ago ↶	861	batch.finish()?;
	862	result
	863	}
	864
75167672 Andy McCall 1 month ago	865	fn import_git_commits_batched(
	866	&self,
	867	git_dir: &Path,
	868	branch: &str,
	869	command: &str,
58dfe0e1 Andy McCall 1 month ago ↶	870	progress: crate::store::Progress<'_>,
75167672 Andy McCall 1 month ago	871	) -> Result<GitImportReport> {
c3069537 Andy McCall 1 month ago ↶	872	let commits = list_commits(git_dir, branch)?;
	873	let mut cat = CatFile::start(git_dir)?;
	874	let mut report = GitImportReport::default();
30ea9c1e Andy McCall 1 month ago ↶	875	let mut known = self.git_known(git_dir)?;
30ea9c1e Andy McCall 1 month ago ↶	876	let old_heads = self.bookmark_heads()?;
dfb23a3a Andy McCall 1 month ago ↶	877	// The bookmark's heads as changes arrive: a change stops being a
	878	// head when another depends on it, so no full state of every
	879	// imported change is needed at the end.
	880	let mut heads_now: BTreeSet<ChangeHash> = old_heads.iter().copied().collect();
958fb27d Andy McCall 1 month ago ↶	881	// The state and Git tree of imported commits whose first-parent
	882	// children are still to come, so a child starts from its parent's
	883	// state instead of rebuilding it from history. The last child takes
dfb23a3a Andy McCall 1 month ago ↶	884	// it; earlier ones clone it. At most `max_kept` are kept (a later
958fb27d Andy McCall 1 month ago ↶	885	// child of an evicted one rebuilds).
dfb23a3a Andy McCall 1 month ago ↶	886	// OXY_IMPORT_KEEP overrides it, for measuring.
	887	let max_kept: usize = std::env::var("OXY_IMPORT_KEEP")
	888	.ok()
	889	.and_then(\|v\| v.parse().ok())
	890	.unwrap_or(8);
958fb27d Andy McCall 1 month ago ↶	891	let mut waiting: HashMap<&str, usize> = HashMap::new();
	892	for gc in &commits {
	893	if let Some(p1) = gc.parents.first() {
	894	*waiting.entry(p1.as_str()).or_default() += 1;
	895	}
	896	}
	897	type Kept = (State, BTreeMap<String, (String, String)>);
	898	let mut kept: HashMap<String, Kept> = HashMap::new();
2306a4d2 Andy McCall 1 month ago ↶	899	// Every commit to import that has a parent, diffed against its first.
	900	let pairs: Vec<(String, String)> = commits
	901	.iter()
	902	.filter(\|gc\| {
	903	!known.imported.contains_key(&gc.sha) && !known.exported.contains_key(&gc.sha)
	904	})
	905	.filter_map(\|gc\| Some((gc.sha.clone(), gc.parents.first()?.clone())))
	906	.collect();
	907	let mut diffs = DiffTree::start(git_dir, pairs)?;
	908	let mut rebuilt = 0usize;
958fb27d Andy McCall 1 month ago ↶	909	let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
958fb27d Andy McCall 1 month ago ↶	910	let mut last: Option<String> = None;
c3069537 Andy McCall 1 month ago ↶	911	let mut blob_cache: HashMap<String, Vec<u8>> = HashMap::new();
c3069537 Andy McCall 1 month ago ↶	912	let mut cache_bytes: usize = 0;
958fb27d Andy McCall 1 month ago ↶	913	// Time spent per phase over all commits (`OXY_TIMINGS=1`).
	914	let mut spent = [std::time::Duration::ZERO; 6];
	915	let mut clock = std::time::Instant::now();
fb5b6181 Andy McCall 1 month ago ↶	916	// One line per commit on stderr with OXY_IMPORT_TRACE=1: where its
	917	// base came from, what a merge added, the graphs its state holds,
	918	// the kept states and the process's resident memory.
	919	let trace = std::env::var_os("OXY_IMPORT_TRACE").is_some();
	920	let rss_mb = \|\| -> u64 {
	921	std::fs::read_to_string("/proc/self/statm")
	922	.ok()
	923	.and_then(\|s\| s.split_whitespace().nth(1)?.parse::<u64>().ok())
d2e3c39d Andy McCall 1 month ago ↶	924	.map(\|pages\| (pages * 4096) >> 20)
fb5b6181 Andy McCall 1 month ago ↶	925	.unwrap_or(0)
	926	};
	927	let mut done = 0usize;
d45dab04 Andy McCall 1 month ago ↶	928	// Atoms a kept state's graphs may hold before they are shed;
	929	// OXY_IMPORT_KEPT_ATOMS overrides it, for measuring.
	930	let kept_atoms: usize = std::env::var("OXY_IMPORT_KEPT_ATOMS")
	931	.ok()
	932	.and_then(\|v\| v.parse().ok())
	933	.unwrap_or(KEPT_ATOMS);
fd280228 Andy McCall 1 month ago ↶	934	// Graphs turned back into history (see `State::shed_graphs`).
fd280228 Andy McCall 1 month ago ↶	935	let mut shed = 0usize;
958fb27d Andy McCall 1 month ago ↶	936	let mut lap = \|i: usize, clock: &mut std::time::Instant\| {
	937	let now = std::time::Instant::now();
	938	spent[i] += now - *clock;
	939	*clock = now;
	940	};
c3069537 Andy McCall 1 month ago ↶	941
9935d490 Andy McCall 29 days ago ↶	942	// OXY_IMPORT_CHECKPOINT overrides it, for testing.
	943	let checkpoint_every: usize = std::env::var("OXY_IMPORT_CHECKPOINT")
	944	.ok()
	945	.and_then(\|v\| v.parse().ok())
	946	.filter(\|n\| *n > 0)
	947	.unwrap_or(CHECKPOINT_COMMITS);
810691ca Andy McCall 29 days ago ↶	948	// OXY_IMPORT_STOP_AFTER ends the import cleanly after that many
	949	// commits, its timings reported, for measuring at depth.
	950	let stop_after: Option<usize> = std::env::var("OXY_IMPORT_STOP_AFTER")
	951	.ok()
	952	.and_then(\|v\| v.parse().ok());
579afd10 Andy McCall 27 days ago ↶	953	// Merges built from the paths they can change (oxygen#80) only with
	954	// OXY_IMPORT_PARTIAL_MERGES=1 until verified at VS Code's depth;
	955	// otherwise from the whole tree, as before.
	956	let partial_merges =
	957	std::env::var_os("OXY_IMPORT_PARTIAL_MERGES").is_some_and(\|v\| v != "0");
58dfe0e1 Andy McCall 1 month ago ↶	958	let total = commits.len();
58dfe0e1 Andy McCall 1 month ago ↶	959	for (index, gc) in commits.iter().enumerate() {
30ea9c1e Andy McCall 1 month ago ↶	960	if known.imported.contains_key(&gc.sha) \|\| known.exported.contains_key(&gc.sha) {
30ea9c1e Andy McCall 1 month ago ↶	961	continue;
810691ca Andy McCall 29 days ago ↶	962	}
	963	if stop_after.is_some_and(\|n\| done >= n) {
	964	break;
58dfe0e1 Andy McCall 1 month ago ↶	965	}
810691ca Andy McCall 29 days ago ↶	966	let commit_started = std::time::Instant::now();
58dfe0e1 Andy McCall 1 month ago ↶	967	if index % 500 == 0 {
	968	progress(&format!(
	969	"{index} of {total} commits, {} merges; at {} {}",
	970	report.merges,
	971	&gc.sha[..12.min(gc.sha.len())],
	972	gc.message
	973	.lines()
	974	.next()
	975	.unwrap_or("")
	976	.chars()
	977	.take(60)
	978	.collect::<String>()
	979	));
30ea9c1e Andy McCall 1 month ago ↶	980	}
fb5b6181 Andy McCall 1 month ago ↶	981	let mut base_how = "root";
958fb27d Andy McCall 1 month ago ↶	982	// Base: union of every parent's state, starting from the first
	983	// parent's kept state when there is one; its tree tells which
	984	// paths this commit leaves alone.
	985	let (mut base, prev_tree) = match gc.parents.first() {
	986	Some(p1) => {
	987	let left = waiting.get_mut(p1.as_str()).map(\|n\| {
	988	*n -= 1;
	989	*n
	990	});
	991	let found = if left == Some(0) {
	992	kept_order.retain(\|k\| k != p1);
	993	kept.remove(p1.as_str())
	994	} else {
	995	kept.get(p1.as_str()).cloned()
fb5b6181 Andy McCall 1 month ago ↶	996	};
	997	let how = match (&found, left) {
	998	(Some(_), Some(0)) => "kept",
	999	(Some(_), _) => "cloned",
	1000	(None, _) => "rebuilt",
958fb27d Andy McCall 1 month ago ↶	1001	};
fb5b6181 Andy McCall 1 month ago ↶	1002	if trace {
	1003	base_how = how;
	1004	}
958fb27d Andy McCall 1 month ago ↶	1005	match found {
	1006	Some((st, tree)) => (st, Some(tree)),
	1007	None => {
b493d993 Andy McCall 1 month ago ↶	1008	// History only: graphs are built for the files a
	1009	// commit changes, so memory follows what changes,
	1010	// not the whole tree.
	1011	let mut st = State::new_history_only();
958fb27d Andy McCall 1 month ago ↶	1012	self.add_git_ancestors(&mut st, git_dir, &[p1.as_str()], &known)?;
2306a4d2 Andy McCall 1 month ago ↶	1013	// The parent's tree, so this commit still
	1014	// renders only what it changes.
	1015	rebuilt += 1;
	1016	(st, Some(list_tree(git_dir, p1)?))
958fb27d Andy McCall 1 month ago ↶	1017	}
958fb27d Andy McCall 1 month ago ↶	1018	}
30ea9c1e Andy McCall 1 month ago ↶	1019	}
b493d993 Andy McCall 1 month ago ↶	1020	None => (State::new_history_only(), None),
c3069537 Andy McCall 1 month ago ↶	1021	};
958fb27d Andy McCall 1 month ago ↶	1022	// Nodes the other parents' changes touch: the first parent's
	1023	// tree cannot vouch for them.
	1024	let mut touched: HashSet<NodeId> = HashSet::new();
fb5b6181 Andy McCall 1 month ago ↶	1025	let mut merged = 0usize;
ce118287 Andy McCall 29 days ago ↶	1026	// Where the first parent has each node, read before the merged
	1027	// changes move anything: a merge's partial build names the paths
	1028	// they touch in the first parent's tree as well as in the merged
	1029	// state (oxygen#80).
	1030	let mut first_parent_view: Option<crate::tree::TreeView> = None;
c3069537 Andy McCall 1 month ago ↶	1031	if gc.parents.len() > 1 {
c3069537 Andy McCall 1 month ago ↶	1032	report.merges += 1;
579afd10 Andy McCall 27 days ago ↶	1033	if prev_tree.is_some() && partial_merges {
ce118287 Andy McCall 29 days ago ↶	1034	first_parent_view = Some(base.tree().view());
ce118287 Andy McCall 29 days ago ↶	1035	}
c3069537 Andy McCall 1 month ago ↶	1036	let mut args: Vec<String> = gc.parents[1..].to_vec();
c3069537 Andy McCall 1 month ago ↶	1037	args.push(format!("^{}", gc.parents[0]));
30ea9c1e Andy McCall 1 month ago ↶	1038	let refs: Vec<&str> = args.iter().map(String::as_str).collect();
fd280228 Andy McCall 1 month ago ↶	1039	// A merge of a long-lived branch brings thousands of
	1040	// changes: added to built graphs, which kept states share,
	1041	// they were copied and applied whole (vscode's import passed
	1042	// 14 GB on such merges). Shed the graphs first, so the
	1043	// changes go in as skeletons and only what this commit
	1044	// touches is built again.
	1045	if base.built_count() > 0 {
	1046	let mut count_args = vec!["rev-list", "--count"];
	1047	count_args.extend(refs.iter().copied());
	1048	let others: usize = git(git_dir, &count_args)?.trim().parse().unwrap_or(0);
33be2409 Andy McCall 1 month ago ↶	1049	let shed_at = std::env::var("OXY_MERGE_SHED")
	1050	.ok()
	1051	.and_then(\|v\| v.parse().ok())
	1052	.unwrap_or(MERGE_SHED_COMMITS);
	1053	if others > shed_at {
fd280228 Andy McCall 1 month ago ↶	1054	shed += base.shed_graphs(\|b\| self.store_blob(b))?;
	1055	}
	1056	}
fb5b6181 Andy McCall 1 month ago ↶	1057	let before = base.len();
958fb27d Andy McCall 1 month ago ↶	1058	self.add_git_ancestors_touching(&mut base, git_dir, &refs, &known, &mut touched)?;
fb5b6181 Andy McCall 1 month ago ↶	1059	merged = base.len() - before;
03571027 Andy McCall 1 month ago ↶	1060	}
958fb27d Andy McCall 1 month ago ↶	1061	lap(0, &mut clock);
2306a4d2 Andy McCall 1 month ago ↶	1062	// The commit's tree: its first parent's with the diff applied,
	1063	// or listed whole when that tree is not kept.
	1064	let diff = match gc.parents.first() {
	1065	Some(_) => Some(diffs.diff(&gc.sha)?),
	1066	None => None,
	1067	};
	1068	let (tree, changed) = match (prev_tree, diff) {
	1069	(Some(mut t), Some(diff)) => {
	1070	let mut changed: HashSet<String> = HashSet::new();
	1071	for (path, entry) in diff {
	1072	match entry {
	1073	Some(v) => {
	1074	t.insert(path.clone(), v);
	1075	}
	1076	None => {
	1077	t.remove(&path);
	1078	}
	1079	}
	1080	changed.insert(path);
	1081	}
	1082	(t, Some(changed))
	1083	}
	1084	_ => (list_tree(git_dir, &gc.sha)?, None),
	1085	};
958fb27d Andy McCall 1 month ago ↶	1086	lap(1, &mut clock);
03571027 Andy McCall 1 month ago ↶	1087	// Text files with the same blob and mode as in the first
	1088	// parent's tree equal the base's rendering: they are neither
	1089	// rendered nor read nor compared (spec §12.3, as for a clean
	1090	// working copy), so a commit costs what it changes.
	1091	let mut same: HashMap<&str, NodeId> = HashMap::new();
2306a4d2 Andy McCall 1 month ago ↶	1092	// Opaque files and symlinks likewise, when the base has one
	1093	// blob for them: they are neither read nor stored again.
	1094	let mut same_opaque: HashSet<&str> = HashSet::new();
	1095	// An ordinary commit in a tree without submodules is built from
	1096	// the paths it changes alone (oxygen#25): with the `.oxyattributes`
	1097	// files that decide how they are stored, and the directories its
	1098	// removals may leave empty. A merge, or a commit whose parent's
	1099	// tree is not at hand, is built from its whole tree.
ce118287 Andy McCall 29 days ago ↶	1100	// A merge can differ from its first parent only at the paths
	1101	// Git's diff names and where the merged changes touched: those
	1102	// paths in the first parent's tree and in the merged state. With
	1103	// them added it is built like an ordinary commit, instead of
	1104	// from its whole tree (18 s for any merge at 118k commits of
579afd10 Andy McCall 27 days ago ↶	1105	// VS Code, oxygen#80). For now only with
579afd10 Andy McCall 27 days ago ↶	1106	// OXY_IMPORT_PARTIAL_MERGES=1; otherwise the whole-tree path.
ce118287 Andy McCall 29 days ago ↶	1107	let mut changed = changed;
	1108	// A tree conflict in the merged state (both sides placed a file
	1109	// at one path) is resolved by the whole-tree path.
	1110	let mut merge_how = if gc.parents.len() > 1 {
	1111	if first_parent_view.is_some() {
	1112	"partial"
	1113	} else {
	1114	"whole"
	1115	}
	1116	} else {
	1117	"-"
	1118	};
	1119	// Only a conflict among what the merged changes touched; one
	1120	// elsewhere stays as it was in the first parent either way.
	1121	if first_parent_view.is_some()
	1122	&& base.tree().view().conflicts.iter().any(\|c\| {
	1123	use crate::tree::TreeConflict::*;
	1124	match c {
	1125	Name { nodes, .. } => nodes.iter().any(\|n\| touched.contains(n)),
	1126	MultiPlacement { node, .. } \| Cycle { node } \| Mode { node, .. } => {
	1127	touched.contains(node)
	1128	}
	1129	Orphan { node, parent } => {
	1130	touched.contains(node) \|\| touched.contains(parent)
	1131	}
	1132	}
	1133	})
	1134	{
	1135	first_parent_view = None;
	1136	merge_how = "conflicted";
	1137	}
	1138	if let (Some(ch), Some(fp)) = (changed.as_mut(), &first_parent_view) {
	1139	let union = base.tree().view();
	1140	for v in [fp, &union] {
	1141	for (path, e) in v.entries.iter() {
	1142	if touched.contains(&e.node) {
	1143	ch.insert(path.clone());
	1144	}
	1145	}
	1146	}
	1147	}
2306a4d2 Andy McCall 1 month ago ↶	1148	let partial = changed.is_some()
ce118287 Andy McCall 29 days ago ↶	1149	&& (gc.parents.len() == 1 \|\| first_parent_view.is_some())
2306a4d2 Andy McCall 1 month ago ↶	1150	&& !tree.values().any(\|(mode, _)\| mode == "160000");
2306a4d2 Andy McCall 1 month ago ↶	1151	let mut rel: BTreeSet<&str> = BTreeSet::new();
ce118287 Andy McCall 29 days ago ↶	1152	// Text files whose graphs this commit builds, for the trace.
ce118287 Andy McCall 29 days ago ↶	1153	let need_n: usize;
2306a4d2 Andy McCall 1 month ago ↶	1154	let base_snapshot = if let (true, Some(changed)) = (partial, &changed) {
03571027 Andy McCall 1 month ago ↶	1155	let view = base.tree().view();
2306a4d2 Andy McCall 1 month ago ↶	1156	rel.extend(changed.iter().map(String::as_str));
	1157	rel.extend(
	1158	tree.keys()
	1159	.map(String::as_str)
	1160	.filter(\|p\| *p == ".oxyattributes" \|\| p.ends_with("/.oxyattributes")),
	1161	);
	1162	let mut dirs: BTreeSet<&str> = BTreeSet::new();
	1163	for p in changed.iter().filter(\|p\| !tree.contains_key(*p)) {
	1164	let mut end = p.len();
	1165	while let Some(i) = p[..end].rfind('/') {
	1166	if !dirs.insert(&p[..i]) {
	1167	break;
	1168	}
	1169	end = i;
03571027 Andy McCall 1 month ago ↶	1170	}
2306a4d2 Andy McCall 1 month ago ↶	1171	}
	1172	let need: Vec<NodeId> = rel
	1173	.iter()
	1174	.filter_map(\|p\| view.entries.get(*p))
	1175	.filter(\|e\| e.kind == NodeKind::File && !e.mode.opaque)
	1176	.map(\|e\| e.node)
	1177	.collect();
	1178	crate::timing::add("render: same and need", clock.elapsed());
ce118287 Andy McCall 29 days ago ↶	1179	need_n = need.len();
2306a4d2 Andy McCall 1 month ago ↶	1180	let t = std::time::Instant::now();
	1181	self.load_graphs_of(&mut base, &need)?;
	1182	crate::timing::add("render: load graphs", t.elapsed());
	1183	let t = std::time::Instant::now();
	1184	let snap = base.materialize_paths(view, rel.iter().chain(dirs.iter()).copied());
	1185	crate::timing::add("render: materialize", t.elapsed());
	1186	snap
	1187	} else {
	1188	if let Some(changed) = &changed {
	1189	let view = base.tree().view();
	1190	for path in tree.keys() {
	1191	// Submodule handling reads these two whatever changed.
	1192	if changed.contains(path) \|\| path == ".gitmodules" \|\| path == ".oxymodules"
958fb27d Andy McCall 1 month ago ↶	1193	{
2306a4d2 Andy McCall 1 month ago ↶	1194	continue;
	1195	}
	1196	if let Some(e) = view.entries.get(path) {
	1197	if touched.contains(&e.node) {
	1198	continue;
	1199	}
	1200	let opaque = e.kind == NodeKind::Symlink
	1201	\|\| (e.kind == NodeKind::File && e.mode.opaque);
	1202	if e.kind == NodeKind::File && !e.mode.opaque {
	1203	same.insert(path.as_str(), e.node);
	1204	} else if opaque && base.live_blobs(&e.node).len() == 1 {
	1205	same_opaque.insert(path.as_str());
	1206	}
03571027 Andy McCall 1 month ago ↶	1207	}
	1208	}
	1209	}
2306a4d2 Andy McCall 1 month ago ↶	1210	// Build the graphs of every text file the commit may change.
	1211	let skip: HashSet<NodeId> = same.values().copied().collect();
	1212	let need: Vec<NodeId> = base
	1213	.tree()
	1214	.view()
	1215	.entries
	1216	.values()
	1217	.filter(\|e\| {
	1218	e.kind == NodeKind::File && !e.mode.opaque && !skip.contains(&e.node)
	1219	})
	1220	.map(\|e\| e.node)
	1221	.collect();
	1222	crate::timing::add("render: same and need", clock.elapsed());
ce118287 Andy McCall 29 days ago ↶	1223	need_n = need.len();
2306a4d2 Andy McCall 1 month ago ↶	1224	let t = std::time::Instant::now();
c3d2ef61 Andy McCall 1 month ago ↶	1225	let snap = if need.len() > merge_batch() {
	1226	// A merge onto a base with no built graphs: thousands
	1227	// of files, whose graphs would not fit at once.
	1228	self.materialize_batched(&mut base, &need, &skip)?
	1229	} else {
ce118287 Andy McCall 29 days ago ↶	1230	let g = std::time::Instant::now();
c3d2ef61 Andy McCall 1 month ago ↶	1231	self.load_graphs_of(&mut base, &need)?;
ce118287 Andy McCall 29 days ago ↶	1232	crate::timing::add("render: whole, load graphs", g.elapsed());
	1233	let m = std::time::Instant::now();
	1234	let snap = base.materialize_except(&skip);
	1235	crate::timing::add("render: whole, materialize", m.elapsed());
	1236	snap
c3d2ef61 Andy McCall 1 month ago ↶	1237	};
c3d2ef61 Andy McCall 1 month ago ↶	1238	crate::timing::add("render: whole tree", t.elapsed());
2306a4d2 Andy McCall 1 month ago ↶	1239	snap
2306a4d2 Andy McCall 1 month ago ↶	1240	};
c3069537 Andy McCall 1 month ago ↶	1241
958fb27d Andy McCall 1 month ago ↶	1242	lap(2, &mut clock);
c3069537 Andy McCall 1 month ago ↶	1243	// The commit's tree as in-memory entries.
c3069537 Andy McCall 1 month ago ↶	1244	let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();
4dee277c Andy McCall 1 month ago ↶	1245	let mut gitlinks: Vec<(String, String)> = Vec::new();
2306a4d2 Andy McCall 1 month ago ↶	1246	let listed: Box<dyn Iterator<Item = (&String, &(String, String))>> = if partial {
	1247	Box::new(rel.iter().filter_map(\|p\| tree.get_key_value(*p)))
	1248	} else {
	1249	Box::new(tree.iter())
	1250	};
	1251	for (path, (mode, sha)) in listed {
03571027 Andy McCall 1 month ago ↶	1252	let (path, sha) = (path.clone(), sha.clone());
c3069537 Andy McCall 1 month ago ↶	1253	let (kind, exec) = match mode.as_str() {
	1254	"100644" \| "100664" => (NodeKind::File, false),
	1255	"100755" => (NodeKind::File, true),
	1256	"120000" => (NodeKind::Symlink, false),
	1257	"160000" => {
4dee277c Andy McCall 1 month ago ↶	1258	gitlinks.push((path, sha));
c3069537 Andy McCall 1 month ago ↶	1259	continue;
	1260	}
	1261	_ => continue,
	1262	};
2306a4d2 Andy McCall 1 month ago ↶	1263	if same.contains_key(path.as_str()) \|\| same_opaque.contains(path.as_str()) {
03571027 Andy McCall 1 month ago ↶	1264	let mut e = FsEntry::new(kind, exec, Vec::new());
	1265	e.unchanged = true;
	1266	entries.insert(path, e);
	1267	continue;
	1268	}
c3069537 Andy McCall 1 month ago ↶	1269	let content = match blob_cache.get(&sha) {
	1270	Some(c) => c.clone(),
	1271	None => {
	1272	let c = cat.read(&sha)?;
dfb23a3a Andy McCall 1 month ago ↶	1273	if cache_bytes + c.len() > 64 * 1024 * 1024 {
c3069537 Andy McCall 1 month ago ↶	1274	blob_cache.clear();
	1275	cache_bytes = 0;
	1276	}
	1277	cache_bytes += c.len();
	1278	blob_cache.insert(sha.clone(), c.clone());
	1279	c
	1280	}
	1281	};
b7f102fc Andy McCall 1 month ago ↶	1282	entries.insert(path, FsEntry::new(kind, exec, content));
c3069537 Andy McCall 1 month ago ↶	1283	}
c3069537 Andy McCall 1 month ago ↶	1284
4dee277c Andy McCall 1 month ago ↶	1285	// Submodules (spec §23): gitlinks become subrepo nodes with Git
	1286	// pins, and `.gitmodules` also yields `.oxymodules`.
	1287	if !gitlinks.is_empty() {
	1288	report.submodules += gitlinks.len();
75167672 Andy McCall 1 month ago	1289	let gm = entries
	1290	.get(".gitmodules")
	1291	.map(\|e\| {
	1292	crate::subrepo::Modules::from_gitmodules(&String::from_utf8_lossy(
	1293	&e.content,
	1294	))
	1295	})
	1296	.unwrap_or_default();
	1297	let mut modules = entries
	1298	.get(".oxymodules")
	1299	.map(\|e\| crate::subrepo::Modules::parse(&String::from_utf8_lossy(&e.content)))
	1300	.unwrap_or_default();
4dee277c Andy McCall 1 month ago ↶	1301	for (path, sha) in &gitlinks {
4dee277c Andy McCall 1 month ago ↶	1302	let m = gm.entries.get(path);
75167672 Andy McCall 1 month ago	1303	let branch = m
	1304	.and_then(\|m\| m.bookmark.clone())
	1305	.unwrap_or_else(\|\| "HEAD".into());
	1306	entries.insert(
	1307	path.clone(),
	1308	FsEntry::new(
	1309	NodeKind::Subrepo,
	1310	false,
	1311	crate::subrepo::Pin::Git {
	1312	sha: sha.clone(),
	1313	branch,
	1314	}
	1315	.encode(),
	1316	),
	1317	);
4dee277c Andy McCall 1 month ago ↶	1318	if let Some(m) = m {
	1319	modules.entries.insert(path.clone(), m.clone());
	1320	}
	1321	}
	1322	if !modules.entries.is_empty() {
75167672 Andy McCall 1 month ago	1323	entries.insert(
	1324	".oxymodules".into(),
	1325	FsEntry::new(NodeKind::File, false, modules.to_text().into_bytes()),
	1326	);
4dee277c Andy McCall 1 month ago ↶	1327	}
4dee277c Andy McCall 1 month ago ↶	1328	}
958fb27d Andy McCall 1 month ago ↶	1329	lap(3, &mut clock);
dfb23a3a Andy McCall 1 month ago ↶	1330	// New nodes' identifiers derive from the commit, so importing the
	1331	// same Git history twice gives the same changes (spec §23).
	1332	let seed = *blake3::hash(format!("oxy git import\0{}", gc.sha).as_bytes()).as_bytes();
2306a4d2 Andy McCall 1 month ago ↶	1333	let mut built = if partial {
	1334	// Whether the new tree has anything under `dir`.
	1335	let dir_needed = \|dir: &str\| {
	1336	let prefix = format!("{dir}/");
	1337	tree.range(prefix.clone()..)
	1338	.next()
	1339	.is_some_and(\|(k, _)\| k.starts_with(&prefix))
	1340	};
	1341	self.build_from_entries_seeded(
	1342	&base,
	1343	&base_snapshot,
	1344	&entries,
	1345	seed,
	1346	Some(&dir_needed),
	1347	)?
	1348	} else {
	1349	self.build_from_entries_seeded(&base, &base_snapshot, &entries, seed, None)?
	1350	};
958fb27d Andy McCall 1 month ago ↶	1351	lap(4, &mut clock);
62794ecc Andy McCall 1 month ago ↶	1352	// A file or submodule with several live blobs (branches that
	1353	// replaced it independently) is resolved to the commit's blob:
	1354	// Git's merge is the authority here, unlike a working copy,
	1355	// where a candidate on disk is not yet a decision (spec §23).
	1356	for (path, f) in &base_snapshot.files {
	1357	let (FileContent::Opaque { blobs }, Some(e)) = (&f.content, entries.get(path))
	1358	else {
	1359	continue;
	1360	};
	1361	if blobs.len() < 2 \|\| f.kind != e.kind {
	1362	continue;
	1363	}
	1364	let chosen = self.store_blob(&e.content)?;
	1365	for b in blobs {
	1366	built.edits.push(Edit::Replace {
	1367	node: f.node,
	1368	old: Some(*b),
	1369	new: Some(chosen),
	1370	});
	1371	}
	1372	built.resolves = true;
	1373	built
	1374	.contributors
	1375	.extend(base.live_replaces(&f.node).into_iter().map(\|(c, _)\| c));
	1376	}
e83354cd Andy McCall 1 month ago ↶	1377	// It shares the base's tree view, which applying the change
	1378	// then updates in place rather than copying.
	1379	drop(base_snapshot);
	1380	drop(entries);
c3069537 Andy McCall 1 month ago ↶	1381	let mut deps: BTreeSet<ChangeHash> = built.contributors.iter().copied().collect();
	1382	let mut message = gc.message.clone();
	1383	if !message.ends_with('\n') {
	1384	message.push('\n');
	1385	}
75167672 Andy McCall 1 month ago	1386	message.push_str(&format!(
	1387	"\nGit-Commit: {}\nGit-Author: {}\n",
	1388	gc.sha, gc.author
	1389	));
c3069537 Andy McCall 1 month ago ↶	1390	let mut change = Change {
6f8aeb99 Andy McCall 1 month ago ↶	1391	version: crate::objects::CHANGE_VERSION,
c3069537 Andy McCall 1 month ago ↶	1392	change_id: change_id_for(&gc.sha, &gc.message),
	1393	deps: vec![],
	1394	author: ObjectRef::ZERO,
75167672 Andy McCall 1 month ago	1395	hlc: Hlc {
	1396	wall_ms: gc.author_secs * 1000,
	1397	logical: 0,
	1398	},
c3069537 Andy McCall 1 month ago ↶	1399	message,
	1400	tree_ops: built.tree_ops,
	1401	edits: built.edits,
75167672 Andy McCall 1 month ago	1402	kind: if built.resolves {
	1403	ChangeKind::Resolution
	1404	} else {
	1405	ChangeKind::Normal
	1406	},
c3069537 Andy McCall 1 month ago ↶	1407	};
2306a4d2 Andy McCall 1 month ago ↶	1408	let t = std::time::Instant::now();
73d592e7 Andy McCall 1 month ago ↶	1409	// A deletion depends on whoever inserted the live atoms after
	1410	// it, which only the file's graph knows: build the few the
	1411	// batched rendering left deferred.
	1412	let need: Vec<NodeId> = change
	1413	.edits
	1414	.iter()
	1415	.filter(\|e\| matches!(e, Edit::Delete { .. }))
	1416	.map(\|e\| e.node())
	1417	.filter(\|n\| !base.is_built(n))
	1418	.collect();
	1419	if !need.is_empty() {
	1420	self.load_graphs_of(&mut base, &need)?;
	1421	}
c3069537 Andy McCall 1 month ago ↶	1422	deps.extend(referenced_changes(&change, &base));
2306a4d2 Andy McCall 1 month ago ↶	1423	crate::timing::add("store: referenced", t.elapsed());
c3069537 Andy McCall 1 month ago ↶	1424	change.deps = deps.into_iter().collect();
	1425	change.normalize();
	1426	let hash = self.store.put_change(&change)?;
dfb23a3a Andy McCall 1 month ago ↶	1427	for d in &change.deps {
	1428	heads_now.remove(d);
	1429	}
	1430	heads_now.insert(hash);
ab4d4e31 Andy McCall 1 month ago ↶	1431	// Built graphs take its edits whole; files not built keep only
ab4d4e31 Andy McCall 1 month ago ↶	1432	// where its text is, read back from the store.
2306a4d2 Andy McCall 1 month ago ↶	1433	let t = std::time::Instant::now();
ab4d4e31 Andy McCall 1 month ago ↶	1434	let (skeleton, texts) = self.store.get_change_lazy(&hash)?;
ab4d4e31 Andy McCall 1 month ago ↶	1435	base.add_known_mixed(hash, change, skeleton, texts);
2306a4d2 Andy McCall 1 month ago ↶	1436	crate::timing::add("store: add to state", t.elapsed());
30ea9c1e Andy McCall 1 month ago ↶	1437	known.imported.insert(gc.sha.clone(), hash);
958fb27d Andy McCall 1 month ago ↶	1438	if waiting.get(gc.sha.as_str()).is_some_and(\|n\| *n > 0) {
fd280228 Andy McCall 1 month ago ↶	1439	// A kept state holds its graphs until its children come:
fd280228 Andy McCall 1 month ago ↶	1440	// past a few thousand, they go back into its history.
d45dab04 Andy McCall 1 month ago ↶	1441	// Past a few thousand graphs, or past a budget of atoms: a
	1442	// long-lived file's graph keeps every line it ever had, so a
	1443	// handful of them can hold gigabytes (#25).
	1444	if base.built_count() > KEPT_GRAPHS \|\| base.built_atoms() > kept_atoms {
fd280228 Andy McCall 1 month ago ↶	1445	shed += base.shed_graphs(\|b\| self.store_blob(b))?;
fd280228 Andy McCall 1 month ago ↶	1446	}
958fb27d Andy McCall 1 month ago ↶	1447	kept.insert(gc.sha.clone(), (base, tree));
958fb27d Andy McCall 1 month ago ↶	1448	kept_order.push_back(gc.sha.clone());
dfb23a3a Andy McCall 1 month ago ↶	1449	if kept_order.len() > max_kept {
958fb27d Andy McCall 1 month ago ↶	1450	if let Some(old) = kept_order.pop_front() {
	1451	kept.remove(&old);
	1452	}
	1453	}
	1454	}
	1455	last = Some(gc.sha.clone());
c3069537 Andy McCall 1 month ago ↶	1456	report.commits += 1;
958fb27d Andy McCall 1 month ago ↶	1457	lap(5, &mut clock);
fb5b6181 Andy McCall 1 month ago ↶	1458	done += 1;
9935d490 Andy McCall 29 days ago ↶	1459	if done.is_multiple_of(checkpoint_every) {
	1460	self.checkpoint_import(&heads_now, done)?;
	1461	}
d45dab04 Andy McCall 1 month ago ↶	1462	// A merge renders thousands of files on many threads and frees
	1463	// it all; the allocator keeps those pages unless asked to give
	1464	// them back, and the next merge piles on top (#25).
	1465	let before_trim = if trace { rss_mb() } else { 0 };
	1466	if gc.parents.len() > 1 {
	1467	crate::alloc_stats::release_freed_memory();
	1468	}
fb5b6181 Andy McCall 1 month ago ↶	1469	if trace {
	1470	let built = kept
	1471	.get(gc.sha.as_str())
	1472	.map(\|(st, _)\| st.built_count())
d45dab04 Andy McCall 1 month ago ↶	1473	.unwrap_or(0);
	1474	let (edits, most) = kept
	1475	.get(gc.sha.as_str())
	1476	.map(\|(st, _)\| st.deferred_edit_count())
	1477	.unwrap_or((0, 0));
	1478	let atoms = kept
	1479	.get(gc.sha.as_str())
	1480	.map(\|(st, _)\| st.built_atoms())
fb5b6181 Andy McCall 1 month ago ↶	1481	.unwrap_or(0);
fb5b6181 Andy McCall 1 month ago ↶	1482	eprintln!(
ce118287 Andy McCall 29 days ago ↶	1483	"trace {done} {} parents={} base={base_how} merged={merged} built={built} atoms={atoms} kept={} shed={shed} rss_mb={} before_trim_mb={before_trim} deferred_edits={edits} most={most} need={need_n} merge={merge_how} ms={}",
fb5b6181 Andy McCall 1 month ago ↶	1484	&gc.sha[..12.min(gc.sha.len())],
	1485	gc.parents.len(),
	1486	kept.len(),
810691ca Andy McCall 29 days ago ↶	1487	rss_mb(),
810691ca Andy McCall 29 days ago ↶	1488	commit_started.elapsed().as_millis()
fb5b6181 Andy McCall 1 month ago ↶	1489	);
fb5b6181 Andy McCall 1 month ago ↶	1490	}
958fb27d Andy McCall 1 month ago ↶	1491	}
	1492	for (name, d) in [
	1493	"git import: bases",
2306a4d2 Andy McCall 1 month ago ↶	1494	"git import: tree",
958fb27d Andy McCall 1 month ago ↶	1495	"git import: render base",
	1496	"git import: read blobs",
	1497	"git import: build change",
	1498	"git import: store change",
	1499	]
	1500	.iter()
	1501	.zip(spent)
	1502	{
	1503	crate::timing::record(name, d);
2306a4d2 Andy McCall 1 month ago ↶	1504	}
	1505	crate::timing::report_totals();
	1506	if crate::timing::enabled() {
fd280228 Andy McCall 1 month ago ↶	1507	eprintln!(
	1508	"timing git import: {rebuilt} bases rebuilt from history, {shed} graphs shed"
	1509	);
c3069537 Andy McCall 1 month ago ↶	1510	}
c3069537 Andy McCall 1 month ago ↶	1511
30ea9c1e Andy McCall 1 month ago ↶	1512	// The bookmark: heads of the old state plus every new change.
	1513	report.heads = old_heads.len();
	1514	if last.is_none() {
	1515	return Ok(report);
	1516	}
dfb23a3a Andy McCall 1 month ago ↶	1517	let heads: Vec<ChangeHash> = heads_now.into_iter().collect();
c3069537 Andy McCall 1 month ago ↶	1518	report.heads = heads.len();
	1519	let (op_ref, op) = self.head_op()?;
	1520	let mut view = op.view.clone();
30ea9c1e Andy McCall 1 month ago ↶	1521	let old_state_ref = view.bookmarks.get(&self.config.bookmark).copied();
30ea9c1e Andy McCall 1 month ago ↶	1522	let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?;
75167672 Andy McCall 1 month ago	1523	view.bookmarks
75167672 Andy McCall 1 month ago	1524	.insert(self.config.bookmark.clone(), state_ref);
30ea9c1e Andy McCall 1 month ago ↶	1525	self.record_op(&[op_ref], command, view)?;
	1526	// The new state equals the tip's tree when the old one was itself a
	1527	// Git commit's tree (fetched or exported), or there was none.
75167672 Andy McCall 1 month ago	1528	let old_known = old_heads.is_empty()
75167672 Andy McCall 1 month ago	1529	\|\| old_state_ref.is_some_and(\|s\| known.exported.values().any(\|e\| *e == s));
30ea9c1e Andy McCall 1 month ago ↶	1530	if let (Some(tip), true) = (commits.last(), old_known) {
	1531	self.record_git_state(git_dir, &state_ref, &tip.sha)?;
	1532	}
	1533	// The working copy follows when it was clean on the old heads.
	1534	if old_heads.is_empty() {
	1535	self.checkout(&heads, true, &format!("{command} (checkout)"))?;
	1536	} else {
	1537	let snap = self.snapshot(command)?;
	1538	let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
	1539	let mut base = snap.base.maximal(&snap.wc.deps);
	1540	base.sort();
	1541	let mut sorted_old = old_heads.clone();
	1542	sorted_old.sort();
9935d490 Andy McCall 29 days ago ↶	1543	// An empty base too: an import that stopped after a checkpoint
	1544	// never checked anything out, and resuming it should.
	1545	if clean && (base == sorted_old \|\| base.is_empty()) {
30ea9c1e Andy McCall 1 month ago ↶	1546	self.checkout(&heads, false, &format!("{command} (checkout)"))?;
	1547	}
	1548	}
c3069537 Andy McCall 1 month ago ↶	1549	Ok(report)
	1550	}
	1551
30ea9c1e Andy McCall 1 month ago ↶	1552	/// Add to `st` the changes of every Git commit `rev-list args` lists:
	1553	/// an imported commit's change, or the whole state an exported commit
	1554	/// was made from (which may hold changes Git never saw).
75167672 Andy McCall 1 month ago	1555	fn add_git_ancestors(
958fb27d Andy McCall 1 month ago ↶	1556	&self,
	1557	st: &mut State,
	1558	git_dir: &Path,
	1559	args: &[&str],
	1560	known: &GitKnown,
	1561	) -> Result<()> {
	1562	self.add_git_ancestors_touching(st, git_dir, args, known, &mut HashSet::new())
	1563	}
	1564
	1565	/// `add_git_ancestors`, also collecting in `touched` every node the
	1566	/// changes it adds edit, place or remove.
	1567	fn add_git_ancestors_touching(
75167672 Andy McCall 1 month ago	1568	&self,
	1569	st: &mut State,
	1570	git_dir: &Path,
	1571	args: &[&str],
	1572	known: &GitKnown,
958fb27d Andy McCall 1 month ago ↶	1573	touched: &mut HashSet<NodeId>,
75167672 Andy McCall 1 month ago	1574	) -> Result<()> {
30ea9c1e Andy McCall 1 month ago ↶	1575	let mut full = vec!["rev-list"];
	1576	full.extend(args);
	1577	let out = git(git_dir, &full)?;
958fb27d Andy McCall 1 month ago ↶	1578	let mut add = \|st: &mut State, h: &ChangeHash\| -> Result<()> {
958fb27d Andy McCall 1 month ago ↶	1579	if st.contains(h) {
e83354cd Andy McCall 1 month ago ↶	1580	return Ok(());
	1581	}
	1582	if !st.is_history_only() {
	1583	let c = self.store.get_change(h)?;
	1584	touched.extend(c.edits.iter().map(\|e\| e.node()));
	1585	touched.extend(c.tree_ops.iter().map(\|o\| o.node()));
	1586	st.add_unchecked(c);
958fb27d Andy McCall 1 month ago ↶	1587	return Ok(());
e83354cd Andy McCall 1 month ago ↶	1588	}
	1589	// History only: the text of an edit waits unread unless its
	1590	// file's graph is built.
	1591	let (skeleton, texts) = self.store.get_change_lazy(h)?;
	1592	touched.extend(skeleton.edits.iter().map(\|e\| e.node()));
	1593	touched.extend(skeleton.tree_ops.iter().map(\|o\| o.node()));
	1594	let built = skeleton
	1595	.edits
	1596	.iter()
	1597	.any(\|e\| !matches!(e, Edit::Replace { .. }) && st.is_built(&e.node()));
	1598	if built {
	1599	let full = self.store.get_change(h)?;
	1600	st.add_known_mixed(*h, full, skeleton, texts);
	1601	} else {
	1602	st.add_known(*h, skeleton, texts);
958fb27d Andy McCall 1 month ago ↶	1603	}
	1604	Ok(())
	1605	};
c3069537 Andy McCall 1 month ago ↶	1606	for sha in out.lines() {
30ea9c1e Andy McCall 1 month ago ↶	1607	if let Some(h) = known.imported.get(sha) {
958fb27d Andy McCall 1 month ago ↶	1608	add(st, h)?;
30ea9c1e Andy McCall 1 month ago ↶	1609	} else if let Some(state_ref) = known.exported.get(sha) {
	1610	let heads = self.store.get_state(state_ref)?.heads;
	1611	for h in self.negotiate(&[], &heads)? {
958fb27d Andy McCall 1 month ago ↶	1612	add(st, &h)?;
30ea9c1e Andy McCall 1 month ago ↶	1613	}
	1614	}
	1615	}
9935d490 Andy McCall 29 days ago ↶	1616	Ok(())
	1617	}
	1618
54e2e630 Andy McCall 29 days ago ↶	1619	/// Recover an import that stopped before it moved the bookmark: its
	1620	/// unfinished pack indexed (`Store::adopt_unfinished_packs`), then every
	1621	/// change in the store that carries a `Git-Commit:` trailer, its heads
	1622	/// (those no other such change depends on) made the bookmark's heads,
	1623	/// so `oxy git fetch` carries on from there. Reads every object once.
	1624	/// Returns how many imported changes it found and the heads.
	1625	pub fn recover_git_import(&self, progress: &dyn Fn(&str)) -> Result<(usize, usize)> {
	1626	// A stopped import's objects are in a pack it never indexed.
	1627	// OXY_RECOVER_MIN_AGE (seconds) overrides the five minutes it must
	1628	// have been left alone, for testing.
	1629	let min_age = std::env::var("OXY_RECOVER_MIN_AGE")
	1630	.ok()
	1631	.and_then(\|v\| v.parse().ok())
	1632	.unwrap_or(300);
	1633	self.store
	1634	.adopt_unfinished_packs(std::time::Duration::from_secs(min_age), progress)?;
	1635	let all = self
	1636	.store
	1637	.list()
	1638	.map_err(\|e\| RepoError::Msg(format!("listing the store: {e}")))?;
	1639	let total = all.len();
	1640	let mut found: HashMap<ChangeHash, Vec<ChangeHash>> = HashMap::new();
	1641	for (i, r) in all.iter().enumerate() {
	1642	if i.is_multiple_of(100_000) {
	1643	progress(&format!(
	1644	"{i} of {total} objects read, {} imported changes",
	1645	found.len()
	1646	));
	1647	}
	1648	let Ok(meta) = self.store.get_change_meta(r) else {
	1649	continue;
	1650	};
	1651	if meta.message.lines().any(\|l\| l.starts_with("Git-Commit: ")) {
	1652	found.insert(*r, meta.deps.clone());
	1653	}
	1654	}
	1655	if found.is_empty() {
	1656	return fail("no imported Git commit's change in this store");
	1657	}
	1658	let depended: HashSet<ChangeHash> = found.values().flatten().copied().collect();
	1659	let mut heads: Vec<ChangeHash> = found
	1660	.keys()
	1661	.filter(\|h\| !depended.contains(h))
	1662	.copied()
	1663	.collect();
	1664	// The bookmark's present heads stay: a recovery only adds.
	1665	heads.extend(self.bookmark_heads().unwrap_or_default());
	1666	let st = self.load_state_history(&heads)?;
	1667	let mut heads = st.maximal(&heads);
	1668	heads.sort();
	1669	heads.dedup();
	1670	let (op_ref, op) = self.head_op()?;
	1671	let mut view = op.view.clone();
	1672	let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?;
	1673	view.bookmarks
	1674	.insert(self.config.bookmark.clone(), state_ref);
	1675	self.record_op(&[op_ref], "oxy git recover", view)?;
	1676	Ok((found.len(), heads.len()))
	1677	}
	1678
9935d490 Andy McCall 29 days ago ↶	1679	/// Point the bookmark at an import's heads so far (see
	1680	/// `CHECKPOINT_COMMITS`); the working copy is left alone until the end.
	1681	fn checkpoint_import(&self, heads: &BTreeSet<ChangeHash>, done: usize) -> Result<()> {
	1682	let heads: Vec<ChangeHash> = heads.iter().copied().collect();
	1683	let (op_ref, op) = self.head_op()?;
	1684	let mut view = op.view.clone();
	1685	let state_ref = self.store.put_state(&StateObject::new(heads))?;
	1686	view.bookmarks
	1687	.insert(self.config.bookmark.clone(), state_ref);
	1688	self.record_op(
	1689	&[op_ref],
	1690	&format!("oxy git import (checkpoint after {done} commits)"),
	1691	view,
	1692	)?;
30ea9c1e Andy McCall 1 month ago ↶	1693	Ok(())
	1694	}
	1695
	1696	/// Commits this repository can relate to Oxygen history: imported
	1697	/// ones by change, exported ones (through `git_dir`'s marks) by state.
	1698	fn git_known(&self, git_dir: &Path) -> Result<GitKnown> {
	1699	let imported = self.imported_commits()?;
	1700	let marks_dir = self.git_marks_dir(git_dir)?;
	1701	let oxy_marks = Self::read_oxy_marks(&marks_dir.join("oxy-marks"))?;
	1702	let mut sha_of_mark: HashMap<u64, String> = HashMap::new();
	1703	if let Ok(text) = fs::read_to_string(marks_dir.join("git-marks")) {
	1704	for line in text.lines() {
	1705	if let Some((m, sha)) = line.split_once(' ') {
	1706	if let Ok(m) = m.trim_start_matches(':').parse::<u64>() {
	1707	sha_of_mark.insert(m, sha.trim().to_string());
	1708	}
	1709	}
	1710	}
	1711	}
	1712	let mut exported = HashMap::new();
	1713	for (state_ref, m) in oxy_marks {
	1714	if let Some(sha) = sha_of_mark.get(&m) {
	1715	exported.insert(sha.clone(), state_ref);
c3069537 Andy McCall 1 month ago ↶	1716	}
c3069537 Andy McCall 1 month ago ↶	1717	}
30ea9c1e Andy McCall 1 month ago ↶	1718	for (state_ref, sha) in Self::read_git_states(&marks_dir.join("git-states"))? {
	1719	exported.entry(sha).or_insert(state_ref);
	1720	}
	1721	Ok(GitKnown { imported, exported })
c3069537 Andy McCall 1 month ago ↶	1722	}
c3069537 Andy McCall 1 month ago ↶	1723
79ab8528 Andy McCall 1 month ago ↶	1724	fn git_author(&self, c: &Change) -> String {
79ab8528 Andy McCall 1 month ago ↶	1725	match self.store.get_identity(&c.author) {
75167672 Andy McCall 1 month ago	1726	Ok(id) if !id.name.is_empty() => format!(
	1727	"{} <{}>",
	1728	id.name.replace(['<', '>', '\n'], " "),
	1729	id.email.replace(['<', '>', '\n', ' '], "")
	1730	),
79ab8528 Andy McCall 1 month ago ↶	1731	_ => "oxy <oxy@localhost>".to_string(),
	1732	}
	1733	}
	1734
	1735	#[allow(clippy::too_many_arguments)]
75167672 Andy McCall 1 month ago	1736	fn write_commit(
	1737	&self,
	1738	out: &mut Vec<u8>,
	1739	reference: &str,
	1740	mark: u64,
	1741	author: &str,
	1742	secs: u64,
	1743	message: &str,
	1744	parents: &[String],
	1745	snapshot: &Snapshot,
	1746	) -> Result<()> {
79ab8528 Andy McCall 1 month ago ↶	1747	out.extend_from_slice(format!("commit {reference}\nmark :{mark}\nauthor {author} {secs} +0000\ncommitter {author} {secs} +0000\ndata {}\n{message}\n", message.len()).as_bytes());
79ab8528 Andy McCall 1 month ago ↶	1748	for (i, p) in parents.iter().enumerate() {
75167672 Andy McCall 1 month ago	1749	out.extend_from_slice(
	1750	format!("{} {p}\n", if i == 0 { "from" } else { "merge" }).as_bytes(),
	1751	);
79ab8528 Andy McCall 1 month ago ↶	1752	}
79ab8528 Andy McCall 1 month ago ↶	1753	out.extend_from_slice(b"deleteall\n");
4dee277c Andy McCall 1 month ago ↶	1754	let has_gitmodules = snapshot.files.contains_key(".gitmodules");
79ab8528 Andy McCall 1 month ago ↶	1755	for (path, f) in &snapshot.files {
4dee277c Andy McCall 1 month ago ↶	1756	if path == ".oxymodules" {
	1757	// Git reads `.gitmodules`; derive it unless the tree has one.
	1758	if !has_gitmodules {
	1759	if let FileContent::Text { bytes, .. } = &f.content {
75167672 Andy McCall 1 month ago	1760	let gm = crate::subrepo::Modules::parse(&String::from_utf8_lossy(bytes))
75167672 Andy McCall 1 month ago	1761	.to_gitmodules();
4dee277c Andy McCall 1 month ago ↶	1762	if !gm.is_empty() {
75167672 Andy McCall 1 month ago	1763	out.extend_from_slice(
	1764	format!("M 100644 inline .gitmodules\ndata {}\n", gm.len())
	1765	.as_bytes(),
	1766	);
4dee277c Andy McCall 1 month ago ↶	1767	out.extend_from_slice(gm.as_bytes());
	1768	out.push(b'\n');
	1769	}
	1770	}
	1771	}
	1772	continue;
	1773	}
	1774	if f.kind == NodeKind::Subrepo {
	1775	if let FileContent::Opaque { blobs } = &f.content {
	1776	if let Some(b) = blobs.first() {
	1777	match crate::subrepo::Pin::decode(&self.read_blob(b)?)? {
75167672 Andy McCall 1 month ago	1778	crate::subrepo::Pin::Git { sha, .. } => {
	1779	out.extend_from_slice(format!("M 160000 {sha} {path}\n").as_bytes())
	1780	}
	1781	crate::subrepo::Pin::Oxy { .. } => self
	1782	.git_export_skipped
	1783	.set(self.git_export_skipped.get() + 1),
4dee277c Andy McCall 1 month ago ↶	1784	}
	1785	}
	1786	}
	1787	continue;
	1788	}
79ab8528 Andy McCall 1 month ago ↶	1789	let (mode, bytes): (&str, Vec<u8>) = match (&f.kind, &f.content) {
79ab8528 Andy McCall 1 month ago ↶	1790	(NodeKind::Dir, _) => continue,
75167672 Andy McCall 1 month ago	1791	(NodeKind::Symlink, FileContent::Opaque { blobs }) => (
	1792	"120000",
	1793	blobs
	1794	.first()
	1795	.map(\|b\| self.read_blob(b))
	1796	.transpose()?
	1797	.unwrap_or_default(),
	1798	),
	1799	(_, FileContent::Text { bytes, .. }) => {
	1800	(if f.mode.exec { "100755" } else { "100644" }, bytes.clone())
	1801	}
	1802	(_, FileContent::Opaque { blobs }) => (
	1803	if f.mode.exec { "100755" } else { "100644" },
	1804	blobs
	1805	.first()
	1806	.map(\|b\| self.read_blob(b))
	1807	.transpose()?
	1808	.unwrap_or_default(),
	1809	),
79ab8528 Andy McCall 1 month ago ↶	1810	(_, FileContent::Dir) => continue,
79ab8528 Andy McCall 1 month ago ↶	1811	};
75167672 Andy McCall 1 month ago	1812	out.extend_from_slice(
	1813	format!("M {mode} inline {path}\ndata {}\n", bytes.len()).as_bytes(),
	1814	);
79ab8528 Andy McCall 1 month ago ↶	1815	out.extend_from_slice(&bytes);
	1816	out.push(b'\n');
	1817	}
	1818	out.push(b'\n');
	1819	Ok(())
	1820	}
	1821	}
62794ecc Andy McCall 1 month ago ↶	1822
	1823	/// What `verify_git_import` found (spec §23: import must reproduce every tree).
	1824	#[derive(Debug, Clone, Default, PartialEq, Eq)]
	1825	pub struct GitVerifyReport {
	1826	pub commits: usize,
7eac74ab Andy McCall 1 month ago ↶	1827	/// Commits followed but not rebuilt (all but the last few, when asked).
7eac74ab Andy McCall 1 month ago ↶	1828	pub skipped: usize,
62794ecc Andy McCall 1 month ago ↶	1829	pub files_compared: usize,
	1830	/// `(commit, what differs)`; empty when every commit's tree is reproduced.
	1831	pub mismatches: Vec<(String, String)>,
f0406c3b Andy McCall 27 days ago ↶	1832	/// Commits of the branch after the last imported one: an import not
	1833	/// finished yet. Not checked.
	1834	pub not_yet: usize,
62794ecc Andy McCall 1 month ago ↶	1835	}
	1836
	1837	impl Repo {
	1838	/// Check an import against its Git source: for every commit of
	1839	/// `branch`, the state formed by the changes imported for it and all
	1840	/// its ancestors must materialize to exactly that commit's tree, byte
	1841	/// for byte, with the same executable bits, symlink targets and
	1842	/// submodule pins. Reads only; stops listing after 50 differences.
	1843	pub fn verify_git_import(&self, git_dir: &Path, branch: &str) -> Result<GitVerifyReport> {
7eac74ab Andy McCall 1 month ago ↶	1844	self.verify_git_import_last(git_dir, branch, None)
	1845	}
	1846
	1847	/// [`Repo::verify_git_import`] of the last `last` commits only (all
	1848	/// when `None`): earlier ones are only followed, not rebuilt, for
	1849	/// looking again at where an import went wrong.
	1850	pub fn verify_git_import_last(
	1851	&self,
	1852	git_dir: &Path,
	1853	branch: &str,
	1854	last: Option<usize>,
	1855	) -> Result<GitVerifyReport> {
f0406c3b Andy McCall 27 days ago ↶	1856	let mut rep = GitVerifyReport::default();
f0406c3b Andy McCall 27 days ago ↶	1857	let mut commits = list_commits(git_dir, branch)?;
62794ecc Andy McCall 1 month ago ↶	1858	let imported = self.imported_commits()?;
f0406c3b Andy McCall 27 days ago ↶	1859	// An import still going (or stopped) has reached part of the
	1860	// branch: what follows its last imported commit is not checked,
	1861	// only counted, and --last counts back from there (oxygen#81).
	1862	if let Some(end) = commits.iter().rposition(\|c\| imported.contains_key(&c.sha)) {
	1863	let rest = commits.len() - end - 1;
	1864	if rest > 0 {
	1865	commits.truncate(end + 1);
	1866	}
	1867	rep.not_yet = rest;
	1868	}
62794ecc Andy McCall 1 month ago ↶	1869	let mut cat = CatFile::start(git_dir)?;
	1870	// Changes of each commit and all its ancestors, built up in topological order.
	1871	let mut closure: HashMap<String, Arc<BTreeSet<ChangeHash>>> = HashMap::new();
a87c7b9d Andy McCall 1 month ago ↶	1872	// Graphs (by key) and blobs already found equal to a Git blob.
	1873	let mut verified_graphs: HashMap<[u8; 32], String> = HashMap::new();
	1874	let mut verified_blobs: HashMap<ObjectRef, String> = HashMap::new();
57d8dc05 Andy McCall 1 month ago ↶	1875	// The state of a commit whose first-parent children are still to
	1876	// come, so a child adds its own changes instead of loading all its
	1877	// history again (as the import does).
	1878	let mut waiting: HashMap<&str, usize> = HashMap::new();
ee6aa3e8 Andy McCall 28 days ago ↶	1879	// Children still to come per commit, through any parent: a commit's
	1880	// closure is dropped after its last child, so only the open
	1881	// frontier is held. Kept for every commit, the closures grew with
	1882	// the square of the history and VS Code's verify ran out of memory
	1883	// (oxygen#81).
	1884	let mut children: HashMap<&str, usize> = HashMap::new();
62794ecc Andy McCall 1 month ago ↶	1885	for c in &commits {
57d8dc05 Andy McCall 1 month ago ↶	1886	if let Some(p1) = c.parents.first() {
	1887	*waiting.entry(p1.as_str()).or_default() += 1;
	1888	}
ee6aa3e8 Andy McCall 28 days ago ↶	1889	for p in &c.parents {
	1890	*children.entry(p.as_str()).or_default() += 1;
	1891	}
57d8dc05 Andy McCall 1 month ago ↶	1892	}
	1893	let mut kept: HashMap<String, State> = HashMap::new();
	1894	let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
	1895	for c in &commits {
62794ecc Andy McCall 1 month ago ↶	1896	rep.commits += 1;
ee6aa3e8 Andy McCall 28 days ago ↶	1897	// The first parent's closure, taken over when this is its last
ee6aa3e8 Andy McCall 28 days ago ↶	1898	// child (no copy), then the other parents' added.
62794ecc Andy McCall 1 month ago ↶	1899	let mut set: BTreeSet<ChangeHash> = BTreeSet::new();
ee6aa3e8 Andy McCall 28 days ago ↶	1900	for (i, p) in c.parents.iter().enumerate() {
	1901	let left = children.get_mut(p.as_str()).map(\|n\| {
	1902	*n -= 1;
	1903	*n
	1904	});
	1905	let ps = if left == Some(0) {
	1906	closure.remove(p)
	1907	} else {
	1908	closure.get(p).cloned()
	1909	};
	1910	if let Some(ps) = ps {
	1911	if i == 0 && set.is_empty() {
	1912	set = Arc::try_unwrap(ps).unwrap_or_else(\|a\| (*a).clone());
	1913	} else {
	1914	set.extend(ps.iter().copied());
	1915	}
62794ecc Andy McCall 1 month ago ↶	1916	}
	1917	}
	1918	match imported.get(&c.sha) {
	1919	Some(h) => {
	1920	set.insert(*h);
	1921	}
	1922	None => {
	1923	rep.mismatches.push((
	1924	c.sha.clone(),
	1925	"no imported change carries this commit".into(),
	1926	));
	1927	}
	1928	}
	1929	let set = Arc::new(set);
ee6aa3e8 Andy McCall 28 days ago ↶	1930	if children.get(c.sha.as_str()).is_some_and(\|n\| *n > 0) {
	1931	closure.insert(c.sha.clone(), set.clone());
	1932	}
62794ecc Andy McCall 1 month ago ↶	1933	if rep.mismatches.len() >= 50 {
62794ecc Andy McCall 1 month ago ↶	1934	break;
7eac74ab Andy McCall 1 month ago ↶	1935	}
	1936	if last.is_some_and(\|n\| rep.commits + n <= commits.len()) {
	1937	rep.skipped += 1;
	1938	continue;
62794ecc Andy McCall 1 month ago ↶	1939	}
62794ecc Andy McCall 1 month ago ↶	1940	let heads: Vec<ChangeHash> = set.iter().copied().collect();
a87c7b9d Andy McCall 1 month ago ↶	1941	// History only: a file whose graph (by key) or blob was already
	1942	// found equal to this Git blob is not built, rendered or read
	1943	// again; the rest are.
57d8dc05 Andy McCall 1 month ago ↶	1944	let from_parent = c.parents.first().and_then(\|p1\| {
	1945	let left = waiting.get_mut(p1.as_str()).map(\|n\| {
	1946	*n -= 1;
	1947	*n
	1948	});
	1949	if left == Some(0) {
	1950	kept_order.retain(\|k\| k != p1);
	1951	kept.remove(p1.as_str())
	1952	} else {
	1953	kept.get(p1.as_str()).cloned()
	1954	}
	1955	});
	1956	let state = match from_parent {
	1957	Some(mut st) => {
	1958	// The changes the parent's state lacks. The state never
	1959	// builds a graph (they are built beside it), so they
	1960	// are read without their text.
	1961	for h in set.iter() {
	1962	if !st.contains(h) {
	1963	let (change, texts) = self.store.get_change_lazy(h)?;
	1964	st.add_known(*h, change, texts);
	1965	}
	1966	}
	1967	st
	1968	}
	1969	None => self.load_state_history(&heads)?,
	1970	};
a87c7b9d Andy McCall 1 month ago ↶	1971	let view = state.tree().view();
62794ecc Andy McCall 1 month ago ↶	1972	let tree = list_tree(git_dir, &c.sha)?;
	1973	let short = &c.sha[..12.min(c.sha.len())];
	1974	let mut differ = \|what: String\| rep.mismatches.push((c.sha.clone(), what));
a87c7b9d Andy McCall 1 month ago ↶	1975	let mut conflicted: Vec<String> =
	1976	view.conflicts.iter().map(\|c\| format!("{c:?}")).collect();
	1977	let mut need: Vec<NodeId> = Vec::new();
	1978	for (path, (mode, sha)) in &tree {
	1979	let Some(e) = view.entries.get(path) else {
	1980	continue;
	1981	};
	1982	if mode != "160000"
	1983	&& mode != "120000"
	1984	&& e.kind == NodeKind::File
	1985	&& !e.mode.opaque
	1986	{
	1987	let known = state
	1988	.graph_key(&e.node)
	1989	.is_some_and(\|k\| verified_graphs.get(&k) == Some(sha));
	1990	if !known {
	1991	need.push(e.node);
62794ecc Andy McCall 1 month ago ↶	1992	}
	1993	}
	1994	}
a87c7b9d Andy McCall 1 month ago ↶	1995	// A graph's key is known only before it is built.
	1996	let keys: HashMap<NodeId, [u8; 32]> = need
	1997	.iter()
	1998	.filter_map(\|n\| Some((*n, state.graph_key(n)?)))
	1999	.collect();
	2000	let need_set: HashSet<NodeId> = need.iter().copied().collect();
57d8dc05 Andy McCall 1 month ago ↶	2001	let graphs = self.build_graphs_detached(&state, &need)?;
62794ecc Andy McCall 1 month ago ↶	2002	for (path, (mode, sha)) in &tree {
a87c7b9d Andy McCall 1 month ago ↶	2003	let Some(e) = view.entries.get(path) else {
62794ecc Andy McCall 1 month ago ↶	2004	differ(format!("{short}: {path} is missing"));
	2005	continue;
	2006	};
	2007	rep.files_compared += 1;
	2008	match mode.as_str() {
a87c7b9d Andy McCall 1 month ago ↶	2009	"160000" => {
	2010	let blobs = state.live_blobs(&e.node);
	2011	match (e.kind, blobs.as_slice()) {
	2012	(NodeKind::Subrepo, [b]) => {
	2013	let pin = crate::subrepo::Pin::decode(&self.read_blob(b)?)?;
	2014	if !matches!(&pin, crate::subrepo::Pin::Git { sha: s, .. } if s == sha)
	2015	{
	2016	differ(format!("{short}: {path} pins another commit"));
	2017	}
62794ecc Andy McCall 1 month ago ↶	2018	}
a87c7b9d Andy McCall 1 month ago ↶	2019	(kind, _) => differ(format!(
	2020	"{short}: {path} is not one submodule pin: {kind:?} with {} pins",
	2021	blobs.len()
	2022	)),
62794ecc Andy McCall 1 month ago ↶	2023	}
a87c7b9d Andy McCall 1 month ago ↶	2024	}
62794ecc Andy McCall 1 month ago ↶	2025	"120000" => {
62794ecc Andy McCall 1 month ago ↶	2026	let target = cat.read(sha)?;
a87c7b9d Andy McCall 1 month ago ↶	2027	let blobs = state.live_blobs(&e.node);
	2028	match (e.kind, blobs.as_slice()) {
	2029	(NodeKind::Symlink, [b]) if self.read_blob(b)? == target => {}
62794ecc Andy McCall 1 month ago ↶	2030	_ => differ(format!("{short}: symlink {path} differs")),
	2031	}
	2032	}
	2033	_ => {
a87c7b9d Andy McCall 1 month ago ↶	2034	if e.mode.exec != (mode == "100755") {
	2035	differ(format!("{short}: {path} executable bit differs"));
	2036	}
	2037	if e.kind == NodeKind::File && !e.mode.opaque {
	2038	if !need_set.contains(&e.node) {
	2039	// Known equal: skipped when the graphs were chosen.
62794ecc Andy McCall 1 month ago ↶	2040	continue;
62794ecc Andy McCall 1 month ago ↶	2041	}
57d8dc05 Andy McCall 1 month ago ↶	2042	let (have, m) = match graphs.get(&e.node) {
	2043	Some(g) => state.render_graph(g),
	2044	None => (Vec::new(), Default::default()),
	2045	};
a87c7b9d Andy McCall 1 month ago ↶	2046	if m.is_conflicted() {
57d8dc05 Andy McCall 1 month ago ↶	2047	// Who conflicts: a copy of the state with this graph built.
	2048	let mut detail = state.clone();
	2049	self.load_graphs_of(&mut detail, &[e.node])?;
	2050	let who: Vec<String> = detail
a87c7b9d Andy McCall 1 month ago ↶	2051	.conflict_regions(&e.node)
	2052	.contributors
	2053	.iter()
	2054	.filter_map(\|h\| state.get(h))
	2055	.map(\|m\| {
	2056	let sha = m
	2057	.message
	2058	.lines()
	2059	.rev()
	2060	.find_map(\|l\| l.strip_prefix("Git-Commit: "))
	2061	.unwrap_or("?");
	2062	format!(
	2063	"{} {:?}",
	2064	&sha[..12.min(sha.len())],
	2065	m.message.lines().next().unwrap_or("")
	2066	)
	2067	})
	2068	.collect();
	2069	conflicted.push(format!("{path} between {}", who.join(" and ")));
	2070	}
	2071	let want = cat.read(sha)?;
	2072	if have != want {
	2073	differ(format!(
	2074	"{short}: {path} differs: {}",
	2075	first_difference(&have, &want)
	2076	));
	2077	} else if !m.is_conflicted() {
	2078	if let Some(k) = keys.get(&e.node) {
	2079	verified_graphs.insert(*k, sha.clone());
	2080	}
	2081	}
	2082	continue;
	2083	}
	2084	let blobs = state.live_blobs(&e.node);
	2085	let [b] = blobs.as_slice() else {
	2086	differ(format!("{short}: {path} is not one file"));
	2087	continue;
62794ecc Andy McCall 1 month ago ↶	2088	};
a87c7b9d Andy McCall 1 month ago ↶	2089	if verified_blobs.get(b) == Some(sha) {
a87c7b9d Andy McCall 1 month ago ↶	2090	continue;
62794ecc Andy McCall 1 month ago ↶	2091	}
a87c7b9d Andy McCall 1 month ago ↶	2092	if self.read_blob(b)? != cat.read(sha)? {
	2093	differ(format!("{short}: {path} differs"));
	2094	} else {
	2095	verified_blobs.insert(*b, sha.clone());
62794ecc Andy McCall 1 month ago ↶	2096	}
	2097	}
	2098	}
	2099	}
a87c7b9d Andy McCall 1 month ago ↶	2100	if !conflicted.is_empty() {
	2101	differ(format!(
	2102	"{short}: the state is conflicted: {}",
	2103	conflicted.join("; ")
	2104	));
	2105	}
62794ecc Andy McCall 1 month ago ↶	2106	// Import records submodules in `.oxymodules` as well (spec §23),
	2107	// and export leaves it out again.
	2108	let has_submodules = tree.values().any(\|(m, _)\| m == "160000");
e83354cd Andy McCall 1 month ago ↶	2109	for (path, e) in view.entries.iter() {
a87c7b9d Andy McCall 1 month ago ↶	2110	if e.kind != NodeKind::Dir
62794ecc Andy McCall 1 month ago ↶	2111	&& !tree.contains_key(path)
	2112	&& !(has_submodules && path == ".oxymodules")
	2113	{
	2114	differ(format!("{short}: {path} is extra"));
57d8dc05 Andy McCall 1 month ago ↶	2115	}
	2116	}
	2117	if waiting.get(c.sha.as_str()).is_some_and(\|n\| *n > 0) {
	2118	kept.insert(c.sha.clone(), state);
	2119	kept_order.push_back(c.sha.clone());
	2120	if kept_order.len() > 8 {
	2121	if let Some(old) = kept_order.pop_front() {
	2122	kept.remove(&old);
	2123	}
62794ecc Andy McCall 1 month ago ↶	2124	}
	2125	}
	2126	if rep.mismatches.len() >= 50 {
	2127	break;
	2128	}
	2129	}
	2130	Ok(rep)
	2131	}
	2132	}
	2133
	2134	/// Where two versions of a file first part, for a verification report.
	2135	fn first_difference(have: &[u8], want: &[u8]) -> String {
	2136	let (h, w): (Vec<&[u8]>, Vec<&[u8]>) = (
	2137	have.split(\|b\| *b == b'\n').collect(),
	2138	want.split(\|b\| *b == b'\n').collect(),
	2139	);
	2140	let i = h
	2141	.iter()
	2142	.zip(&w)
	2143	.position(\|(a, b)\| a != b)
	2144	.unwrap_or(h.len().min(w.len()));
	2145	let show = \|l: Option<&&[u8]>\| {
	2146	l.map(\|l\| String::from_utf8_lossy(&l[..l.len().min(60)]).into_owned())
	2147	.unwrap_or_else(\|\| "(end)".into())
	2148	};
	2149	format!(
	2150	"line {}: has {:?}, Git has {:?} ({} lines, Git has {})",
	2151	i + 1,
	2152	show(h.get(i)),
	2153	show(w.get(i)),
	2154	h.len(),
	2155	w.len()
	2156	)
	2157	}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

Keyboard