crates/oxy-core/src/repo/git.rs at v0.4.3 · andymccall/oxygen

◆ v0.4.3 oxygen / crates / oxy-core / src / repo / git.rs

90.9 KiB · Rust · last changed git import and fetch --claim: author your own commits with your key 25 days ago source blame history raw

79ab8528 Andy McCall 1 month ago ↶	1	//! Git bridge (spec §23), export side: each state the current bookmark
	2	//! passed through becomes a Git commit through `git fast-import`, with the
	3	//! state's full tree and the previous state's commit as parent. A state
	4	//! that added exactly one change carries that change's message, author,
	5	//! and `Change-Id`/`Oxy-Hash` trailers; every commit carries `Oxy-State`.
	6	//!
	7	//! Marks make re-exports incremental: `.oxy/git/<target>/git-marks` is
	8	//! git's own marks file and `oxy-marks` maps state refs to marks.
	9
	10	use super::*;
	11	use std::process::{Command, Stdio};
62794ecc Andy McCall 1 month ago ↶	12	use std::sync::Arc;
79ab8528 Andy McCall 1 month ago ↶	13
79ab8528 Andy McCall 1 month ago ↶	14	#[derive(Debug, Clone, Default, PartialEq, Eq)]
c3069537 Andy McCall 1 month ago ↶	15	pub struct GitImportReport {
	16	pub commits: usize,
	17	pub merges: usize,
4dee277c Andy McCall 1 month ago ↶	18	/// Gitlink entries imported as subrepo nodes with Git pins.
4dee277c Andy McCall 1 month ago ↶	19	pub submodules: usize,
c3069537 Andy McCall 1 month ago ↶	20	pub heads: usize,
	21	}
	22
	23	/// A running `git cat-file --batch`, for reading many blobs cheaply.
	24	struct CatFile {
	25	child: std::process::Child,
	26	stdin: std::process::ChildStdin,
	27	stdout: std::io::BufReader<std::process::ChildStdout>,
	28	}
	29
	30	impl CatFile {
	31	fn start(git_dir: &Path) -> Result<CatFile> {
	32	let mut child = Command::new("git")
	33	.arg("-C")
	34	.arg(git_dir)
	35	.args(["cat-file", "--batch"])
	36	.stdin(Stdio::piped())
	37	.stdout(Stdio::piped())
	38	.stderr(Stdio::null())
	39	.spawn()
	40	.map_err(\|e\| RepoError::Msg(format!("running git cat-file: {e}")))?;
	41	let stdin = child.stdin.take().unwrap();
	42	let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
75167672 Andy McCall 1 month ago	43	Ok(CatFile {
	44	child,
	45	stdin,
	46	stdout,
	47	})
c3069537 Andy McCall 1 month ago ↶	48	}
	49
	50	fn read(&mut self, sha: &str) -> Result<Vec<u8>> {
	51	use std::io::{BufRead, Read};
	52	writeln!(self.stdin, "{sha}")?;
	53	self.stdin.flush()?;
	54	let mut header = String::new();
	55	self.stdout.read_line(&mut header)?;
	56	let parts: Vec<&str> = header.trim_end().split(' ').collect();
	57	if parts.len() != 3 {
	58	return fail(format!("git cat-file: object {sha} not found"));
	59	}
75167672 Andy McCall 1 month ago	60	let size: usize = parts[2]
	61	.parse()
	62	.map_err(\|_\| RepoError::Msg("git cat-file: bad size".into()))?;
c3069537 Andy McCall 1 month ago ↶	63	let mut body = vec![0u8; size];
	64	self.stdout.read_exact(&mut body)?;
	65	let mut nl = [0u8; 1];
	66	self.stdout.read_exact(&mut nl)?;
	67	Ok(body)
	68	}
	69	}
	70
	71	impl Drop for CatFile {
	72	fn drop(&mut self) {
	73	let _ = self.child.kill();
	74	let _ = self.child.wait();
	75	}
	76	}
	77
2306a4d2 Andy McCall 1 month ago ↶	78	/// One path a commit changes against its first parent: its new mode and
	79	/// blob, or `None` when the commit deletes it.
	80	type TreeDiff = Vec<(String, Option<(String, String)>)>;
	81
	82	/// A running `git diff-tree --stdin`, fed every `commit parent` pair up
	83	/// front, whose diffs are read in the same order as they are needed: a
	84	/// commit's tree is its first parent's plus its diff, at a cost that
	85	/// follows what it changes rather than the whole tree (oxygen#25).
	86	struct DiffTree {
	87	child: std::process::Child,
	88	stdout: std::io::BufReader<std::process::ChildStdout>,
	89	feeder: Option<std::thread::JoinHandle<()>>,
	90	/// The header read at the end of the previous diff.
	91	next: Option<String>,
	92	}
	93
	94	impl DiffTree {
	95	fn start(git_dir: &Path, pairs: Vec<(String, String)>) -> Result<DiffTree> {
	96	let mut child = Command::new("git")
	97	.arg("-C")
	98	.arg(git_dir)
	99	.args([
	100	"diff-tree",
	101	"--stdin",
	102	"--always",
	103	"-r",
	104	"-z",
	105	"--no-renames",
	106	])
	107	.stdin(Stdio::piped())
	108	.stdout(Stdio::piped())
	109	.stderr(Stdio::null())
	110	.spawn()
	111	.map_err(\|e\| RepoError::Msg(format!("running git diff-tree: {e}")))?;
	112	let mut stdin = child.stdin.take().unwrap();
	113	let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
	114	// A thread writes, so git never waits on a full pipe for us to read.
	115	let feeder = std::thread::spawn(move \|\| {
f0406c3b Andy McCall 27 days ago ↶	116	// An import that stops before reading every diff kills git
	117	// while this thread may still be writing. oxy restores
	118	// SIGPIPE's default (to end quietly under `\| head`), so that
	119	// write would kill the whole process (signal 13, oxygen#81);
	120	// blocked here, it fails with EPIPE and the thread ends.
	121	#[cfg(unix)]
	122	unsafe {
	123	let mut set: libc::sigset_t = std::mem::zeroed();
	124	libc::sigemptyset(&mut set);
	125	libc::sigaddset(&mut set, libc::SIGPIPE);
	126	libc::pthread_sigmask(libc::SIG_BLOCK, &set, std::ptr::null_mut());
	127	}
2306a4d2 Andy McCall 1 month ago ↶	128	let mut w = std::io::BufWriter::new(&mut stdin);
	129	for (c, p) in pairs {
	130	if writeln!(w, "{c} {p}").is_err() {
	131	return;
	132	}
	133	}
	134	let _ = w.flush();
	135	});
	136	Ok(DiffTree {
	137	child,
	138	stdout,
	139	feeder: Some(feeder),
	140	next: None,
	141	})
	142	}
	143
	144	fn field(&mut self) -> Result<Option<String>> {
	145	use std::io::BufRead;
	146	let mut buf = Vec::new();
	147	if self.stdout.read_until(0, &mut buf)? == 0 {
	148	return Ok(None);
	149	}
	150	if buf.last() == Some(&0) {
	151	buf.pop();
	152	}
	153	Ok(Some(String::from_utf8_lossy(&buf).into_owned()))
	154	}
	155
	156	/// The diff of `sha` against the parent it was fed with.
	157	fn diff(&mut self, sha: &str) -> Result<TreeDiff> {
	158	let header = match self.next.take() {
	159	Some(h) => Some(h),
	160	None => self.field()?,
	161	};
	162	if header.as_deref() != Some(sha) {
	163	return fail(format!("git diff-tree: expected {sha}, got {header:?}"));
	164	}
	165	let mut out = Vec::new();
	166	while let Some(f) = self.field()? {
	167	let Some(meta) = f.strip_prefix(':') else {
	168	self.next = Some(f);
	169	break;
	170	};
	171	let Some(path) = self.field()? else {
	172	return fail("git diff-tree: record without a path");
	173	};
	174	// `oldmode newmode oldsha newsha status`
	175	let m: Vec<&str> = meta.split(' ').collect();
	176	if m.len() != 5 {
	177	return fail(format!("git diff-tree: bad record {f:?}"));
	178	}
	179	let entry = if m[4].starts_with('D') {
	180	None
	181	} else {
	182	Some((m[1].to_string(), m[3].to_string()))
	183	};
	184	out.push((path, entry));
	185	}
	186	Ok(out)
	187	}
	188	}
	189
	190	impl Drop for DiffTree {
	191	fn drop(&mut self) {
192	let _ = self.child.kill();
193	let _ = self.child.wait();
194	if let Some(f) = self.feeder.take() {
195	let _ = f.join();
196	}
197	}
198	}
199
d6e0d5b8 Andy McCall 25 days ago ↶	200	/// The email of a Git author `Name <email>`, lowercased.
	201	fn git_email(author: &str) -> Option<String> {
	202	let (_, rest) = author.rsplit_once('<')?;
	203	let email = rest.strip_suffix('>')?.trim();
	204	(!email.is_empty()).then(\|\| email.to_lowercase())
	205	}
	206
c3069537 Andy McCall 1 month ago ↶	207	/// One commit as `git log` describes it.
	208	struct GitCommit {
	209	sha: String,
	210	parents: Vec<String>,
	211	author_secs: u64,
	212	author: String,
	213	message: String,
	214	}
	215
	216	fn list_commits(git_dir: &Path, branch: &str) -> Result<Vec<GitCommit>> {
	217	let out = Command::new("git")
	218	.arg("-C")
	219	.arg(git_dir)
75167672 Andy McCall 1 month ago	220	.args([
	221	"log",
	222	"--reverse",
	223	"--topo-order",
	224	"--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e",
	225	branch,
	226	])
c3069537 Andy McCall 1 month ago ↶	227	.output()
	228	.map_err(\|e\| RepoError::Msg(format!("running git log: {e}")))?;
	229	if !out.status.success() {
75167672 Andy McCall 1 month ago	230	return fail(format!(
	231	"git log {branch} failed: {}",
	232	String::from_utf8_lossy(&out.stderr).trim()
	233	));
c3069537 Andy McCall 1 month ago ↶	234	}
	235	let text = String::from_utf8_lossy(&out.stdout);
	236	let mut commits = Vec::new();
	237	for record in text.split('\x1e') {
	238	let record = record.trim_start_matches('\n');
	239	if record.trim().is_empty() {
	240	continue;
	241	}
	242	let f: Vec<&str> = record.splitn(5, '\0').collect();
	243	if f.len() != 5 {
	244	return fail("unexpected git log record");
	245	}
	246	commits.push(GitCommit {
	247	sha: f[0].to_string(),
	248	parents: f[1].split_whitespace().map(str::to_string).collect(),
	249	author_secs: f[2].parse().unwrap_or(0),
	250	author: f[3].to_string(),
	251	message: f[4].trim_end().to_string(),
	252	});
	253	}
	254	Ok(commits)
	255	}
	256
	257	/// `path → (mode, blob sha)` for a commit's tree.
	258	fn list_tree(git_dir: &Path, sha: &str) -> Result<BTreeMap<String, (String, String)>> {
	259	let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?;
	260	let mut out = BTreeMap::new();
	261	for entry in text.split('\0') {
	262	if entry.is_empty() {
	263	continue;
	264	}
75167672 Andy McCall 1 month ago	265	let Some((meta, path)) = entry.split_once('\t') else {
	266	continue;
	267	};
c3069537 Andy McCall 1 month ago ↶	268	let m: Vec<&str> = meta.split(' ').collect();
	269	if m.len() == 3 {
	270	out.insert(path.to_string(), (m[0].to_string(), m[2].to_string()));
	271	}
	272	}
	273	Ok(out)
	274	}
	275
	276	/// The ChangeId for a Git commit: its `Change-Id` trailer if it has a
	277	/// 32-hex one, else BLAKE3-128 of the commit hash (spec §23).
	278	fn change_id_for(sha: &str, message: &str) -> ChangeId {
	279	for line in message.lines().rev() {
	280	if let Some(v) = line.strip_prefix("Change-Id:") {
	281	if let Some(id) = ChangeId::from_hex(v.trim()) {
	282	return id;
	283	}
	284	}
	285	}
	286	let h = blake3::hash(sha.as_bytes());
	287	ChangeId(h.as_bytes()[..16].try_into().unwrap())
	288	}
	289
30ea9c1e Andy McCall 1 month ago ↶	290	/// Git commits with a known Oxygen counterpart.
	291	struct GitKnown {
	292	/// Imported commit → its change.
	293	imported: HashMap<String, ChangeHash>,
	294	/// Exported commit (or a fetched tip) → the bookmark state it equals.
	295	exported: HashMap<String, ObjectRef>,
	296	}
	297
c3069537 Andy McCall 1 month ago ↶	298	#[derive(Debug, Clone, Default, PartialEq, Eq)]
79ab8528 Andy McCall 1 month ago ↶	299	pub struct GitExportReport {
	300	pub exported: usize,
	301	pub skipped: usize,
4dee277c Andy McCall 1 month ago ↶	302	/// Subrepo nodes with Oxygen pins, which Git cannot represent.
4dee277c Andy McCall 1 month ago ↶	303	pub skipped_subrepos: usize,
79ab8528 Andy McCall 1 month ago ↶	304	pub branch: String,
	305	pub heads: usize,
	306	}
	307
fd280228 Andy McCall 1 month ago ↶	308	/// A merge bringing more commits than this sheds its base's graphs first.
fd280228 Andy McCall 1 month ago ↶	309	const MERGE_SHED_COMMITS: usize = 100;
d45dab04 Andy McCall 1 month ago ↶	310
	311	/// The atoms a kept state's built graphs may hold before they go back into
	312	/// its history (#25): about a few hundred megabytes.
	313	const KEPT_ATOMS: usize = 1_500_000;
fd280228 Andy McCall 1 month ago ↶	314	/// A kept state with more built graphs than this sheds them.
fd280228 Andy McCall 1 month ago ↶	315	const KEPT_GRAPHS: usize = 2000;
9935d490 Andy McCall 29 days ago ↶	316	/// An import moves its bookmark to the commits done so far every this
	317	/// many commits, so one that stops part way resumes from there: imported
	318	/// commits are found from the bookmark's history (`imported_commits`).
	319	const CHECKPOINT_COMMITS: usize = 2000;
fd280228 Andy McCall 1 month ago ↶	320
c3d2ef61 Andy McCall 1 month ago ↶	321	/// Text files of a merge rendered this many at a time (oxygen#25): a
	322	/// batch's graphs are built beside the state and dropped once rendered,
	323	/// so what is held for the whole tree is the renderings, not the graphs
	324	/// with every deleted line of their history.
	325	const MERGE_BATCH: usize = 500;
	326
	327	thread_local! {
	328	/// A smaller batch for this thread's imports: tests set 1, so a small
	329	/// merge takes the batched path.
	330	static MERGE_BATCH_OVERRIDE: std::cell::Cell<Option<usize>> = const { std::cell::Cell::new(None) };
	331	}
	332
	333	#[cfg(test)]
	334	pub(crate) fn set_merge_batch_for_tests(n: Option<usize>) {
	335	MERGE_BATCH_OVERRIDE.with(\|c\| c.set(n));
	336	}
	337
	338	fn merge_batch() -> usize {
	339	MERGE_BATCH_OVERRIDE
	340	.with(\|c\| c.get())
	341	.filter(\|n\| *n > 0)
33be2409 Andy McCall 1 month ago ↶	342	.or_else(\|\| {
	343	std::env::var("OXY_MERGE_BATCH")
	344	.ok()
	345	.and_then(\|v\| v.parse().ok())
	346	})
c3d2ef61 Andy McCall 1 month ago ↶	347	.unwrap_or(MERGE_BATCH)
	348	}
	349
	350	impl Repo {
	351	/// As `State::materialize_except`, for a tree whose graphs would not
	352	/// fit in memory at once. The few files the change builder reads
	353	/// graphs for (conflicted ones, and renderings whose lines are not
	354	/// one per live atom) are built in the state afterwards.
	355	fn materialize_batched(
	356	&self,
	357	base: &mut State,
	358	need: &[crate::ids::NodeId],
	359	skip: &HashSet<crate::ids::NodeId>,
	360	) -> Result<crate::state::Snapshot> {
	361	use crate::graph::Materialized;
	362	use crate::state::{FileContent, FileState};
	363	use rayon::prelude::*;
	364	let view = base.tree().view();
	365	let mut rendered: HashMap<crate::ids::NodeId, (Vec<u8>, Materialized)> =
	366	HashMap::with_capacity(need.len());
	367	let mut keep_built: Vec<crate::ids::NodeId> = Vec::new();
	368	for batch in need.chunks(merge_batch()) {
	369	let t = std::time::Instant::now();
	370	let graphs = self.build_graphs_detached(base, batch)?;
	371	crate::timing::add("render: build graphs, batch", t.elapsed());
	372	let t = std::time::Instant::now();
	373	let batch_rendered: Vec<(crate::ids::NodeId, Vec<u8>, Materialized)> = graphs
	374	.par_iter()
	375	.map(\|(node, g)\| {
	376	let (bytes, m) = base.render_graph(g);
	377	(*node, bytes, m)
	378	})
	379	.collect();
	380	drop(graphs);
	381	crate::timing::add("render: render, batch", t.elapsed());
	382	for (node, bytes, m) in batch_rendered {
	383	if m.is_conflicted()
	384	\|\| m.live_atoms().len() != crate::diff::split_lines(&bytes).len()
	385	{
	386	keep_built.push(node);
	387	}
	388	rendered.insert(node, (bytes, m));
	389	}
	390	}
	391	if !keep_built.is_empty() {
	392	self.load_graphs_of(base, &keep_built)?;
	393	}
	394	let mut files = BTreeMap::new();
	395	let mut unrendered = BTreeSet::new();
	396	for (path, e) in view.entries.iter() {
	397	let f = if e.kind == NodeKind::File && !e.mode.opaque {
	398	match rendered.remove(&e.node) {
	399	Some((bytes, materialized)) => FileState {
	400	node: e.node,
	401	kind: e.kind,
	402	mode: e.mode,
	403	content: FileContent::Text {
	404	bytes,
	405	materialized,
	406	},
	407	},
	408	None if skip.contains(&e.node) => {
	409	unrendered.insert(path.clone());
	410	FileState {
411	node: e.node,
412	kind: e.kind,
413	mode: e.mode,
414	content: FileContent::Text {
415	bytes: Vec::new(),
416	materialized: Materialized::default(),
417	},
418	}
419	}
420	None => {
421	// Neither asked for nor known unchanged: render it
422	// from a built graph, as the whole-tree path does.
423	self.load_graphs_of(base, &[e.node])?;
424	base.entry_state(e)
425	}
426	}
427	} else {
428	base.entry_state(e)
429	};
430	files.insert(path.clone(), f);
431	}
432	Ok(crate::state::Snapshot {
433	tree: view,
434	files,
435	unrendered,
436	})
437	}
438	}
439
79ab8528 Andy McCall 1 month ago ↶	440	fn git(dir: &Path, args: &[&str]) -> Result<String> {
75167672 Andy McCall 1 month ago	441	let out = Command::new("git")
	442	.arg("-C")
	443	.arg(dir)
	444	.args(args)
	445	.output()
	446	.map_err(\|e\| RepoError::Msg(format!("running git: {e}")))?;
79ab8528 Andy McCall 1 month ago ↶	447	if !out.status.success() {
75167672 Andy McCall 1 month ago	448	return fail(format!(
	449	"git {} failed: {}",
	450	args.join(" "),
	451	String::from_utf8_lossy(&out.stderr).trim()
	452	));
79ab8528 Andy McCall 1 month ago ↶	453	}
	454	Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
	455	}
	456
	457	impl Repo {
	458	fn git_marks_dir(&self, git_dir: &Path) -> Result<PathBuf> {
75167672 Andy McCall 1 month ago	459	let canonical = git_dir
	460	.canonicalize()
	461	.unwrap_or_else(\|_\| git_dir.to_path_buf());
79ab8528 Andy McCall 1 month ago ↶	462	let key = hash_object(ObjectType::Chunk, canonical.to_string_lossy().as_bytes()).short(16);
	463	let dir = self.oxy.join("git").join(key);
	464	fs::create_dir_all(&dir)?;
	465	Ok(dir)
	466	}
	467
30ea9c1e Andy McCall 1 month ago ↶	468	/// `state ref → Git commit sha` for bookmark states that equal a Git
	469	/// commit's tree (recorded by import and fetch).
	470	fn read_git_states(path: &Path) -> Result<BTreeMap<ObjectRef, String>> {
	471	let mut out = BTreeMap::new();
	472	if let Ok(text) = fs::read_to_string(path) {
	473	for line in text.lines() {
	474	if let Some((h, sha)) = line.split_once(' ') {
	475	if let Some(h) = ObjectRef::from_hex(h) {
	476	out.insert(h, sha.trim().to_string());
	477	}
	478	}
	479	}
	480	}
	481	Ok(out)
	482	}
	483
	484	fn record_git_state(&self, git_dir: &Path, state: &ObjectRef, sha: &str) -> Result<()> {
	485	let path = self.git_marks_dir(git_dir)?.join("git-states");
	486	let mut states = Self::read_git_states(&path)?;
	487	states.insert(*state, sha.to_string());
75167672 Andy McCall 1 month ago	488	let text: String = states
	489	.iter()
	490	.map(\|(h, s)\| format!("{} {s}\n", h.to_hex()))
	491	.collect();
30ea9c1e Andy McCall 1 month ago ↶	492	write_atomic(&path, text.as_bytes())
	493	}
	494
	495	/// The bare mirror this repository keeps for the Git remote `url`.
	496	fn git_mirror(&self, url: &str) -> Result<PathBuf> {
	497	let key = hash_object(ObjectType::Chunk, url.as_bytes()).short(16);
	498	let dir = self.oxy.join("git").join(key).join("mirror.git");
	499	if !dir.join("HEAD").exists() {
	500	fs::create_dir_all(&dir)?;
	501	git(&dir, &["init", "--bare", "-q"])?;
	502	}
	503	Ok(dir)
	504	}
	505
	506	/// Git commits already imported into the bookmark, by sha.
7891e21c Andy McCall 1 month ago ↶	507	pub(crate) fn imported_commits(&self) -> Result<HashMap<String, ChangeHash>> {
30ea9c1e Andy McCall 1 month ago ↶	508	let mut map = HashMap::new();
	509	for e in self.log()? {
	510	if e.is_working_copy {
	511	continue;
	512	}
75167672 Andy McCall 1 month ago	513	if let Some(sha) = e
	514	.change
	515	.message
	516	.lines()
	517	.rev()
	518	.find_map(\|l\| l.strip_prefix("Git-Commit: "))
	519	{
30ea9c1e Andy McCall 1 month ago ↶	520	map.insert(sha.trim().to_string(), e.hash);
	521	}
	522	}
	523	Ok(map)
	524	}
	525
	526	/// Bring the mirror's `branch` up to date with the Git remote `url`.
	527	/// A remote without that branch yet leaves the mirror as it is.
	528	fn git_mirror_fetch(&self, mirror: &Path, url: &str, branch: &str) -> Result<()> {
	529	let spec = format!("+refs/heads/{branch}:refs/heads/{branch}");
	530	match git(mirror, &["fetch", "-q", url, &spec]) {
	531	Ok(_) => Ok(()),
	532	Err(e) => {
	533	let ls = git(mirror, &["ls-remote", "--heads", url, branch]).unwrap_or_default();
75167672 Andy McCall 1 month ago	534	if ls.trim().is_empty() {
	535	Ok(())
	536	} else {
	537	Err(e)
	538	}
30ea9c1e Andy McCall 1 month ago ↶	539	}
	540	}
	541	}
	542
	543	/// Fetch `branch` of the Git repository at `url` (a path or a Git URL)
	544	/// through a mirror and import the commits not yet in the bookmark
	545	/// (spec §23). Works on a repository that already has history: the
	546	/// new changes join the bookmark like a pull.
	547	pub fn git_fetch(&self, url: &str, branch: &str) -> Result<GitImportReport> {
9935d490 Andy McCall 29 days ago ↶	548	// A local path, made absolute: git runs it from inside the mirror.
	549	let local = Path::new(url)
	550	.canonicalize()
	551	.ok()
	552	.filter(\|_\| !url.contains("://"));
	553	let absolute = local.as_ref().map(\|p\| p.to_string_lossy().into_owned());
	554	let url = absolute.as_deref().unwrap_or(url);
30ea9c1e Andy McCall 1 month ago ↶	555	let mirror = self.git_mirror(url)?;
30ea9c1e Andy McCall 1 month ago ↶	556	self.git_mirror_fetch(&mirror, url, branch)?;
75167672 Andy McCall 1 month ago	557	if git(
	558	&mirror,
	559	&[
	560	"rev-parse",
	561	"--verify",
	562	"-q",
	563	&format!("refs/heads/{branch}"),
	564	],
	565	)
	566	.is_err()
	567	{
30ea9c1e Andy McCall 1 month ago ↶	568	return fail(format!("{url} has no branch {branch:?}"));
30ea9c1e Andy McCall 1 month ago ↶	569	}
58dfe0e1 Andy McCall 1 month ago ↶	570	self.import_git_commits(
	571	&mirror,
	572	branch,
	573	&format!("oxy git fetch {url} {branch}"),
	574	&mut \|_\| {},
	575	)
30ea9c1e Andy McCall 1 month ago ↶	576	}
	577
	578	/// Export the bookmark to `branch` of the Git repository at `url`
	579	/// through the mirror and push it. The remote's tip must already have
	580	/// been fetched (`git_fetch`), so the push fast-forwards.
	581	pub fn git_push(&self, url: &str, branch: &str) -> Result<GitExportReport> {
	582	let mirror = self.git_mirror(url)?;
	583	self.git_mirror_fetch(&mirror, url, branch)?;
75167672 Andy McCall 1 month ago	584	if let Ok(tip) = git(
	585	&mirror,
	586	&[
	587	"rev-parse",
	588	"--verify",
	589	"-q",
	590	&format!("refs/heads/{branch}"),
	591	],
	592	) {
30ea9c1e Andy McCall 1 month ago ↶	593	let known = self.git_known(&mirror)?;
30ea9c1e Andy McCall 1 month ago ↶	594	if !known.imported.contains_key(&tip) && !known.exported.contains_key(&tip) {
75167672 Andy McCall 1 month ago	595	return fail(format!(
	596	"{url} {branch} has commits not yet fetched; run `oxy git fetch {url}` first"
	597	));
30ea9c1e Andy McCall 1 month ago ↶	598	}
	599	}
	600	let report = self.git_export_as(&mirror, branch)?;
	601	let spec = format!("refs/heads/{branch}:refs/heads/{branch}");
	602	git(&mirror, &["push", "-q", url, &spec])?;
	603	Ok(report)
	604	}
	605
79ab8528 Andy McCall 1 month ago ↶	606	fn read_oxy_marks(path: &Path) -> Result<BTreeMap<ObjectRef, u64>> {
	607	let mut out = BTreeMap::new();
	608	if let Ok(text) = fs::read_to_string(path) {
	609	for line in text.lines() {
	610	if let Some((h, m)) = line.split_once(' ') {
	611	if let (Some(h), Ok(m)) = (ObjectRef::from_hex(h), m.parse()) {
	612	out.insert(h, m);
	613	}
	614	}
	615	}
	616	}
	617	Ok(out)
	618	}
	619
	620	/// Export the current bookmark to the Git repository at `git_dir`,
	621	/// creating a bare one if nothing is there. Each state the bookmark
	622	/// passed through (per the operation log) becomes one commit whose tree
	623	/// is that state, so the Git branch is a linear history of full trees.
	624	pub fn git_export(&self, git_dir: &Path) -> Result<GitExportReport> {
30ea9c1e Andy McCall 1 month ago ↶	625	let bookmark = self.config.bookmark.clone();
	626	self.git_export_as(git_dir, &bookmark)
	627	}
	628
	629	/// `git_export` to the Git branch `branch`. A state that came from a
	630	/// Git commit (`oxy git fetch`) is not re-exported: later states chain
	631	/// onto that commit, so a push back to the Git remote fast-forwards.
	632	pub fn git_export_as(&self, git_dir: &Path, branch: &str) -> Result<GitExportReport> {
79ab8528 Andy McCall 1 month ago ↶	633	if !git_dir.join("HEAD").exists() && !git_dir.join(".git").exists() {
	634	fs::create_dir_all(git_dir)?;
	635	git(git_dir, &["init", "--bare", "-q"])?;
	636	}
	637	let marks_dir = self.git_marks_dir(git_dir)?;
	638	let git_marks = marks_dir.join("git-marks");
	639	let oxy_marks_path = marks_dir.join("oxy-marks");
	640	let mut marks = Self::read_oxy_marks(&oxy_marks_path)?;
30ea9c1e Andy McCall 1 month ago ↶	641	let git_states = Self::read_git_states(&marks_dir.join("git-states"))?;
79ab8528 Andy McCall 1 month ago ↶	642	let mut next_mark = marks.values().copied().max().unwrap_or(0) + 1;
	643
	644	let bookmark = self.config.bookmark.clone();
	645	let mut stream: Vec<u8> = Vec::new();
75167672 Andy McCall 1 month ago	646	let mut report = GitExportReport {
	647	branch: branch.to_string(),
	648	heads: self.bookmark_heads()?.len(),
	649	..Default::default()
	650	};
79ab8528 Andy McCall 1 month ago ↶	651
	652	// States along the bookmark, oldest first, skipping unchanged and pruned ones.
	653	let mut ops = self.op_log()?;
	654	ops.reverse();
	655	let mut prev: Option<(ObjectRef, Vec<ChangeHash>)> = None;
30ea9c1e Andy McCall 1 month ago ↶	656	// The fast-import parent of the next commit: `:mark` or a commit sha.
30ea9c1e Andy McCall 1 month ago ↶	657	let mut last_parent: Option<String> = None;
79ab8528 Andy McCall 1 month ago ↶	658	for (_, op) in &ops {
75167672 Andy McCall 1 month ago	659	let Some(state_ref) = op.view.bookmarks.get(&bookmark).copied() else {
	660	continue;
	661	};
79ab8528 Andy McCall 1 month ago ↶	662	if prev.as_ref().is_some_and(\|(p, _)\| *p == state_ref) {
	663	continue;
	664	}
75167672 Andy McCall 1 month ago	665	let Ok(state) = self.store.get_state(&state_ref) else {
	666	continue;
	667	};
79ab8528 Andy McCall 1 month ago ↶	668	if state.heads.is_empty() {
	669	prev = Some((state_ref, vec![]));
	670	continue;
	671	}
30ea9c1e Andy McCall 1 month ago ↶	672	if let Some(sha) = git_states.get(&state_ref) {
	673	report.skipped += 1;
	674	last_parent = Some(sha.clone());
	675	prev = Some((state_ref, state.heads.clone()));
	676	continue;
	677	}
79ab8528 Andy McCall 1 month ago ↶	678	if let Some(m) = marks.get(&state_ref) {
79ab8528 Andy McCall 1 month ago ↶	679	report.skipped += 1;
30ea9c1e Andy McCall 1 month ago ↶	680	last_parent = Some(format!(":{m}"));
79ab8528 Andy McCall 1 month ago ↶	681	prev = Some((state_ref, state.heads.clone()));
	682	continue;
	683	}
	684	let prev_heads = prev.as_ref().map(\|(_, h)\| h.clone()).unwrap_or_default();
75167672 Andy McCall 1 month ago	685	let added = self
	686	.negotiate(&prev_heads, &state.heads)
	687	.unwrap_or_default();
79ab8528 Andy McCall 1 month ago ↶	688	let loaded = self.load_state(&state.heads)?;
	689	let snapshot = loaded.materialize();
	690	let (author, secs, mut message) = match added.as_slice() {
	691	[h] => {
	692	let c = self.store.get_change(h)?;
	693	let mut m = c.message.clone();
	694	if !m.ends_with('\n') {
	695	m.push('\n');
	696	}
75167672 Andy McCall 1 month ago	697	m.push_str(&format!(
	698	"\nChange-Id: {}\nOxy-Hash: {}\n",
	699	c.change_id.to_hex(),
	700	h.to_hex()
	701	));
79ab8528 Andy McCall 1 month ago ↶	702	(self.git_author(&c), c.hlc.wall_ms / 1000, m)
79ab8528 Andy McCall 1 month ago ↶	703	}
75167672 Andy McCall 1 month ago	704	[] => (
	705	"oxy <oxy@localhost>".to_string(),
	706	op.hlc.wall_ms / 1000,
	707	format!("oxy: {}\n\n", op.command),
	708	),
79ab8528 Andy McCall 1 month ago ↶	709	many => {
	710	let mut m = format!("oxy: {} ({} changes)\n\n", op.command, many.len());
	711	for h in many {
	712	let c = self.store.get_change(h)?;
75167672 Andy McCall 1 month ago	713	m.push_str(&format!(
	714	"- {} {}\n",
	715	c.change_id.short(8),
	716	c.message.lines().next().unwrap_or("")
	717	));
79ab8528 Andy McCall 1 month ago ↶	718	}
75167672 Andy McCall 1 month ago	719	m.push_str(&format!(
	720	"\nOxy-Changes: {}\n",
	721	many.iter()
	722	.map(\|h\| h.to_hex())
	723	.collect::<Vec<_>>()
	724	.join(" ")
	725	));
79ab8528 Andy McCall 1 month ago ↶	726	("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, m)
	727	}
	728	};
	729	message.push_str(&format!("Oxy-State: {}\n", state_ref.to_hex()));
	730	if snapshot.is_conflicted() {
	731	message.push_str("Oxy-Conflicted: true\n");
	732	}
	733	let mark = next_mark;
	734	next_mark += 1;
30ea9c1e Andy McCall 1 month ago ↶	735	let parents: Vec<String> = last_parent.clone().into_iter().collect();
75167672 Andy McCall 1 month ago	736	self.write_commit(
	737	&mut stream,
	738	"refs/oxy/export",
	739	mark,
	740	&author,
	741	secs,
	742	&message,
	743	&parents,
	744	&snapshot,
	745	)?;
79ab8528 Andy McCall 1 month ago ↶	746	marks.insert(state_ref, mark);
30ea9c1e Andy McCall 1 month ago ↶	747	last_parent = Some(format!(":{mark}"));
79ab8528 Andy McCall 1 month ago ↶	748	prev = Some((state_ref, state.heads.clone()));
	749	report.exported += 1;
	750	}
30ea9c1e Andy McCall 1 month ago ↶	751	if let Some(p) = &last_parent {
30ea9c1e Andy McCall 1 month ago ↶	752	stream.extend_from_slice(format!("reset refs/heads/{branch}\nfrom {p}\n\n").as_bytes());
79ab8528 Andy McCall 1 month ago ↶	753	}
132b884b Andy McCall 1 month ago ↶	754	// Tags: a tag whose state was exported as a bookmark state points at
	755	// that commit; otherwise its state gets a commit of its own.
	756	for t in self.tags()? {
30ea9c1e Andy McCall 1 month ago ↶	757	let target = match (git_states.get(&t.state), marks.get(&t.state)) {
	758	(Some(sha), _) => sha.clone(),
	759	(None, Some(m)) => format!(":{m}"),
	760	(None, None) => {
132b884b Andy McCall 1 month ago ↶	761	let loaded = self.load_state(&t.heads)?;
	762	let snapshot = loaded.materialize();
	763	let mark = next_mark;
	764	next_mark += 1;
75167672 Andy McCall 1 month ago	765	let message = format!(
	766	"oxy: tag {}\n\n{}Oxy-State: {}\n",
	767	t.name,
	768	if t.message.is_empty() {
	769	String::new()
	770	} else {
	771	format!("{}\n\n", t.message)
	772	},
	773	t.state.to_hex()
	774	);
	775	self.write_commit(
	776	&mut stream,
	777	"refs/oxy/export",
	778	mark,
	779	"oxy <oxy@localhost>",
	780	Hlc::now().wall_ms / 1000,
	781	&message,
	782	&[],
	783	&snapshot,
	784	)?;
132b884b Andy McCall 1 month ago ↶	785	marks.insert(t.state, mark);
132b884b Andy McCall 1 month ago ↶	786	report.exported += 1;
30ea9c1e Andy McCall 1 month ago ↶	787	format!(":{mark}")
132b884b Andy McCall 1 month ago ↶	788	}
132b884b Andy McCall 1 month ago ↶	789	};
75167672 Andy McCall 1 month ago	790	stream.extend_from_slice(
	791	format!("reset refs/tags/{}\nfrom {target}\n\n", t.name).as_bytes(),
	792	);
132b884b Andy McCall 1 month ago ↶	793	}
132b884b Andy McCall 1 month ago ↶	794	stream.extend_from_slice(b"done\n");
79ab8528 Andy McCall 1 month ago ↶	795
	796	let mut cmd = Command::new("git");
	797	// `--force`: the scratch ref may jump between unrelated states.
75167672 Andy McCall 1 month ago	798	cmd.arg("-C")
	799	.arg(git_dir)
	800	.args(["fast-import", "--quiet", "--done", "--force"])
	801	.arg(format!("--export-marks={}", git_marks.display()));
79ab8528 Andy McCall 1 month ago ↶	802	if git_marks.exists() {
	803	cmd.arg(format!("--import-marks={}", git_marks.display()));
	804	}
75167672 Andy McCall 1 month ago	805	let mut child = cmd
	806	.stdin(Stdio::piped())
	807	.stdout(Stdio::null())
	808	.stderr(Stdio::piped())
	809	.spawn()
	810	.map_err(\|e\| RepoError::Msg(format!("running git fast-import: {e}")))?;
79ab8528 Andy McCall 1 month ago ↶	811	child.stdin.take().unwrap().write_all(&stream)?;
	812	let out = child.wait_with_output()?;
	813	if !out.status.success() {
75167672 Andy McCall 1 month ago	814	return fail(format!(
	815	"git fast-import failed: {}",
	816	String::from_utf8_lossy(&out.stderr).trim()
	817	));
79ab8528 Andy McCall 1 month ago ↶	818	}
75167672 Andy McCall 1 month ago	819	let text: String = marks
	820	.iter()
	821	.map(\|(h, m)\| format!("{} {m}\n", h.to_hex()))
	822	.collect();
79ab8528 Andy McCall 1 month ago ↶	823	write_atomic(&oxy_marks_path, text.as_bytes())?;
79ab8528 Andy McCall 1 month ago ↶	824	let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]);
4dee277c Andy McCall 1 month ago ↶	825	report.skipped_subrepos = self.git_export_skipped.replace(0);
79ab8528 Andy McCall 1 month ago ↶	826	Ok(report)
	827	}
	828
c3069537 Andy McCall 1 month ago ↶	829	/// Import branch `branch` of the Git repository at `git_dir` into this
	830	/// (empty) repository's bookmark (spec §23): one change per commit, its
	831	/// edits the difference between the union of its parents' states and its
	832	/// tree, so merge commits become resolutions. Submodules are skipped.
	833	pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result<GitImportReport> {
58dfe0e1 Andy McCall 1 month ago ↶	834	self.import_git_with(git_dir, branch, &mut \|_\| {})
	835	}
	836
	837	/// `import_git`, telling `progress` where it is every few hundred
	838	/// commits: the count so far, the merges, and the commit in hand, so a
	839	/// long import is never silent and a run that dies says where.
	840	pub fn import_git_with(
	841	&self,
	842	git_dir: &Path,
	843	branch: &str,
	844	progress: crate::store::Progress<'_>,
	845	) -> Result<GitImportReport> {
c3069537 Andy McCall 1 month ago ↶	846	if !self.bookmark_heads()?.is_empty() {
	847	return fail("git import needs an empty bookmark; import into a fresh repository");
	848	}
75167672 Andy McCall 1 month ago	849	self.import_git_commits(
	850	git_dir,
	851	branch,
	852	&format!("oxy git import {} {branch}", git_dir.display()),
58dfe0e1 Andy McCall 1 month ago ↶	853	progress,
75167672 Andy McCall 1 month ago	854	)
30ea9c1e Andy McCall 1 month ago ↶	855	}
	856
	857	/// Import the commits of `branch` that the bookmark does not have yet,
	858	/// then move the bookmark to the union of its heads and theirs.
75167672 Andy McCall 1 month ago	859	fn import_git_commits(
	860	&self,
	861	git_dir: &Path,
	862	branch: &str,
	863	command: &str,
58dfe0e1 Andy McCall 1 month ago ↶	864	progress: crate::store::Progress<'_>,
75167672 Andy McCall 1 month ago	865	) -> Result<GitImportReport> {
8f10a07a Andy McCall 1 month ago ↶	866	let batch = self.store.begin_batch()?;
58dfe0e1 Andy McCall 1 month ago ↶	867	let result = self.import_git_commits_batched(git_dir, branch, command, progress);
8f10a07a Andy McCall 1 month ago ↶	868	batch.finish()?;
	869	result
	870	}
	871
75167672 Andy McCall 1 month ago	872	fn import_git_commits_batched(
	873	&self,
	874	git_dir: &Path,
	875	branch: &str,
	876	command: &str,
58dfe0e1 Andy McCall 1 month ago ↶	877	progress: crate::store::Progress<'_>,
75167672 Andy McCall 1 month ago	878	) -> Result<GitImportReport> {
c3069537 Andy McCall 1 month ago ↶	879	let commits = list_commits(git_dir, branch)?;
	880	let mut cat = CatFile::start(git_dir)?;
	881	let mut report = GitImportReport::default();
30ea9c1e Andy McCall 1 month ago ↶	882	let mut known = self.git_known(git_dir)?;
30ea9c1e Andy McCall 1 month ago ↶	883	let old_heads = self.bookmark_heads()?;
dfb23a3a Andy McCall 1 month ago ↶	884	// The bookmark's heads as changes arrive: a change stops being a
	885	// head when another depends on it, so no full state of every
	886	// imported change is needed at the end.
	887	let mut heads_now: BTreeSet<ChangeHash> = old_heads.iter().copied().collect();
958fb27d Andy McCall 1 month ago ↶	888	// The state and Git tree of imported commits whose first-parent
	889	// children are still to come, so a child starts from its parent's
	890	// state instead of rebuilding it from history. The last child takes
dfb23a3a Andy McCall 1 month ago ↶	891	// it; earlier ones clone it. At most `max_kept` are kept (a later
958fb27d Andy McCall 1 month ago ↶	892	// child of an evicted one rebuilds).
dfb23a3a Andy McCall 1 month ago ↶	893	// OXY_IMPORT_KEEP overrides it, for measuring.
	894	let max_kept: usize = std::env::var("OXY_IMPORT_KEEP")
	895	.ok()
	896	.and_then(\|v\| v.parse().ok())
	897	.unwrap_or(8);
958fb27d Andy McCall 1 month ago ↶	898	let mut waiting: HashMap<&str, usize> = HashMap::new();
	899	for gc in &commits {
	900	if let Some(p1) = gc.parents.first() {
	901	*waiting.entry(p1.as_str()).or_default() += 1;
	902	}
	903	}
	904	type Kept = (State, BTreeMap<String, (String, String)>);
	905	let mut kept: HashMap<String, Kept> = HashMap::new();
2306a4d2 Andy McCall 1 month ago ↶	906	// Every commit to import that has a parent, diffed against its first.
	907	let pairs: Vec<(String, String)> = commits
	908	.iter()
	909	.filter(\|gc\| {
	910	!known.imported.contains_key(&gc.sha) && !known.exported.contains_key(&gc.sha)
	911	})
	912	.filter_map(\|gc\| Some((gc.sha.clone(), gc.parents.first()?.clone())))
	913	.collect();
	914	let mut diffs = DiffTree::start(git_dir, pairs)?;
	915	let mut rebuilt = 0usize;
958fb27d Andy McCall 1 month ago ↶	916	let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
958fb27d Andy McCall 1 month ago ↶	917	let mut last: Option<String> = None;
c3069537 Andy McCall 1 month ago ↶	918	let mut blob_cache: HashMap<String, Vec<u8>> = HashMap::new();
c3069537 Andy McCall 1 month ago ↶	919	let mut cache_bytes: usize = 0;
958fb27d Andy McCall 1 month ago ↶	920	// Time spent per phase over all commits (`OXY_TIMINGS=1`).
	921	let mut spent = [std::time::Duration::ZERO; 6];
	922	let mut clock = std::time::Instant::now();
fb5b6181 Andy McCall 1 month ago ↶	923	// One line per commit on stderr with OXY_IMPORT_TRACE=1: where its
	924	// base came from, what a merge added, the graphs its state holds,
	925	// the kept states and the process's resident memory.
	926	let trace = std::env::var_os("OXY_IMPORT_TRACE").is_some();
	927	let rss_mb = \|\| -> u64 {
	928	std::fs::read_to_string("/proc/self/statm")
	929	.ok()
	930	.and_then(\|s\| s.split_whitespace().nth(1)?.parse::<u64>().ok())
d2e3c39d Andy McCall 1 month ago ↶	931	.map(\|pages\| (pages * 4096) >> 20)
fb5b6181 Andy McCall 1 month ago ↶	932	.unwrap_or(0)
	933	};
	934	let mut done = 0usize;
d45dab04 Andy McCall 1 month ago ↶	935	// Atoms a kept state's graphs may hold before they are shed;
	936	// OXY_IMPORT_KEPT_ATOMS overrides it, for measuring.
	937	let kept_atoms: usize = std::env::var("OXY_IMPORT_KEPT_ATOMS")
	938	.ok()
	939	.and_then(\|v\| v.parse().ok())
	940	.unwrap_or(KEPT_ATOMS);
fd280228 Andy McCall 1 month ago ↶	941	// Graphs turned back into history (see `State::shed_graphs`).
fd280228 Andy McCall 1 month ago ↶	942	let mut shed = 0usize;
958fb27d Andy McCall 1 month ago ↶	943	let mut lap = \|i: usize, clock: &mut std::time::Instant\| {
	944	let now = std::time::Instant::now();
	945	spent[i] += now - *clock;
	946	*clock = now;
	947	};
c3069537 Andy McCall 1 month ago ↶	948
9935d490 Andy McCall 29 days ago ↶	949	// OXY_IMPORT_CHECKPOINT overrides it, for testing.
	950	let checkpoint_every: usize = std::env::var("OXY_IMPORT_CHECKPOINT")
	951	.ok()
	952	.and_then(\|v\| v.parse().ok())
	953	.filter(\|n\| *n > 0)
	954	.unwrap_or(CHECKPOINT_COMMITS);
810691ca Andy McCall 29 days ago ↶	955	// OXY_IMPORT_STOP_AFTER ends the import cleanly after that many
	956	// commits, its timings reported, for measuring at depth.
	957	let stop_after: Option<usize> = std::env::var("OXY_IMPORT_STOP_AFTER")
	958	.ok()
	959	.and_then(\|v\| v.parse().ok());
d256e826 Andy McCall 27 days ago ↶	960	// Merges built from the paths they can change (oxygen#80): on VS
	961	// Code at 139k commits the same 400 commits came out identical to
	962	// whole-tree merges, with merges 3.2x faster. OXY_IMPORT_WHOLE_MERGES=1
	963	// builds them from the whole tree, for comparing.
	964	let partial_merges = std::env::var_os("OXY_IMPORT_WHOLE_MERGES").is_none_or(\|v\| v == "0");
58dfe0e1 Andy McCall 1 month ago ↶	965	let total = commits.len();
d6e0d5b8 Andy McCall 25 days ago ↶	966	// The importing identity's own commits, claimed (`git.claim`,
	967	// oxygen#97): a commit whose Git author email is its email gets
	968	// its key as author; anyone else's keeps the zero key.
	969	let claim: Option<(KeyId, String)> = if self.config.git_claim {
	970	let Some(key) = self.config.author_key else {
	971	return fail("git.claim authors your own commits with your key, and there is none: run `oxy identity create` first, or `oxy config git.claim false`");
	972	};
	973	Some((key, self.identity(&key)?.email.to_lowercase()))
	974	} else {
	975	None
	976	};
58dfe0e1 Andy McCall 1 month ago ↶	977	for (index, gc) in commits.iter().enumerate() {
30ea9c1e Andy McCall 1 month ago ↶	978	if known.imported.contains_key(&gc.sha) \|\| known.exported.contains_key(&gc.sha) {
30ea9c1e Andy McCall 1 month ago ↶	979	continue;
810691ca Andy McCall 29 days ago ↶	980	}
	981	if stop_after.is_some_and(\|n\| done >= n) {
	982	break;
58dfe0e1 Andy McCall 1 month ago ↶	983	}
810691ca Andy McCall 29 days ago ↶	984	let commit_started = std::time::Instant::now();
58dfe0e1 Andy McCall 1 month ago ↶	985	if index % 500 == 0 {
	986	progress(&format!(
	987	"{index} of {total} commits, {} merges; at {} {}",
	988	report.merges,
	989	&gc.sha[..12.min(gc.sha.len())],
	990	gc.message
	991	.lines()
	992	.next()
	993	.unwrap_or("")
	994	.chars()
	995	.take(60)
	996	.collect::<String>()
	997	));
30ea9c1e Andy McCall 1 month ago ↶	998	}
fb5b6181 Andy McCall 1 month ago ↶	999	let mut base_how = "root";
958fb27d Andy McCall 1 month ago ↶	1000	// Base: union of every parent's state, starting from the first
	1001	// parent's kept state when there is one; its tree tells which
	1002	// paths this commit leaves alone.
	1003	let (mut base, prev_tree) = match gc.parents.first() {
	1004	Some(p1) => {
	1005	let left = waiting.get_mut(p1.as_str()).map(\|n\| {
	1006	*n -= 1;
	1007	*n
	1008	});
	1009	let found = if left == Some(0) {
	1010	kept_order.retain(\|k\| k != p1);
	1011	kept.remove(p1.as_str())
	1012	} else {
	1013	kept.get(p1.as_str()).cloned()
fb5b6181 Andy McCall 1 month ago ↶	1014	};
	1015	let how = match (&found, left) {
	1016	(Some(_), Some(0)) => "kept",
	1017	(Some(_), _) => "cloned",
	1018	(None, _) => "rebuilt",
958fb27d Andy McCall 1 month ago ↶	1019	};
fb5b6181 Andy McCall 1 month ago ↶	1020	if trace {
	1021	base_how = how;
	1022	}
958fb27d Andy McCall 1 month ago ↶	1023	match found {
	1024	Some((st, tree)) => (st, Some(tree)),
	1025	None => {
b493d993 Andy McCall 1 month ago ↶	1026	// History only: graphs are built for the files a
	1027	// commit changes, so memory follows what changes,
	1028	// not the whole tree.
	1029	let mut st = State::new_history_only();
958fb27d Andy McCall 1 month ago ↶	1030	self.add_git_ancestors(&mut st, git_dir, &[p1.as_str()], &known)?;
2306a4d2 Andy McCall 1 month ago ↶	1031	// The parent's tree, so this commit still
	1032	// renders only what it changes.
	1033	rebuilt += 1;
	1034	(st, Some(list_tree(git_dir, p1)?))
958fb27d Andy McCall 1 month ago ↶	1035	}
958fb27d Andy McCall 1 month ago ↶	1036	}
30ea9c1e Andy McCall 1 month ago ↶	1037	}
b493d993 Andy McCall 1 month ago ↶	1038	None => (State::new_history_only(), None),
c3069537 Andy McCall 1 month ago ↶	1039	};
958fb27d Andy McCall 1 month ago ↶	1040	// Nodes the other parents' changes touch: the first parent's
	1041	// tree cannot vouch for them.
	1042	let mut touched: HashSet<NodeId> = HashSet::new();
fb5b6181 Andy McCall 1 month ago ↶	1043	let mut merged = 0usize;
ce118287 Andy McCall 29 days ago ↶	1044	// Where the first parent has each node, read before the merged
	1045	// changes move anything: a merge's partial build names the paths
	1046	// they touch in the first parent's tree as well as in the merged
	1047	// state (oxygen#80).
	1048	let mut first_parent_view: Option<crate::tree::TreeView> = None;
c3069537 Andy McCall 1 month ago ↶	1049	if gc.parents.len() > 1 {
c3069537 Andy McCall 1 month ago ↶	1050	report.merges += 1;
579afd10 Andy McCall 27 days ago ↶	1051	if prev_tree.is_some() && partial_merges {
ce118287 Andy McCall 29 days ago ↶	1052	first_parent_view = Some(base.tree().view());
ce118287 Andy McCall 29 days ago ↶	1053	}
c3069537 Andy McCall 1 month ago ↶	1054	let mut args: Vec<String> = gc.parents[1..].to_vec();
c3069537 Andy McCall 1 month ago ↶	1055	args.push(format!("^{}", gc.parents[0]));
30ea9c1e Andy McCall 1 month ago ↶	1056	let refs: Vec<&str> = args.iter().map(String::as_str).collect();
fd280228 Andy McCall 1 month ago ↶	1057	// A merge of a long-lived branch brings thousands of
	1058	// changes: added to built graphs, which kept states share,
	1059	// they were copied and applied whole (vscode's import passed
	1060	// 14 GB on such merges). Shed the graphs first, so the
	1061	// changes go in as skeletons and only what this commit
	1062	// touches is built again.
	1063	if base.built_count() > 0 {
	1064	let mut count_args = vec!["rev-list", "--count"];
	1065	count_args.extend(refs.iter().copied());
	1066	let others: usize = git(git_dir, &count_args)?.trim().parse().unwrap_or(0);
33be2409 Andy McCall 1 month ago ↶	1067	let shed_at = std::env::var("OXY_MERGE_SHED")
	1068	.ok()
	1069	.and_then(\|v\| v.parse().ok())
	1070	.unwrap_or(MERGE_SHED_COMMITS);
	1071	if others > shed_at {
fd280228 Andy McCall 1 month ago ↶	1072	shed += base.shed_graphs(\|b\| self.store_blob(b))?;
	1073	}
	1074	}
fb5b6181 Andy McCall 1 month ago ↶	1075	let before = base.len();
958fb27d Andy McCall 1 month ago ↶	1076	self.add_git_ancestors_touching(&mut base, git_dir, &refs, &known, &mut touched)?;
fb5b6181 Andy McCall 1 month ago ↶	1077	merged = base.len() - before;
03571027 Andy McCall 1 month ago ↶	1078	}
958fb27d Andy McCall 1 month ago ↶	1079	lap(0, &mut clock);
2306a4d2 Andy McCall 1 month ago ↶	1080	// The commit's tree: its first parent's with the diff applied,
	1081	// or listed whole when that tree is not kept.
	1082	let diff = match gc.parents.first() {
	1083	Some(_) => Some(diffs.diff(&gc.sha)?),
	1084	None => None,
	1085	};
	1086	let (tree, changed) = match (prev_tree, diff) {
	1087	(Some(mut t), Some(diff)) => {
	1088	let mut changed: HashSet<String> = HashSet::new();
	1089	for (path, entry) in diff {
	1090	match entry {
	1091	Some(v) => {
	1092	t.insert(path.clone(), v);
	1093	}
	1094	None => {
	1095	t.remove(&path);
	1096	}
	1097	}
	1098	changed.insert(path);
	1099	}
	1100	(t, Some(changed))
	1101	}
	1102	_ => (list_tree(git_dir, &gc.sha)?, None),
	1103	};
958fb27d Andy McCall 1 month ago ↶	1104	lap(1, &mut clock);
03571027 Andy McCall 1 month ago ↶	1105	// Text files with the same blob and mode as in the first
	1106	// parent's tree equal the base's rendering: they are neither
	1107	// rendered nor read nor compared (spec §12.3, as for a clean
	1108	// working copy), so a commit costs what it changes.
	1109	let mut same: HashMap<&str, NodeId> = HashMap::new();
2306a4d2 Andy McCall 1 month ago ↶	1110	// Opaque files and symlinks likewise, when the base has one
	1111	// blob for them: they are neither read nor stored again.
	1112	let mut same_opaque: HashSet<&str> = HashSet::new();
	1113	// An ordinary commit in a tree without submodules is built from
	1114	// the paths it changes alone (oxygen#25): with the `.oxyattributes`
	1115	// files that decide how they are stored, and the directories its
	1116	// removals may leave empty. A merge, or a commit whose parent's
	1117	// tree is not at hand, is built from its whole tree.
ce118287 Andy McCall 29 days ago ↶	1118	// A merge can differ from its first parent only at the paths
	1119	// Git's diff names and where the merged changes touched: those
	1120	// paths in the first parent's tree and in the merged state. With
	1121	// them added it is built like an ordinary commit, instead of
	1122	// from its whole tree (18 s for any merge at 118k commits of
d256e826 Andy McCall 27 days ago ↶	1123	// VS Code, oxygen#80). OXY_IMPORT_WHOLE_MERGES=1 takes the
d256e826 Andy McCall 27 days ago ↶	1124	// whole-tree path.
ce118287 Andy McCall 29 days ago ↶	1125	let mut changed = changed;
	1126	// A tree conflict in the merged state (both sides placed a file
	1127	// at one path) is resolved by the whole-tree path.
	1128	let mut merge_how = if gc.parents.len() > 1 {
	1129	if first_parent_view.is_some() {
	1130	"partial"
	1131	} else {
	1132	"whole"
	1133	}
	1134	} else {
	1135	"-"
	1136	};
	1137	// Only a conflict among what the merged changes touched; one
	1138	// elsewhere stays as it was in the first parent either way.
	1139	if first_parent_view.is_some()
	1140	&& base.tree().view().conflicts.iter().any(\|c\| {
	1141	use crate::tree::TreeConflict::*;
	1142	match c {
	1143	Name { nodes, .. } => nodes.iter().any(\|n\| touched.contains(n)),
	1144	MultiPlacement { node, .. } \| Cycle { node } \| Mode { node, .. } => {
	1145	touched.contains(node)
	1146	}
	1147	Orphan { node, parent } => {
	1148	touched.contains(node) \|\| touched.contains(parent)
	1149	}
	1150	}
	1151	})
	1152	{
	1153	first_parent_view = None;
	1154	merge_how = "conflicted";
	1155	}
	1156	if let (Some(ch), Some(fp)) = (changed.as_mut(), &first_parent_view) {
	1157	let union = base.tree().view();
	1158	for v in [fp, &union] {
	1159	for (path, e) in v.entries.iter() {
	1160	if touched.contains(&e.node) {
	1161	ch.insert(path.clone());
	1162	}
	1163	}
	1164	}
	1165	}
2306a4d2 Andy McCall 1 month ago ↶	1166	let partial = changed.is_some()
ce118287 Andy McCall 29 days ago ↶	1167	&& (gc.parents.len() == 1 \|\| first_parent_view.is_some())
2306a4d2 Andy McCall 1 month ago ↶	1168	&& !tree.values().any(\|(mode, _)\| mode == "160000");
2306a4d2 Andy McCall 1 month ago ↶	1169	let mut rel: BTreeSet<&str> = BTreeSet::new();
ce118287 Andy McCall 29 days ago ↶	1170	// Text files whose graphs this commit builds, for the trace.
ce118287 Andy McCall 29 days ago ↶	1171	let need_n: usize;
2306a4d2 Andy McCall 1 month ago ↶	1172	let base_snapshot = if let (true, Some(changed)) = (partial, &changed) {
03571027 Andy McCall 1 month ago ↶	1173	let view = base.tree().view();
2306a4d2 Andy McCall 1 month ago ↶	1174	rel.extend(changed.iter().map(String::as_str));
	1175	rel.extend(
	1176	tree.keys()
	1177	.map(String::as_str)
	1178	.filter(\|p\| *p == ".oxyattributes" \|\| p.ends_with("/.oxyattributes")),
	1179	);
	1180	let mut dirs: BTreeSet<&str> = BTreeSet::new();
	1181	for p in changed.iter().filter(\|p\| !tree.contains_key(*p)) {
	1182	let mut end = p.len();
	1183	while let Some(i) = p[..end].rfind('/') {
	1184	if !dirs.insert(&p[..i]) {
	1185	break;
	1186	}
	1187	end = i;
03571027 Andy McCall 1 month ago ↶	1188	}
2306a4d2 Andy McCall 1 month ago ↶	1189	}
	1190	let need: Vec<NodeId> = rel
	1191	.iter()
	1192	.filter_map(\|p\| view.entries.get(*p))
	1193	.filter(\|e\| e.kind == NodeKind::File && !e.mode.opaque)
	1194	.map(\|e\| e.node)
	1195	.collect();
	1196	crate::timing::add("render: same and need", clock.elapsed());
ce118287 Andy McCall 29 days ago ↶	1197	need_n = need.len();
2306a4d2 Andy McCall 1 month ago ↶	1198	let t = std::time::Instant::now();
	1199	self.load_graphs_of(&mut base, &need)?;
	1200	crate::timing::add("render: load graphs", t.elapsed());
	1201	let t = std::time::Instant::now();
	1202	let snap = base.materialize_paths(view, rel.iter().chain(dirs.iter()).copied());
	1203	crate::timing::add("render: materialize", t.elapsed());
	1204	snap
	1205	} else {
	1206	if let Some(changed) = &changed {
	1207	let view = base.tree().view();
	1208	for path in tree.keys() {
	1209	// Submodule handling reads these two whatever changed.
	1210	if changed.contains(path) \|\| path == ".gitmodules" \|\| path == ".oxymodules"
958fb27d Andy McCall 1 month ago ↶	1211	{
2306a4d2 Andy McCall 1 month ago ↶	1212	continue;
	1213	}
	1214	if let Some(e) = view.entries.get(path) {
	1215	if touched.contains(&e.node) {
	1216	continue;
	1217	}
	1218	let opaque = e.kind == NodeKind::Symlink
	1219	\|\| (e.kind == NodeKind::File && e.mode.opaque);
	1220	if e.kind == NodeKind::File && !e.mode.opaque {
	1221	same.insert(path.as_str(), e.node);
	1222	} else if opaque && base.live_blobs(&e.node).len() == 1 {
	1223	same_opaque.insert(path.as_str());
	1224	}
03571027 Andy McCall 1 month ago ↶	1225	}
	1226	}
	1227	}
2306a4d2 Andy McCall 1 month ago ↶	1228	// Build the graphs of every text file the commit may change.
	1229	let skip: HashSet<NodeId> = same.values().copied().collect();
	1230	let need: Vec<NodeId> = base
	1231	.tree()
	1232	.view()
	1233	.entries
	1234	.values()
	1235	.filter(\|e\| {
	1236	e.kind == NodeKind::File && !e.mode.opaque && !skip.contains(&e.node)
	1237	})
	1238	.map(\|e\| e.node)
	1239	.collect();
	1240	crate::timing::add("render: same and need", clock.elapsed());
ce118287 Andy McCall 29 days ago ↶	1241	need_n = need.len();
2306a4d2 Andy McCall 1 month ago ↶	1242	let t = std::time::Instant::now();
c3d2ef61 Andy McCall 1 month ago ↶	1243	let snap = if need.len() > merge_batch() {
	1244	// A merge onto a base with no built graphs: thousands
	1245	// of files, whose graphs would not fit at once.
	1246	self.materialize_batched(&mut base, &need, &skip)?
	1247	} else {
ce118287 Andy McCall 29 days ago ↶	1248	let g = std::time::Instant::now();
c3d2ef61 Andy McCall 1 month ago ↶	1249	self.load_graphs_of(&mut base, &need)?;
ce118287 Andy McCall 29 days ago ↶	1250	crate::timing::add("render: whole, load graphs", g.elapsed());
	1251	let m = std::time::Instant::now();
	1252	let snap = base.materialize_except(&skip);
	1253	crate::timing::add("render: whole, materialize", m.elapsed());
	1254	snap
c3d2ef61 Andy McCall 1 month ago ↶	1255	};
c3d2ef61 Andy McCall 1 month ago ↶	1256	crate::timing::add("render: whole tree", t.elapsed());
2306a4d2 Andy McCall 1 month ago ↶	1257	snap
2306a4d2 Andy McCall 1 month ago ↶	1258	};
c3069537 Andy McCall 1 month ago ↶	1259
958fb27d Andy McCall 1 month ago ↶	1260	lap(2, &mut clock);
c3069537 Andy McCall 1 month ago ↶	1261	// The commit's tree as in-memory entries.
c3069537 Andy McCall 1 month ago ↶	1262	let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();
4dee277c Andy McCall 1 month ago ↶	1263	let mut gitlinks: Vec<(String, String)> = Vec::new();
2306a4d2 Andy McCall 1 month ago ↶	1264	let listed: Box<dyn Iterator<Item = (&String, &(String, String))>> = if partial {
	1265	Box::new(rel.iter().filter_map(\|p\| tree.get_key_value(*p)))
	1266	} else {
	1267	Box::new(tree.iter())
	1268	};
	1269	for (path, (mode, sha)) in listed {
03571027 Andy McCall 1 month ago ↶	1270	let (path, sha) = (path.clone(), sha.clone());
c3069537 Andy McCall 1 month ago ↶	1271	let (kind, exec) = match mode.as_str() {
	1272	"100644" \| "100664" => (NodeKind::File, false),
	1273	"100755" => (NodeKind::File, true),
	1274	"120000" => (NodeKind::Symlink, false),
	1275	"160000" => {
4dee277c Andy McCall 1 month ago ↶	1276	gitlinks.push((path, sha));
c3069537 Andy McCall 1 month ago ↶	1277	continue;
	1278	}
	1279	_ => continue,
	1280	};
2306a4d2 Andy McCall 1 month ago ↶	1281	if same.contains_key(path.as_str()) \|\| same_opaque.contains(path.as_str()) {
03571027 Andy McCall 1 month ago ↶	1282	let mut e = FsEntry::new(kind, exec, Vec::new());
	1283	e.unchanged = true;
	1284	entries.insert(path, e);
	1285	continue;
	1286	}
c3069537 Andy McCall 1 month ago ↶	1287	let content = match blob_cache.get(&sha) {
	1288	Some(c) => c.clone(),
	1289	None => {
	1290	let c = cat.read(&sha)?;
dfb23a3a Andy McCall 1 month ago ↶	1291	if cache_bytes + c.len() > 64 * 1024 * 1024 {
c3069537 Andy McCall 1 month ago ↶	1292	blob_cache.clear();
	1293	cache_bytes = 0;
	1294	}
	1295	cache_bytes += c.len();
	1296	blob_cache.insert(sha.clone(), c.clone());
	1297	c
	1298	}
	1299	};
b7f102fc Andy McCall 1 month ago ↶	1300	entries.insert(path, FsEntry::new(kind, exec, content));
c3069537 Andy McCall 1 month ago ↶	1301	}
c3069537 Andy McCall 1 month ago ↶	1302
4dee277c Andy McCall 1 month ago ↶	1303	// Submodules (spec §23): gitlinks become subrepo nodes with Git
	1304	// pins, and `.gitmodules` also yields `.oxymodules`.
	1305	if !gitlinks.is_empty() {
	1306	report.submodules += gitlinks.len();
75167672 Andy McCall 1 month ago	1307	let gm = entries
	1308	.get(".gitmodules")
	1309	.map(\|e\| {
	1310	crate::subrepo::Modules::from_gitmodules(&String::from_utf8_lossy(
	1311	&e.content,
	1312	))
	1313	})
	1314	.unwrap_or_default();
	1315	let mut modules = entries
	1316	.get(".oxymodules")
	1317	.map(\|e\| crate::subrepo::Modules::parse(&String::from_utf8_lossy(&e.content)))
	1318	.unwrap_or_default();
4dee277c Andy McCall 1 month ago ↶	1319	for (path, sha) in &gitlinks {
4dee277c Andy McCall 1 month ago ↶	1320	let m = gm.entries.get(path);
75167672 Andy McCall 1 month ago	1321	let branch = m
	1322	.and_then(\|m\| m.bookmark.clone())
	1323	.unwrap_or_else(\|\| "HEAD".into());
	1324	entries.insert(
	1325	path.clone(),
	1326	FsEntry::new(
	1327	NodeKind::Subrepo,
	1328	false,
	1329	crate::subrepo::Pin::Git {
	1330	sha: sha.clone(),
	1331	branch,
	1332	}
	1333	.encode(),
	1334	),
	1335	);
4dee277c Andy McCall 1 month ago ↶	1336	if let Some(m) = m {
	1337	modules.entries.insert(path.clone(), m.clone());
	1338	}
	1339	}
	1340	if !modules.entries.is_empty() {
75167672 Andy McCall 1 month ago	1341	entries.insert(
	1342	".oxymodules".into(),
	1343	FsEntry::new(NodeKind::File, false, modules.to_text().into_bytes()),
	1344	);
4dee277c Andy McCall 1 month ago ↶	1345	}
4dee277c Andy McCall 1 month ago ↶	1346	}
958fb27d Andy McCall 1 month ago ↶	1347	lap(3, &mut clock);
dfb23a3a Andy McCall 1 month ago ↶	1348	// New nodes' identifiers derive from the commit, so importing the
	1349	// same Git history twice gives the same changes (spec §23).
	1350	let seed = *blake3::hash(format!("oxy git import\0{}", gc.sha).as_bytes()).as_bytes();
2306a4d2 Andy McCall 1 month ago ↶	1351	let mut built = if partial {
	1352	// Whether the new tree has anything under `dir`.
	1353	let dir_needed = \|dir: &str\| {
	1354	let prefix = format!("{dir}/");
	1355	tree.range(prefix.clone()..)
	1356	.next()
	1357	.is_some_and(\|(k, _)\| k.starts_with(&prefix))
	1358	};
	1359	self.build_from_entries_seeded(
	1360	&base,
	1361	&base_snapshot,
	1362	&entries,
	1363	seed,
	1364	Some(&dir_needed),
	1365	)?
	1366	} else {
	1367	self.build_from_entries_seeded(&base, &base_snapshot, &entries, seed, None)?
	1368	};
958fb27d Andy McCall 1 month ago ↶	1369	lap(4, &mut clock);
62794ecc Andy McCall 1 month ago ↶	1370	// A file or submodule with several live blobs (branches that
	1371	// replaced it independently) is resolved to the commit's blob:
	1372	// Git's merge is the authority here, unlike a working copy,
	1373	// where a candidate on disk is not yet a decision (spec §23).
	1374	for (path, f) in &base_snapshot.files {
	1375	let (FileContent::Opaque { blobs }, Some(e)) = (&f.content, entries.get(path))
	1376	else {
	1377	continue;
	1378	};
	1379	if blobs.len() < 2 \|\| f.kind != e.kind {
	1380	continue;
	1381	}
	1382	let chosen = self.store_blob(&e.content)?;
	1383	for b in blobs {
	1384	built.edits.push(Edit::Replace {
	1385	node: f.node,
	1386	old: Some(*b),
	1387	new: Some(chosen),
	1388	});
	1389	}
	1390	built.resolves = true;
	1391	built
	1392	.contributors
	1393	.extend(base.live_replaces(&f.node).into_iter().map(\|(c, _)\| c));
	1394	}
e83354cd Andy McCall 1 month ago ↶	1395	// It shares the base's tree view, which applying the change
	1396	// then updates in place rather than copying.
	1397	drop(base_snapshot);
	1398	drop(entries);
c3069537 Andy McCall 1 month ago ↶	1399	let mut deps: BTreeSet<ChangeHash> = built.contributors.iter().copied().collect();
	1400	let mut message = gc.message.clone();
	1401	if !message.ends_with('\n') {
	1402	message.push('\n');
	1403	}
75167672 Andy McCall 1 month ago	1404	message.push_str(&format!(
	1405	"\nGit-Commit: {}\nGit-Author: {}\n",
	1406	gc.sha, gc.author
	1407	));
c3069537 Andy McCall 1 month ago ↶	1408	let mut change = Change {
6f8aeb99 Andy McCall 1 month ago ↶	1409	version: crate::objects::CHANGE_VERSION,
c3069537 Andy McCall 1 month ago ↶	1410	change_id: change_id_for(&gc.sha, &gc.message),
c3069537 Andy McCall 1 month ago ↶	1411	deps: vec![],
d6e0d5b8 Andy McCall 25 days ago ↶	1412	author: claim
	1413	.as_ref()
	1414	.filter(\|(_, email)\| git_email(&gc.author).is_some_and(\|e\| e == *email))
	1415	.map_or(ObjectRef::ZERO, \|(key, _)\| *key),
75167672 Andy McCall 1 month ago	1416	hlc: Hlc {
	1417	wall_ms: gc.author_secs * 1000,
	1418	logical: 0,
	1419	},
c3069537 Andy McCall 1 month ago ↶	1420	message,
	1421	tree_ops: built.tree_ops,
	1422	edits: built.edits,
75167672 Andy McCall 1 month ago	1423	kind: if built.resolves {
	1424	ChangeKind::Resolution
	1425	} else {
	1426	ChangeKind::Normal
	1427	},
c3069537 Andy McCall 1 month ago ↶	1428	};
2306a4d2 Andy McCall 1 month ago ↶	1429	let t = std::time::Instant::now();
73d592e7 Andy McCall 1 month ago ↶	1430	// A deletion depends on whoever inserted the live atoms after
	1431	// it, which only the file's graph knows: build the few the
	1432	// batched rendering left deferred.
	1433	let need: Vec<NodeId> = change
	1434	.edits
	1435	.iter()
	1436	.filter(\|e\| matches!(e, Edit::Delete { .. }))
	1437	.map(\|e\| e.node())
	1438	.filter(\|n\| !base.is_built(n))
	1439	.collect();
	1440	if !need.is_empty() {
	1441	self.load_graphs_of(&mut base, &need)?;
	1442	}
c3069537 Andy McCall 1 month ago ↶	1443	deps.extend(referenced_changes(&change, &base));
2306a4d2 Andy McCall 1 month ago ↶	1444	crate::timing::add("store: referenced", t.elapsed());
c3069537 Andy McCall 1 month ago ↶	1445	change.deps = deps.into_iter().collect();
	1446	change.normalize();
	1447	let hash = self.store.put_change(&change)?;
dfb23a3a Andy McCall 1 month ago ↶	1448	for d in &change.deps {
	1449	heads_now.remove(d);
	1450	}
	1451	heads_now.insert(hash);
ab4d4e31 Andy McCall 1 month ago ↶	1452	// Built graphs take its edits whole; files not built keep only
ab4d4e31 Andy McCall 1 month ago ↶	1453	// where its text is, read back from the store.
2306a4d2 Andy McCall 1 month ago ↶	1454	let t = std::time::Instant::now();
ab4d4e31 Andy McCall 1 month ago ↶	1455	let (skeleton, texts) = self.store.get_change_lazy(&hash)?;
ab4d4e31 Andy McCall 1 month ago ↶	1456	base.add_known_mixed(hash, change, skeleton, texts);
2306a4d2 Andy McCall 1 month ago ↶	1457	crate::timing::add("store: add to state", t.elapsed());
30ea9c1e Andy McCall 1 month ago ↶	1458	known.imported.insert(gc.sha.clone(), hash);
958fb27d Andy McCall 1 month ago ↶	1459	if waiting.get(gc.sha.as_str()).is_some_and(\|n\| *n > 0) {
fd280228 Andy McCall 1 month ago ↶	1460	// A kept state holds its graphs until its children come:
fd280228 Andy McCall 1 month ago ↶	1461	// past a few thousand, they go back into its history.
d45dab04 Andy McCall 1 month ago ↶	1462	// Past a few thousand graphs, or past a budget of atoms: a
	1463	// long-lived file's graph keeps every line it ever had, so a
	1464	// handful of them can hold gigabytes (#25).
	1465	if base.built_count() > KEPT_GRAPHS \|\| base.built_atoms() > kept_atoms {
fd280228 Andy McCall 1 month ago ↶	1466	shed += base.shed_graphs(\|b\| self.store_blob(b))?;
fd280228 Andy McCall 1 month ago ↶	1467	}
958fb27d Andy McCall 1 month ago ↶	1468	kept.insert(gc.sha.clone(), (base, tree));
958fb27d Andy McCall 1 month ago ↶	1469	kept_order.push_back(gc.sha.clone());
dfb23a3a Andy McCall 1 month ago ↶	1470	if kept_order.len() > max_kept {
958fb27d Andy McCall 1 month ago ↶	1471	if let Some(old) = kept_order.pop_front() {
	1472	kept.remove(&old);
	1473	}
	1474	}
	1475	}
	1476	last = Some(gc.sha.clone());
c3069537 Andy McCall 1 month ago ↶	1477	report.commits += 1;
958fb27d Andy McCall 1 month ago ↶	1478	lap(5, &mut clock);
fb5b6181 Andy McCall 1 month ago ↶	1479	done += 1;
9935d490 Andy McCall 29 days ago ↶	1480	if done.is_multiple_of(checkpoint_every) {
	1481	self.checkpoint_import(&heads_now, done)?;
	1482	}
d45dab04 Andy McCall 1 month ago ↶	1483	// A merge renders thousands of files on many threads and frees
	1484	// it all; the allocator keeps those pages unless asked to give
	1485	// them back, and the next merge piles on top (#25).
	1486	let before_trim = if trace { rss_mb() } else { 0 };
	1487	if gc.parents.len() > 1 {
	1488	crate::alloc_stats::release_freed_memory();
	1489	}
fb5b6181 Andy McCall 1 month ago ↶	1490	if trace {
	1491	let built = kept
	1492	.get(gc.sha.as_str())
	1493	.map(\|(st, _)\| st.built_count())
d45dab04 Andy McCall 1 month ago ↶	1494	.unwrap_or(0);
	1495	let (edits, most) = kept
	1496	.get(gc.sha.as_str())
	1497	.map(\|(st, _)\| st.deferred_edit_count())
	1498	.unwrap_or((0, 0));
	1499	let atoms = kept
	1500	.get(gc.sha.as_str())
	1501	.map(\|(st, _)\| st.built_atoms())
fb5b6181 Andy McCall 1 month ago ↶	1502	.unwrap_or(0);
fb5b6181 Andy McCall 1 month ago ↶	1503	eprintln!(
ce118287 Andy McCall 29 days ago ↶	1504	"trace {done} {} parents={} base={base_how} merged={merged} built={built} atoms={atoms} kept={} shed={shed} rss_mb={} before_trim_mb={before_trim} deferred_edits={edits} most={most} need={need_n} merge={merge_how} ms={}",
fb5b6181 Andy McCall 1 month ago ↶	1505	&gc.sha[..12.min(gc.sha.len())],
	1506	gc.parents.len(),
	1507	kept.len(),
810691ca Andy McCall 29 days ago ↶	1508	rss_mb(),
810691ca Andy McCall 29 days ago ↶	1509	commit_started.elapsed().as_millis()
fb5b6181 Andy McCall 1 month ago ↶	1510	);
fb5b6181 Andy McCall 1 month ago ↶	1511	}
958fb27d Andy McCall 1 month ago ↶	1512	}
	1513	for (name, d) in [
	1514	"git import: bases",
2306a4d2 Andy McCall 1 month ago ↶	1515	"git import: tree",
958fb27d Andy McCall 1 month ago ↶	1516	"git import: render base",
	1517	"git import: read blobs",
	1518	"git import: build change",
	1519	"git import: store change",
	1520	]
	1521	.iter()
	1522	.zip(spent)
	1523	{
	1524	crate::timing::record(name, d);
2306a4d2 Andy McCall 1 month ago ↶	1525	}
	1526	crate::timing::report_totals();
	1527	if crate::timing::enabled() {
fd280228 Andy McCall 1 month ago ↶	1528	eprintln!(
	1529	"timing git import: {rebuilt} bases rebuilt from history, {shed} graphs shed"
	1530	);
c3069537 Andy McCall 1 month ago ↶	1531	}
c3069537 Andy McCall 1 month ago ↶	1532
30ea9c1e Andy McCall 1 month ago ↶	1533	// The bookmark: heads of the old state plus every new change.
	1534	report.heads = old_heads.len();
	1535	if last.is_none() {
	1536	return Ok(report);
	1537	}
dfb23a3a Andy McCall 1 month ago ↶	1538	let heads: Vec<ChangeHash> = heads_now.into_iter().collect();
c3069537 Andy McCall 1 month ago ↶	1539	report.heads = heads.len();
	1540	let (op_ref, op) = self.head_op()?;
	1541	let mut view = op.view.clone();
30ea9c1e Andy McCall 1 month ago ↶	1542	let old_state_ref = view.bookmarks.get(&self.config.bookmark).copied();
30ea9c1e Andy McCall 1 month ago ↶	1543	let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?;
75167672 Andy McCall 1 month ago	1544	view.bookmarks
75167672 Andy McCall 1 month ago	1545	.insert(self.config.bookmark.clone(), state_ref);
30ea9c1e Andy McCall 1 month ago ↶	1546	self.record_op(&[op_ref], command, view)?;
	1547	// The new state equals the tip's tree when the old one was itself a
	1548	// Git commit's tree (fetched or exported), or there was none.
75167672 Andy McCall 1 month ago	1549	let old_known = old_heads.is_empty()
75167672 Andy McCall 1 month ago	1550	\|\| old_state_ref.is_some_and(\|s\| known.exported.values().any(\|e\| *e == s));
30ea9c1e Andy McCall 1 month ago ↶	1551	if let (Some(tip), true) = (commits.last(), old_known) {
	1552	self.record_git_state(git_dir, &state_ref, &tip.sha)?;
	1553	}
	1554	// The working copy follows when it was clean on the old heads.
	1555	if old_heads.is_empty() {
	1556	self.checkout(&heads, true, &format!("{command} (checkout)"))?;
	1557	} else {
	1558	let snap = self.snapshot(command)?;
	1559	let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
	1560	let mut base = snap.base.maximal(&snap.wc.deps);
	1561	base.sort();
	1562	let mut sorted_old = old_heads.clone();
	1563	sorted_old.sort();
9935d490 Andy McCall 29 days ago ↶	1564	// An empty base too: an import that stopped after a checkpoint
	1565	// never checked anything out, and resuming it should.
	1566	if clean && (base == sorted_old \|\| base.is_empty()) {
30ea9c1e Andy McCall 1 month ago ↶	1567	self.checkout(&heads, false, &format!("{command} (checkout)"))?;
	1568	}
	1569	}
c3069537 Andy McCall 1 month ago ↶	1570	Ok(report)
	1571	}
	1572
30ea9c1e Andy McCall 1 month ago ↶	1573	/// Add to `st` the changes of every Git commit `rev-list args` lists:
	1574	/// an imported commit's change, or the whole state an exported commit
	1575	/// was made from (which may hold changes Git never saw).
75167672 Andy McCall 1 month ago	1576	fn add_git_ancestors(
958fb27d Andy McCall 1 month ago ↶	1577	&self,
	1578	st: &mut State,
	1579	git_dir: &Path,
	1580	args: &[&str],
	1581	known: &GitKnown,
	1582	) -> Result<()> {
	1583	self.add_git_ancestors_touching(st, git_dir, args, known, &mut HashSet::new())
	1584	}
	1585
	1586	/// `add_git_ancestors`, also collecting in `touched` every node the
	1587	/// changes it adds edit, place or remove.
	1588	fn add_git_ancestors_touching(
75167672 Andy McCall 1 month ago	1589	&self,
	1590	st: &mut State,
	1591	git_dir: &Path,
	1592	args: &[&str],
	1593	known: &GitKnown,
958fb27d Andy McCall 1 month ago ↶	1594	touched: &mut HashSet<NodeId>,
75167672 Andy McCall 1 month ago	1595	) -> Result<()> {
30ea9c1e Andy McCall 1 month ago ↶	1596	let mut full = vec!["rev-list"];
	1597	full.extend(args);
	1598	let out = git(git_dir, &full)?;
958fb27d Andy McCall 1 month ago ↶	1599	let mut add = \|st: &mut State, h: &ChangeHash\| -> Result<()> {
958fb27d Andy McCall 1 month ago ↶	1600	if st.contains(h) {
e83354cd Andy McCall 1 month ago ↶	1601	return Ok(());
	1602	}
	1603	if !st.is_history_only() {
	1604	let c = self.store.get_change(h)?;
	1605	touched.extend(c.edits.iter().map(\|e\| e.node()));
	1606	touched.extend(c.tree_ops.iter().map(\|o\| o.node()));
	1607	st.add_unchecked(c);
958fb27d Andy McCall 1 month ago ↶	1608	return Ok(());
e83354cd Andy McCall 1 month ago ↶	1609	}
	1610	// History only: the text of an edit waits unread unless its
	1611	// file's graph is built.
	1612	let (skeleton, texts) = self.store.get_change_lazy(h)?;
	1613	touched.extend(skeleton.edits.iter().map(\|e\| e.node()));
	1614	touched.extend(skeleton.tree_ops.iter().map(\|o\| o.node()));
	1615	let built = skeleton
	1616	.edits
	1617	.iter()
	1618	.any(\|e\| !matches!(e, Edit::Replace { .. }) && st.is_built(&e.node()));
	1619	if built {
	1620	let full = self.store.get_change(h)?;
	1621	st.add_known_mixed(*h, full, skeleton, texts);
	1622	} else {
	1623	st.add_known(*h, skeleton, texts);
958fb27d Andy McCall 1 month ago ↶	1624	}
	1625	Ok(())
	1626	};
c3069537 Andy McCall 1 month ago ↶	1627	for sha in out.lines() {
30ea9c1e Andy McCall 1 month ago ↶	1628	if let Some(h) = known.imported.get(sha) {
958fb27d Andy McCall 1 month ago ↶	1629	add(st, h)?;
30ea9c1e Andy McCall 1 month ago ↶	1630	} else if let Some(state_ref) = known.exported.get(sha) {
	1631	let heads = self.store.get_state(state_ref)?.heads;
	1632	for h in self.negotiate(&[], &heads)? {
958fb27d Andy McCall 1 month ago ↶	1633	add(st, &h)?;
30ea9c1e Andy McCall 1 month ago ↶	1634	}
	1635	}
	1636	}
9935d490 Andy McCall 29 days ago ↶	1637	Ok(())
	1638	}
	1639
54e2e630 Andy McCall 29 days ago ↶	1640	/// Recover an import that stopped before it moved the bookmark: its
	1641	/// unfinished pack indexed (`Store::adopt_unfinished_packs`), then every
	1642	/// change in the store that carries a `Git-Commit:` trailer, its heads
	1643	/// (those no other such change depends on) made the bookmark's heads,
	1644	/// so `oxy git fetch` carries on from there. Reads every object once.
	1645	/// Returns how many imported changes it found and the heads.
	1646	pub fn recover_git_import(&self, progress: &dyn Fn(&str)) -> Result<(usize, usize)> {
	1647	// A stopped import's objects are in a pack it never indexed.
	1648	// OXY_RECOVER_MIN_AGE (seconds) overrides the five minutes it must
	1649	// have been left alone, for testing.
	1650	let min_age = std::env::var("OXY_RECOVER_MIN_AGE")
	1651	.ok()
	1652	.and_then(\|v\| v.parse().ok())
	1653	.unwrap_or(300);
	1654	self.store
	1655	.adopt_unfinished_packs(std::time::Duration::from_secs(min_age), progress)?;
	1656	let all = self
	1657	.store
	1658	.list()
	1659	.map_err(\|e\| RepoError::Msg(format!("listing the store: {e}")))?;
	1660	let total = all.len();
	1661	let mut found: HashMap<ChangeHash, Vec<ChangeHash>> = HashMap::new();
	1662	for (i, r) in all.iter().enumerate() {
	1663	if i.is_multiple_of(100_000) {
	1664	progress(&format!(
	1665	"{i} of {total} objects read, {} imported changes",
	1666	found.len()
	1667	));
	1668	}
	1669	let Ok(meta) = self.store.get_change_meta(r) else {
	1670	continue;
	1671	};
	1672	if meta.message.lines().any(\|l\| l.starts_with("Git-Commit: ")) {
	1673	found.insert(*r, meta.deps.clone());
	1674	}
	1675	}
	1676	if found.is_empty() {
	1677	return fail("no imported Git commit's change in this store");
	1678	}
	1679	let depended: HashSet<ChangeHash> = found.values().flatten().copied().collect();
	1680	let mut heads: Vec<ChangeHash> = found
	1681	.keys()
	1682	.filter(\|h\| !depended.contains(h))
	1683	.copied()
	1684	.collect();
	1685	// The bookmark's present heads stay: a recovery only adds.
	1686	heads.extend(self.bookmark_heads().unwrap_or_default());
	1687	let st = self.load_state_history(&heads)?;
	1688	let mut heads = st.maximal(&heads);
	1689	heads.sort();
	1690	heads.dedup();
	1691	let (op_ref, op) = self.head_op()?;
	1692	let mut view = op.view.clone();
	1693	let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?;
	1694	view.bookmarks
	1695	.insert(self.config.bookmark.clone(), state_ref);
	1696	self.record_op(&[op_ref], "oxy git recover", view)?;
	1697	Ok((found.len(), heads.len()))
	1698	}
	1699
9935d490 Andy McCall 29 days ago ↶	1700	/// Point the bookmark at an import's heads so far (see
	1701	/// `CHECKPOINT_COMMITS`); the working copy is left alone until the end.
	1702	fn checkpoint_import(&self, heads: &BTreeSet<ChangeHash>, done: usize) -> Result<()> {
	1703	let heads: Vec<ChangeHash> = heads.iter().copied().collect();
	1704	let (op_ref, op) = self.head_op()?;
	1705	let mut view = op.view.clone();
	1706	let state_ref = self.store.put_state(&StateObject::new(heads))?;
	1707	view.bookmarks
	1708	.insert(self.config.bookmark.clone(), state_ref);
	1709	self.record_op(
	1710	&[op_ref],
	1711	&format!("oxy git import (checkpoint after {done} commits)"),
	1712	view,
	1713	)?;
30ea9c1e Andy McCall 1 month ago ↶	1714	Ok(())
	1715	}
	1716
	1717	/// Commits this repository can relate to Oxygen history: imported
	1718	/// ones by change, exported ones (through `git_dir`'s marks) by state.
	1719	fn git_known(&self, git_dir: &Path) -> Result<GitKnown> {
	1720	let imported = self.imported_commits()?;
	1721	let marks_dir = self.git_marks_dir(git_dir)?;
	1722	let oxy_marks = Self::read_oxy_marks(&marks_dir.join("oxy-marks"))?;
	1723	let mut sha_of_mark: HashMap<u64, String> = HashMap::new();
	1724	if let Ok(text) = fs::read_to_string(marks_dir.join("git-marks")) {
	1725	for line in text.lines() {
	1726	if let Some((m, sha)) = line.split_once(' ') {
	1727	if let Ok(m) = m.trim_start_matches(':').parse::<u64>() {
	1728	sha_of_mark.insert(m, sha.trim().to_string());
	1729	}
	1730	}
	1731	}
	1732	}
	1733	let mut exported = HashMap::new();
	1734	for (state_ref, m) in oxy_marks {
	1735	if let Some(sha) = sha_of_mark.get(&m) {
	1736	exported.insert(sha.clone(), state_ref);
c3069537 Andy McCall 1 month ago ↶	1737	}
c3069537 Andy McCall 1 month ago ↶	1738	}
30ea9c1e Andy McCall 1 month ago ↶	1739	for (state_ref, sha) in Self::read_git_states(&marks_dir.join("git-states"))? {
	1740	exported.entry(sha).or_insert(state_ref);
	1741	}
	1742	Ok(GitKnown { imported, exported })
c3069537 Andy McCall 1 month ago ↶	1743	}
c3069537 Andy McCall 1 month ago ↶	1744
79ab8528 Andy McCall 1 month ago ↶	1745	fn git_author(&self, c: &Change) -> String {
79ab8528 Andy McCall 1 month ago ↶	1746	match self.store.get_identity(&c.author) {
75167672 Andy McCall 1 month ago	1747	Ok(id) if !id.name.is_empty() => format!(
	1748	"{} <{}>",
	1749	id.name.replace(['<', '>', '\n'], " "),
	1750	id.email.replace(['<', '>', '\n', ' '], "")
	1751	),
79ab8528 Andy McCall 1 month ago ↶	1752	_ => "oxy <oxy@localhost>".to_string(),
	1753	}
	1754	}
	1755
	1756	#[allow(clippy::too_many_arguments)]
75167672 Andy McCall 1 month ago	1757	fn write_commit(
	1758	&self,
	1759	out: &mut Vec<u8>,
	1760	reference: &str,
	1761	mark: u64,
	1762	author: &str,
	1763	secs: u64,
	1764	message: &str,
	1765	parents: &[String],
	1766	snapshot: &Snapshot,
	1767	) -> Result<()> {
79ab8528 Andy McCall 1 month ago ↶	1768	out.extend_from_slice(format!("commit {reference}\nmark :{mark}\nauthor {author} {secs} +0000\ncommitter {author} {secs} +0000\ndata {}\n{message}\n", message.len()).as_bytes());
79ab8528 Andy McCall 1 month ago ↶	1769	for (i, p) in parents.iter().enumerate() {
75167672 Andy McCall 1 month ago	1770	out.extend_from_slice(
	1771	format!("{} {p}\n", if i == 0 { "from" } else { "merge" }).as_bytes(),
	1772	);
79ab8528 Andy McCall 1 month ago ↶	1773	}
79ab8528 Andy McCall 1 month ago ↶	1774	out.extend_from_slice(b"deleteall\n");
4dee277c Andy McCall 1 month ago ↶	1775	let has_gitmodules = snapshot.files.contains_key(".gitmodules");
79ab8528 Andy McCall 1 month ago ↶	1776	for (path, f) in &snapshot.files {
4dee277c Andy McCall 1 month ago ↶	1777	if path == ".oxymodules" {
	1778	// Git reads `.gitmodules`; derive it unless the tree has one.
	1779	if !has_gitmodules {
	1780	if let FileContent::Text { bytes, .. } = &f.content {
75167672 Andy McCall 1 month ago	1781	let gm = crate::subrepo::Modules::parse(&String::from_utf8_lossy(bytes))
75167672 Andy McCall 1 month ago	1782	.to_gitmodules();
4dee277c Andy McCall 1 month ago ↶	1783	if !gm.is_empty() {
75167672 Andy McCall 1 month ago	1784	out.extend_from_slice(
	1785	format!("M 100644 inline .gitmodules\ndata {}\n", gm.len())
	1786	.as_bytes(),
	1787	);
4dee277c Andy McCall 1 month ago ↶	1788	out.extend_from_slice(gm.as_bytes());
	1789	out.push(b'\n');
	1790	}
	1791	}
	1792	}
	1793	continue;
	1794	}
	1795	if f.kind == NodeKind::Subrepo {
	1796	if let FileContent::Opaque { blobs } = &f.content {
	1797	if let Some(b) = blobs.first() {
	1798	match crate::subrepo::Pin::decode(&self.read_blob(b)?)? {
75167672 Andy McCall 1 month ago	1799	crate::subrepo::Pin::Git { sha, .. } => {
	1800	out.extend_from_slice(format!("M 160000 {sha} {path}\n").as_bytes())
	1801	}
	1802	crate::subrepo::Pin::Oxy { .. } => self
	1803	.git_export_skipped
	1804	.set(self.git_export_skipped.get() + 1),
4dee277c Andy McCall 1 month ago ↶	1805	}
	1806	}
	1807	}
	1808	continue;
	1809	}
79ab8528 Andy McCall 1 month ago ↶	1810	let (mode, bytes): (&str, Vec<u8>) = match (&f.kind, &f.content) {
79ab8528 Andy McCall 1 month ago ↶	1811	(NodeKind::Dir, _) => continue,
75167672 Andy McCall 1 month ago	1812	(NodeKind::Symlink, FileContent::Opaque { blobs }) => (
	1813	"120000",
	1814	blobs
	1815	.first()
	1816	.map(\|b\| self.read_blob(b))
	1817	.transpose()?
	1818	.unwrap_or_default(),
	1819	),
	1820	(_, FileContent::Text { bytes, .. }) => {
	1821	(if f.mode.exec { "100755" } else { "100644" }, bytes.clone())
	1822	}
	1823	(_, FileContent::Opaque { blobs }) => (
	1824	if f.mode.exec { "100755" } else { "100644" },
	1825	blobs
	1826	.first()
	1827	.map(\|b\| self.read_blob(b))
	1828	.transpose()?
	1829	.unwrap_or_default(),
	1830	),
79ab8528 Andy McCall 1 month ago ↶	1831	(_, FileContent::Dir) => continue,
79ab8528 Andy McCall 1 month ago ↶	1832	};
75167672 Andy McCall 1 month ago	1833	out.extend_from_slice(
	1834	format!("M {mode} inline {path}\ndata {}\n", bytes.len()).as_bytes(),
	1835	);
79ab8528 Andy McCall 1 month ago ↶	1836	out.extend_from_slice(&bytes);
	1837	out.push(b'\n');
	1838	}
	1839	out.push(b'\n');
	1840	Ok(())
	1841	}
	1842	}
62794ecc Andy McCall 1 month ago ↶	1843
	1844	/// What `verify_git_import` found (spec §23: import must reproduce every tree).
	1845	#[derive(Debug, Clone, Default, PartialEq, Eq)]
	1846	pub struct GitVerifyReport {
	1847	pub commits: usize,
7eac74ab Andy McCall 1 month ago ↶	1848	/// Commits followed but not rebuilt (all but the last few, when asked).
7eac74ab Andy McCall 1 month ago ↶	1849	pub skipped: usize,
62794ecc Andy McCall 1 month ago ↶	1850	pub files_compared: usize,
	1851	/// `(commit, what differs)`; empty when every commit's tree is reproduced.
	1852	pub mismatches: Vec<(String, String)>,
f0406c3b Andy McCall 27 days ago ↶	1853	/// Commits of the branch after the last imported one: an import not
	1854	/// finished yet. Not checked.
	1855	pub not_yet: usize,
62794ecc Andy McCall 1 month ago ↶	1856	}
	1857
	1858	impl Repo {
	1859	/// Check an import against its Git source: for every commit of
	1860	/// `branch`, the state formed by the changes imported for it and all
	1861	/// its ancestors must materialize to exactly that commit's tree, byte
	1862	/// for byte, with the same executable bits, symlink targets and
	1863	/// submodule pins. Reads only; stops listing after 50 differences.
	1864	pub fn verify_git_import(&self, git_dir: &Path, branch: &str) -> Result<GitVerifyReport> {
8b7986dd Andy McCall 27 days ago ↶	1865	self.verify_git_import_last(git_dir, branch, None, 50)
7eac74ab Andy McCall 1 month ago ↶	1866	}
	1867
	1868	/// [`Repo::verify_git_import`] of the last `last` commits only (all
	1869	/// when `None`): earlier ones are only followed, not rebuilt, for
	1870	/// looking again at where an import went wrong.
8b7986dd Andy McCall 27 days ago ↶	1871	/// It stops after `max` differences (0: never), so one that repeats
8b7986dd Andy McCall 27 days ago ↶	1872	/// through history does not hide the commits after it.
7eac74ab Andy McCall 1 month ago ↶	1873	pub fn verify_git_import_last(
	1874	&self,
	1875	git_dir: &Path,
	1876	branch: &str,
	1877	last: Option<usize>,
8b7986dd Andy McCall 27 days ago ↶	1878	max: usize,
7eac74ab Andy McCall 1 month ago ↶	1879	) -> Result<GitVerifyReport> {
f0406c3b Andy McCall 27 days ago ↶	1880	let mut rep = GitVerifyReport::default();
f0406c3b Andy McCall 27 days ago ↶	1881	let mut commits = list_commits(git_dir, branch)?;
62794ecc Andy McCall 1 month ago ↶	1882	let imported = self.imported_commits()?;
f0406c3b Andy McCall 27 days ago ↶	1883	// An import still going (or stopped) has reached part of the
	1884	// branch: what follows its last imported commit is not checked,
	1885	// only counted, and --last counts back from there (oxygen#81).
	1886	if let Some(end) = commits.iter().rposition(\|c\| imported.contains_key(&c.sha)) {
	1887	let rest = commits.len() - end - 1;
	1888	if rest > 0 {
	1889	commits.truncate(end + 1);
	1890	}
	1891	rep.not_yet = rest;
	1892	}
62794ecc Andy McCall 1 month ago ↶	1893	let mut cat = CatFile::start(git_dir)?;
	1894	// Changes of each commit and all its ancestors, built up in topological order.
	1895	let mut closure: HashMap<String, Arc<BTreeSet<ChangeHash>>> = HashMap::new();
a87c7b9d Andy McCall 1 month ago ↶	1896	// Graphs (by key) and blobs already found equal to a Git blob.
	1897	let mut verified_graphs: HashMap<[u8; 32], String> = HashMap::new();
	1898	let mut verified_blobs: HashMap<ObjectRef, String> = HashMap::new();
57d8dc05 Andy McCall 1 month ago ↶	1899	// The state of a commit whose first-parent children are still to
	1900	// come, so a child adds its own changes instead of loading all its
	1901	// history again (as the import does).
	1902	let mut waiting: HashMap<&str, usize> = HashMap::new();
ee6aa3e8 Andy McCall 28 days ago ↶	1903	// Children still to come per commit, through any parent: a commit's
	1904	// closure is dropped after its last child, so only the open
	1905	// frontier is held. Kept for every commit, the closures grew with
	1906	// the square of the history and VS Code's verify ran out of memory
	1907	// (oxygen#81).
	1908	let mut children: HashMap<&str, usize> = HashMap::new();
62794ecc Andy McCall 1 month ago ↶	1909	for c in &commits {
57d8dc05 Andy McCall 1 month ago ↶	1910	if let Some(p1) = c.parents.first() {
	1911	*waiting.entry(p1.as_str()).or_default() += 1;
	1912	}
ee6aa3e8 Andy McCall 28 days ago ↶	1913	for p in &c.parents {
	1914	*children.entry(p.as_str()).or_default() += 1;
	1915	}
57d8dc05 Andy McCall 1 month ago ↶	1916	}
	1917	let mut kept: HashMap<String, State> = HashMap::new();
	1918	let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
	1919	for c in &commits {
62794ecc Andy McCall 1 month ago ↶	1920	rep.commits += 1;
ee6aa3e8 Andy McCall 28 days ago ↶	1921	// The first parent's closure, taken over when this is its last
ee6aa3e8 Andy McCall 28 days ago ↶	1922	// child (no copy), then the other parents' added.
62794ecc Andy McCall 1 month ago ↶	1923	let mut set: BTreeSet<ChangeHash> = BTreeSet::new();
ee6aa3e8 Andy McCall 28 days ago ↶	1924	for (i, p) in c.parents.iter().enumerate() {
	1925	let left = children.get_mut(p.as_str()).map(\|n\| {
	1926	*n -= 1;
	1927	*n
	1928	});
	1929	let ps = if left == Some(0) {
	1930	closure.remove(p)
	1931	} else {
	1932	closure.get(p).cloned()
	1933	};
	1934	if let Some(ps) = ps {
	1935	if i == 0 && set.is_empty() {
	1936	set = Arc::try_unwrap(ps).unwrap_or_else(\|a\| (*a).clone());
	1937	} else {
	1938	set.extend(ps.iter().copied());
	1939	}
62794ecc Andy McCall 1 month ago ↶	1940	}
	1941	}
	1942	match imported.get(&c.sha) {
	1943	Some(h) => {
	1944	set.insert(*h);
	1945	}
	1946	None => {
	1947	rep.mismatches.push((
	1948	c.sha.clone(),
	1949	"no imported change carries this commit".into(),
	1950	));
	1951	}
	1952	}
	1953	let set = Arc::new(set);
ee6aa3e8 Andy McCall 28 days ago ↶	1954	if children.get(c.sha.as_str()).is_some_and(\|n\| *n > 0) {
	1955	closure.insert(c.sha.clone(), set.clone());
	1956	}
8b7986dd Andy McCall 27 days ago ↶	1957	if max > 0 && rep.mismatches.len() >= max {
62794ecc Andy McCall 1 month ago ↶	1958	break;
7eac74ab Andy McCall 1 month ago ↶	1959	}
	1960	if last.is_some_and(\|n\| rep.commits + n <= commits.len()) {
	1961	rep.skipped += 1;
	1962	continue;
62794ecc Andy McCall 1 month ago ↶	1963	}
62794ecc Andy McCall 1 month ago ↶	1964	let heads: Vec<ChangeHash> = set.iter().copied().collect();
a87c7b9d Andy McCall 1 month ago ↶	1965	// History only: a file whose graph (by key) or blob was already
	1966	// found equal to this Git blob is not built, rendered or read
	1967	// again; the rest are.
57d8dc05 Andy McCall 1 month ago ↶	1968	let from_parent = c.parents.first().and_then(\|p1\| {
	1969	let left = waiting.get_mut(p1.as_str()).map(\|n\| {
	1970	*n -= 1;
	1971	*n
	1972	});
	1973	if left == Some(0) {
	1974	kept_order.retain(\|k\| k != p1);
	1975	kept.remove(p1.as_str())
	1976	} else {
	1977	kept.get(p1.as_str()).cloned()
	1978	}
	1979	});
	1980	let state = match from_parent {
	1981	Some(mut st) => {
	1982	// The changes the parent's state lacks. The state never
	1983	// builds a graph (they are built beside it), so they
	1984	// are read without their text.
	1985	for h in set.iter() {
	1986	if !st.contains(h) {
	1987	let (change, texts) = self.store.get_change_lazy(h)?;
	1988	st.add_known(*h, change, texts);
	1989	}
	1990	}
	1991	st
	1992	}
	1993	None => self.load_state_history(&heads)?,
	1994	};
a87c7b9d Andy McCall 1 month ago ↶	1995	let view = state.tree().view();
62794ecc Andy McCall 1 month ago ↶	1996	let tree = list_tree(git_dir, &c.sha)?;
	1997	let short = &c.sha[..12.min(c.sha.len())];
	1998	let mut differ = \|what: String\| rep.mismatches.push((c.sha.clone(), what));
a87c7b9d Andy McCall 1 month ago ↶	1999	let mut conflicted: Vec<String> =
	2000	view.conflicts.iter().map(\|c\| format!("{c:?}")).collect();
	2001	let mut need: Vec<NodeId> = Vec::new();
	2002	for (path, (mode, sha)) in &tree {
	2003	let Some(e) = view.entries.get(path) else {
	2004	continue;
	2005	};
	2006	if mode != "160000"
	2007	&& mode != "120000"
	2008	&& e.kind == NodeKind::File
	2009	&& !e.mode.opaque
	2010	{
	2011	let known = state
	2012	.graph_key(&e.node)
	2013	.is_some_and(\|k\| verified_graphs.get(&k) == Some(sha));
	2014	if !known {
	2015	need.push(e.node);
62794ecc Andy McCall 1 month ago ↶	2016	}
	2017	}
	2018	}
a87c7b9d Andy McCall 1 month ago ↶	2019	// A graph's key is known only before it is built.
	2020	let keys: HashMap<NodeId, [u8; 32]> = need
	2021	.iter()
	2022	.filter_map(\|n\| Some((*n, state.graph_key(n)?)))
	2023	.collect();
	2024	let need_set: HashSet<NodeId> = need.iter().copied().collect();
57d8dc05 Andy McCall 1 month ago ↶	2025	let graphs = self.build_graphs_detached(&state, &need)?;
62794ecc Andy McCall 1 month ago ↶	2026	for (path, (mode, sha)) in &tree {
a87c7b9d Andy McCall 1 month ago ↶	2027	let Some(e) = view.entries.get(path) else {
62794ecc Andy McCall 1 month ago ↶	2028	differ(format!("{short}: {path} is missing"));
	2029	continue;
	2030	};
	2031	rep.files_compared += 1;
	2032	match mode.as_str() {
a87c7b9d Andy McCall 1 month ago ↶	2033	"160000" => {
	2034	let blobs = state.live_blobs(&e.node);
	2035	match (e.kind, blobs.as_slice()) {
	2036	(NodeKind::Subrepo, [b]) => {
	2037	let pin = crate::subrepo::Pin::decode(&self.read_blob(b)?)?;
	2038	if !matches!(&pin, crate::subrepo::Pin::Git { sha: s, .. } if s == sha)
	2039	{
	2040	differ(format!("{short}: {path} pins another commit"));
	2041	}
62794ecc Andy McCall 1 month ago ↶	2042	}
a87c7b9d Andy McCall 1 month ago ↶	2043	(kind, _) => differ(format!(
	2044	"{short}: {path} is not one submodule pin: {kind:?} with {} pins",
	2045	blobs.len()
	2046	)),
62794ecc Andy McCall 1 month ago ↶	2047	}
a87c7b9d Andy McCall 1 month ago ↶	2048	}
62794ecc Andy McCall 1 month ago ↶	2049	"120000" => {
62794ecc Andy McCall 1 month ago ↶	2050	let target = cat.read(sha)?;
a87c7b9d Andy McCall 1 month ago ↶	2051	let blobs = state.live_blobs(&e.node);
	2052	match (e.kind, blobs.as_slice()) {
	2053	(NodeKind::Symlink, [b]) if self.read_blob(b)? == target => {}
62794ecc Andy McCall 1 month ago ↶	2054	_ => differ(format!("{short}: symlink {path} differs")),
	2055	}
	2056	}
	2057	_ => {
a87c7b9d Andy McCall 1 month ago ↶	2058	if e.mode.exec != (mode == "100755") {
	2059	differ(format!("{short}: {path} executable bit differs"));
	2060	}
	2061	if e.kind == NodeKind::File && !e.mode.opaque {
	2062	if !need_set.contains(&e.node) {
	2063	// Known equal: skipped when the graphs were chosen.
62794ecc Andy McCall 1 month ago ↶	2064	continue;
62794ecc Andy McCall 1 month ago ↶	2065	}
57d8dc05 Andy McCall 1 month ago ↶	2066	let (have, m) = match graphs.get(&e.node) {
	2067	Some(g) => state.render_graph(g),
	2068	None => (Vec::new(), Default::default()),
	2069	};
a87c7b9d Andy McCall 1 month ago ↶	2070	if m.is_conflicted() {
57d8dc05 Andy McCall 1 month ago ↶	2071	// Who conflicts: a copy of the state with this graph built.
	2072	let mut detail = state.clone();
	2073	self.load_graphs_of(&mut detail, &[e.node])?;
	2074	let who: Vec<String> = detail
a87c7b9d Andy McCall 1 month ago ↶	2075	.conflict_regions(&e.node)
	2076	.contributors
	2077	.iter()
	2078	.filter_map(\|h\| state.get(h))
	2079	.map(\|m\| {
	2080	let sha = m
	2081	.message
	2082	.lines()
	2083	.rev()
	2084	.find_map(\|l\| l.strip_prefix("Git-Commit: "))
	2085	.unwrap_or("?");
	2086	format!(
	2087	"{} {:?}",
	2088	&sha[..12.min(sha.len())],
	2089	m.message.lines().next().unwrap_or("")
	2090	)
	2091	})
	2092	.collect();
	2093	conflicted.push(format!("{path} between {}", who.join(" and ")));
	2094	}
	2095	let want = cat.read(sha)?;
	2096	if have != want {
	2097	differ(format!(
	2098	"{short}: {path} differs: {}",
	2099	first_difference(&have, &want)
	2100	));
	2101	} else if !m.is_conflicted() {
	2102	if let Some(k) = keys.get(&e.node) {
	2103	verified_graphs.insert(*k, sha.clone());
	2104	}
	2105	}
	2106	continue;
	2107	}
	2108	let blobs = state.live_blobs(&e.node);
	2109	let [b] = blobs.as_slice() else {
	2110	differ(format!("{short}: {path} is not one file"));
	2111	continue;
62794ecc Andy McCall 1 month ago ↶	2112	};
a87c7b9d Andy McCall 1 month ago ↶	2113	if verified_blobs.get(b) == Some(sha) {
a87c7b9d Andy McCall 1 month ago ↶	2114	continue;
62794ecc Andy McCall 1 month ago ↶	2115	}
a87c7b9d Andy McCall 1 month ago ↶	2116	if self.read_blob(b)? != cat.read(sha)? {
	2117	differ(format!("{short}: {path} differs"));
	2118	} else {
	2119	verified_blobs.insert(*b, sha.clone());
62794ecc Andy McCall 1 month ago ↶	2120	}
	2121	}
	2122	}
	2123	}
a87c7b9d Andy McCall 1 month ago ↶	2124	if !conflicted.is_empty() {
	2125	differ(format!(
	2126	"{short}: the state is conflicted: {}",
	2127	conflicted.join("; ")
	2128	));
	2129	}
62794ecc Andy McCall 1 month ago ↶	2130	// Import records submodules in `.oxymodules` as well (spec §23),
	2131	// and export leaves it out again.
	2132	let has_submodules = tree.values().any(\|(m, _)\| m == "160000");
e83354cd Andy McCall 1 month ago ↶	2133	for (path, e) in view.entries.iter() {
a87c7b9d Andy McCall 1 month ago ↶	2134	if e.kind != NodeKind::Dir
62794ecc Andy McCall 1 month ago ↶	2135	&& !tree.contains_key(path)
	2136	&& !(has_submodules && path == ".oxymodules")
	2137	{
	2138	differ(format!("{short}: {path} is extra"));
57d8dc05 Andy McCall 1 month ago ↶	2139	}
	2140	}
	2141	if waiting.get(c.sha.as_str()).is_some_and(\|n\| *n > 0) {
	2142	kept.insert(c.sha.clone(), state);
	2143	kept_order.push_back(c.sha.clone());
	2144	if kept_order.len() > 8 {
	2145	if let Some(old) = kept_order.pop_front() {
	2146	kept.remove(&old);
	2147	}
62794ecc Andy McCall 1 month ago ↶	2148	}
62794ecc Andy McCall 1 month ago ↶	2149	}
8b7986dd Andy McCall 27 days ago ↶	2150	if max > 0 && rep.mismatches.len() >= max {
62794ecc Andy McCall 1 month ago ↶	2151	break;
	2152	}
	2153	}
	2154	Ok(rep)
	2155	}
	2156	}
	2157
	2158	/// Where two versions of a file first part, for a verification report.
	2159	fn first_difference(have: &[u8], want: &[u8]) -> String {
	2160	let (h, w): (Vec<&[u8]>, Vec<&[u8]>) = (
	2161	have.split(\|b\| *b == b'\n').collect(),
	2162	want.split(\|b\| *b == b'\n').collect(),
	2163	);
	2164	let i = h
	2165	.iter()
	2166	.zip(&w)
	2167	.position(\|(a, b)\| a != b)
	2168	.unwrap_or(h.len().min(w.len()));
	2169	let show = \|l: Option<&&[u8]>\| {
	2170	l.map(\|l\| String::from_utf8_lossy(&l[..l.len().min(60)]).into_owned())
	2171	.unwrap_or_else(\|\| "(end)".into())
	2172	};
	2173	format!(
	2174	"line {}: has {:?}, Git has {:?} ({} lines, Git has {})",
	2175	i + 1,
	2176	show(h.get(i)),
	2177	show(w.get(i)),
	2178	h.len(),
	2179	w.len()
	2180	)
	2181	}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

Keyboard