crates/oxy-core/src/repo/git.rs at main · andymccall/oxygen

◆ main oxygen / crates / oxy-core / src / repo / git.rs

104.6 KiB · Rust · last changed oxy git colocate: a .git beside .oxy, kept up to date after every command; .git is never content and never written by checkout (closes #160) 4 days ago source blame history raw

79ab8528 Andy McCall 1 month ago ↶	1	//! Git bridge (spec §23), export side: each state the current bookmark
	2	//! passed through becomes a Git commit through `git fast-import`, with the
	3	//! state's full tree and the previous state's commit as parent. A state
	4	//! that added exactly one change carries that change's message, author,
	5	//! and `Change-Id`/`Oxy-Hash` trailers; every commit carries `Oxy-State`.
	6	//!
	7	//! Marks make re-exports incremental: `.oxy/git/<target>/git-marks` is
	8	//! git's own marks file and `oxy-marks` maps state refs to marks.
	9
	10	use super::*;
	11	use std::process::{Command, Stdio};
62794ecc Andy McCall 1 month ago ↶	12	use std::sync::Arc;
79ab8528 Andy McCall 1 month ago ↶	13
79ab8528 Andy McCall 1 month ago ↶	14	#[derive(Debug, Clone, Default, PartialEq, Eq)]
c3069537 Andy McCall 1 month ago ↶	15	pub struct GitImportReport {
	16	pub commits: usize,
	17	pub merges: usize,
4dee277c Andy McCall 1 month ago ↶	18	/// Gitlink entries imported as subrepo nodes with Git pins.
4dee277c Andy McCall 1 month ago ↶	19	pub submodules: usize,
c3069537 Andy McCall 1 month ago ↶	20	pub heads: usize,
	21	}
	22
	23	/// A running `git cat-file --batch`, for reading many blobs cheaply.
	24	struct CatFile {
	25	child: std::process::Child,
	26	stdin: std::process::ChildStdin,
	27	stdout: std::io::BufReader<std::process::ChildStdout>,
	28	}
	29
	30	impl CatFile {
	31	fn start(git_dir: &Path) -> Result<CatFile> {
	32	let mut child = Command::new("git")
	33	.arg("-C")
	34	.arg(git_dir)
	35	.args(["cat-file", "--batch"])
	36	.stdin(Stdio::piped())
	37	.stdout(Stdio::piped())
	38	.stderr(Stdio::null())
	39	.spawn()
	40	.map_err(\|e\| RepoError::Msg(format!("running git cat-file: {e}")))?;
	41	let stdin = child.stdin.take().unwrap();
	42	let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
75167672 Andy McCall 1 month ago	43	Ok(CatFile {
	44	child,
	45	stdin,
	46	stdout,
	47	})
c3069537 Andy McCall 1 month ago ↶	48	}
	49
	50	fn read(&mut self, sha: &str) -> Result<Vec<u8>> {
	51	use std::io::{BufRead, Read};
	52	writeln!(self.stdin, "{sha}")?;
	53	self.stdin.flush()?;
	54	let mut header = String::new();
	55	self.stdout.read_line(&mut header)?;
	56	let parts: Vec<&str> = header.trim_end().split(' ').collect();
	57	if parts.len() != 3 {
	58	return fail(format!("git cat-file: object {sha} not found"));
	59	}
75167672 Andy McCall 1 month ago	60	let size: usize = parts[2]
	61	.parse()
	62	.map_err(\|_\| RepoError::Msg("git cat-file: bad size".into()))?;
c3069537 Andy McCall 1 month ago ↶	63	let mut body = vec![0u8; size];
	64	self.stdout.read_exact(&mut body)?;
	65	let mut nl = [0u8; 1];
	66	self.stdout.read_exact(&mut nl)?;
	67	Ok(body)
	68	}
	69	}
	70
	71	impl Drop for CatFile {
	72	fn drop(&mut self) {
	73	let _ = self.child.kill();
	74	let _ = self.child.wait();
	75	}
	76	}
	77
2306a4d2 Andy McCall 1 month ago ↶	78	/// One path a commit changes against its first parent: its new mode and
	79	/// blob, or `None` when the commit deletes it.
	80	type TreeDiff = Vec<(String, Option<(String, String)>)>;
	81
	82	/// A running `git diff-tree --stdin`, fed every `commit parent` pair up
	83	/// front, whose diffs are read in the same order as they are needed: a
	84	/// commit's tree is its first parent's plus its diff, at a cost that
	85	/// follows what it changes rather than the whole tree (oxygen#25).
	86	struct DiffTree {
	87	child: std::process::Child,
	88	stdout: std::io::BufReader<std::process::ChildStdout>,
	89	feeder: Option<std::thread::JoinHandle<()>>,
	90	/// The header read at the end of the previous diff.
	91	next: Option<String>,
	92	}
	93
	94	impl DiffTree {
	95	fn start(git_dir: &Path, pairs: Vec<(String, String)>) -> Result<DiffTree> {
	96	let mut child = Command::new("git")
	97	.arg("-C")
	98	.arg(git_dir)
	99	.args([
	100	"diff-tree",
	101	"--stdin",
	102	"--always",
	103	"-r",
	104	"-z",
	105	"--no-renames",
	106	])
	107	.stdin(Stdio::piped())
	108	.stdout(Stdio::piped())
	109	.stderr(Stdio::null())
	110	.spawn()
	111	.map_err(\|e\| RepoError::Msg(format!("running git diff-tree: {e}")))?;
	112	let mut stdin = child.stdin.take().unwrap();
	113	let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
	114	// A thread writes, so git never waits on a full pipe for us to read.
	115	let feeder = std::thread::spawn(move \|\| {
f0406c3b Andy McCall 27 days ago ↶	116	// An import that stops before reading every diff kills git
	117	// while this thread may still be writing. oxy restores
	118	// SIGPIPE's default (to end quietly under `\| head`), so that
	119	// write would kill the whole process (signal 13, oxygen#81);
	120	// blocked here, it fails with EPIPE and the thread ends.
	121	#[cfg(unix)]
	122	unsafe {
	123	let mut set: libc::sigset_t = std::mem::zeroed();
	124	libc::sigemptyset(&mut set);
	125	libc::sigaddset(&mut set, libc::SIGPIPE);
	126	libc::pthread_sigmask(libc::SIG_BLOCK, &set, std::ptr::null_mut());
	127	}
2306a4d2 Andy McCall 1 month ago ↶	128	let mut w = std::io::BufWriter::new(&mut stdin);
	129	for (c, p) in pairs {
	130	if writeln!(w, "{c} {p}").is_err() {
	131	return;
	132	}
	133	}
	134	let _ = w.flush();
	135	});
	136	Ok(DiffTree {
	137	child,
	138	stdout,
	139	feeder: Some(feeder),
	140	next: None,
	141	})
	142	}
	143
	144	fn field(&mut self) -> Result<Option<String>> {
	145	use std::io::BufRead;
	146	let mut buf = Vec::new();
	147	if self.stdout.read_until(0, &mut buf)? == 0 {
	148	return Ok(None);
	149	}
	150	if buf.last() == Some(&0) {
	151	buf.pop();
	152	}
	153	Ok(Some(String::from_utf8_lossy(&buf).into_owned()))
	154	}
	155
	156	/// The diff of `sha` against the parent it was fed with.
	157	fn diff(&mut self, sha: &str) -> Result<TreeDiff> {
	158	let header = match self.next.take() {
	159	Some(h) => Some(h),
	160	None => self.field()?,
	161	};
	162	if header.as_deref() != Some(sha) {
	163	return fail(format!("git diff-tree: expected {sha}, got {header:?}"));
	164	}
	165	let mut out = Vec::new();
	166	while let Some(f) = self.field()? {
	167	let Some(meta) = f.strip_prefix(':') else {
	168	self.next = Some(f);
	169	break;
	170	};
	171	let Some(path) = self.field()? else {
	172	return fail("git diff-tree: record without a path");
	173	};
	174	// `oldmode newmode oldsha newsha status`
	175	let m: Vec<&str> = meta.split(' ').collect();
	176	if m.len() != 5 {
	177	return fail(format!("git diff-tree: bad record {f:?}"));
	178	}
	179	let entry = if m[4].starts_with('D') {
	180	None
	181	} else {
	182	Some((m[1].to_string(), m[3].to_string()))
	183	};
	184	out.push((path, entry));
	185	}
	186	Ok(out)
	187	}
	188	}
	189
	190	impl Drop for DiffTree {
	191	fn drop(&mut self) {
192	let _ = self.child.kill();
193	let _ = self.child.wait();
194	if let Some(f) = self.feeder.take() {
195	let _ = f.join();
196	}
197	}
198	}
199
d6e0d5b8 Andy McCall 25 days ago ↶	200	/// The email of a Git author `Name <email>`, lowercased.
	201	fn git_email(author: &str) -> Option<String> {
	202	let (_, rest) = author.rsplit_once('<')?;
	203	let email = rest.strip_suffix('>')?.trim();
	204	(!email.is_empty()).then(\|\| email.to_lowercase())
	205	}
	206
c3069537 Andy McCall 1 month ago ↶	207	/// One commit as `git log` describes it.
	208	struct GitCommit {
	209	sha: String,
	210	parents: Vec<String>,
	211	author_secs: u64,
	212	author: String,
	213	message: String,
	214	}
	215
	216	fn list_commits(git_dir: &Path, branch: &str) -> Result<Vec<GitCommit>> {
	217	let out = Command::new("git")
	218	.arg("-C")
	219	.arg(git_dir)
75167672 Andy McCall 1 month ago	220	.args([
	221	"log",
	222	"--reverse",
	223	"--topo-order",
	224	"--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e",
	225	branch,
	226	])
c3069537 Andy McCall 1 month ago ↶	227	.output()
	228	.map_err(\|e\| RepoError::Msg(format!("running git log: {e}")))?;
	229	if !out.status.success() {
75167672 Andy McCall 1 month ago	230	return fail(format!(
	231	"git log {branch} failed: {}",
	232	String::from_utf8_lossy(&out.stderr).trim()
	233	));
c3069537 Andy McCall 1 month ago ↶	234	}
	235	let text = String::from_utf8_lossy(&out.stdout);
	236	let mut commits = Vec::new();
	237	for record in text.split('\x1e') {
	238	let record = record.trim_start_matches('\n');
	239	if record.trim().is_empty() {
	240	continue;
	241	}
	242	let f: Vec<&str> = record.splitn(5, '\0').collect();
	243	if f.len() != 5 {
	244	return fail("unexpected git log record");
	245	}
	246	commits.push(GitCommit {
	247	sha: f[0].to_string(),
	248	parents: f[1].split_whitespace().map(str::to_string).collect(),
	249	author_secs: f[2].parse().unwrap_or(0),
	250	author: f[3].to_string(),
	251	message: f[4].trim_end().to_string(),
	252	});
	253	}
	254	Ok(commits)
	255	}
	256
	257	/// `path → (mode, blob sha)` for a commit's tree.
	258	fn list_tree(git_dir: &Path, sha: &str) -> Result<BTreeMap<String, (String, String)>> {
	259	let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?;
	260	let mut out = BTreeMap::new();
	261	for entry in text.split('\0') {
	262	if entry.is_empty() {
	263	continue;
	264	}
75167672 Andy McCall 1 month ago	265	let Some((meta, path)) = entry.split_once('\t') else {
	266	continue;
	267	};
c3069537 Andy McCall 1 month ago ↶	268	let m: Vec<&str> = meta.split(' ').collect();
	269	if m.len() == 3 {
	270	out.insert(path.to_string(), (m[0].to_string(), m[2].to_string()));
	271	}
	272	}
	273	Ok(out)
	274	}
	275
	276	/// The ChangeId for a Git commit: its `Change-Id` trailer if it has a
	277	/// 32-hex one, else BLAKE3-128 of the commit hash (spec §23).
	278	fn change_id_for(sha: &str, message: &str) -> ChangeId {
	279	for line in message.lines().rev() {
	280	if let Some(v) = line.strip_prefix("Change-Id:") {
	281	if let Some(id) = ChangeId::from_hex(v.trim()) {
	282	return id;
	283	}
	284	}
	285	}
	286	let h = blake3::hash(sha.as_bytes());
	287	ChangeId(h.as_bytes()[..16].try_into().unwrap())
	288	}
	289
30ea9c1e Andy McCall 1 month ago ↶	290	/// Git commits with a known Oxygen counterpart.
	291	struct GitKnown {
	292	/// Imported commit → its change.
	293	imported: HashMap<String, ChangeHash>,
	294	/// Exported commit (or a fetched tip) → the bookmark state it equals.
	295	exported: HashMap<String, ObjectRef>,
	296	}
	297
c3069537 Andy McCall 1 month ago ↶	298	#[derive(Debug, Clone, Default, PartialEq, Eq)]
79ab8528 Andy McCall 1 month ago ↶	299	pub struct GitExportReport {
	300	pub exported: usize,
	301	pub skipped: usize,
4dee277c Andy McCall 1 month ago ↶	302	/// Subrepo nodes with Oxygen pins, which Git cannot represent.
4dee277c Andy McCall 1 month ago ↶	303	pub skipped_subrepos: usize,
79ab8528 Andy McCall 1 month ago ↶	304	pub branch: String,
	305	pub heads: usize,
	306	}
	307
fd280228 Andy McCall 1 month ago ↶	308	/// A merge bringing more commits than this sheds its base's graphs first.
fd280228 Andy McCall 1 month ago ↶	309	const MERGE_SHED_COMMITS: usize = 100;
d45dab04 Andy McCall 1 month ago ↶	310
	311	/// The atoms a kept state's built graphs may hold before they go back into
	312	/// its history (#25): about a few hundred megabytes.
	313	const KEPT_ATOMS: usize = 1_500_000;
fd280228 Andy McCall 1 month ago ↶	314	/// A kept state with more built graphs than this sheds them.
fd280228 Andy McCall 1 month ago ↶	315	const KEPT_GRAPHS: usize = 2000;
9935d490 Andy McCall 29 days ago ↶	316	/// An import moves its bookmark to the commits done so far every this
	317	/// many commits, so one that stops part way resumes from there: imported
	318	/// commits are found from the bookmark's history (`imported_commits`).
	319	const CHECKPOINT_COMMITS: usize = 2000;
fd280228 Andy McCall 1 month ago ↶	320
c3d2ef61 Andy McCall 1 month ago ↶	321	/// Text files of a merge rendered this many at a time (oxygen#25): a
	322	/// batch's graphs are built beside the state and dropped once rendered,
	323	/// so what is held for the whole tree is the renderings, not the graphs
	324	/// with every deleted line of their history.
	325	const MERGE_BATCH: usize = 500;
	326
	327	thread_local! {
	328	/// A smaller batch for this thread's imports: tests set 1, so a small
	329	/// merge takes the batched path.
	330	static MERGE_BATCH_OVERRIDE: std::cell::Cell<Option<usize>> = const { std::cell::Cell::new(None) };
	331	}
	332
	333	#[cfg(test)]
	334	pub(crate) fn set_merge_batch_for_tests(n: Option<usize>) {
	335	MERGE_BATCH_OVERRIDE.with(\|c\| c.set(n));
	336	}
	337
	338	fn merge_batch() -> usize {
	339	MERGE_BATCH_OVERRIDE
	340	.with(\|c\| c.get())
	341	.filter(\|n\| *n > 0)
33be2409 Andy McCall 1 month ago ↶	342	.or_else(\|\| {
	343	std::env::var("OXY_MERGE_BATCH")
	344	.ok()
	345	.and_then(\|v\| v.parse().ok())
	346	})
c3d2ef61 Andy McCall 1 month ago ↶	347	.unwrap_or(MERGE_BATCH)
	348	}
	349
	350	impl Repo {
	351	/// As `State::materialize_except`, for a tree whose graphs would not
	352	/// fit in memory at once. The few files the change builder reads
	353	/// graphs for (conflicted ones, and renderings whose lines are not
	354	/// one per live atom) are built in the state afterwards.
	355	fn materialize_batched(
	356	&self,
	357	base: &mut State,
	358	need: &[crate::ids::NodeId],
	359	skip: &HashSet<crate::ids::NodeId>,
	360	) -> Result<crate::state::Snapshot> {
	361	use crate::graph::Materialized;
	362	use crate::state::{FileContent, FileState};
	363	use rayon::prelude::*;
	364	let view = base.tree().view();
	365	let mut rendered: HashMap<crate::ids::NodeId, (Vec<u8>, Materialized)> =
	366	HashMap::with_capacity(need.len());
	367	let mut keep_built: Vec<crate::ids::NodeId> = Vec::new();
	368	for batch in need.chunks(merge_batch()) {
	369	let t = std::time::Instant::now();
	370	let graphs = self.build_graphs_detached(base, batch)?;
	371	crate::timing::add("render: build graphs, batch", t.elapsed());
	372	let t = std::time::Instant::now();
	373	let batch_rendered: Vec<(crate::ids::NodeId, Vec<u8>, Materialized)> = graphs
	374	.par_iter()
	375	.map(\|(node, g)\| {
	376	let (bytes, m) = base.render_graph(g);
	377	(*node, bytes, m)
	378	})
	379	.collect();
	380	drop(graphs);
	381	crate::timing::add("render: render, batch", t.elapsed());
	382	for (node, bytes, m) in batch_rendered {
	383	if m.is_conflicted()
	384	\|\| m.live_atoms().len() != crate::diff::split_lines(&bytes).len()
	385	{
	386	keep_built.push(node);
	387	}
	388	rendered.insert(node, (bytes, m));
	389	}
	390	}
	391	if !keep_built.is_empty() {
	392	self.load_graphs_of(base, &keep_built)?;
	393	}
	394	let mut files = BTreeMap::new();
	395	let mut unrendered = BTreeSet::new();
	396	for (path, e) in view.entries.iter() {
	397	let f = if e.kind == NodeKind::File && !e.mode.opaque {
	398	match rendered.remove(&e.node) {
	399	Some((bytes, materialized)) => FileState {
	400	node: e.node,
	401	kind: e.kind,
	402	mode: e.mode,
	403	content: FileContent::Text {
	404	bytes,
	405	materialized,
	406	},
	407	},
	408	None if skip.contains(&e.node) => {
	409	unrendered.insert(path.clone());
	410	FileState {
411	node: e.node,
412	kind: e.kind,
413	mode: e.mode,
414	content: FileContent::Text {
415	bytes: Vec::new(),
416	materialized: Materialized::default(),
417	},
418	}
419	}
420	None => {
421	// Neither asked for nor known unchanged: render it
422	// from a built graph, as the whole-tree path does.
423	self.load_graphs_of(base, &[e.node])?;
424	base.entry_state(e)
425	}
426	}
427	} else {
428	base.entry_state(e)
429	};
430	files.insert(path.clone(), f);
431	}
432	Ok(crate::state::Snapshot {
433	tree: view,
434	files,
435	unrendered,
436	})
437	}
438	}
439
79ab8528 Andy McCall 1 month ago ↶	440	fn git(dir: &Path, args: &[&str]) -> Result<String> {
75167672 Andy McCall 1 month ago	441	let out = Command::new("git")
	442	.arg("-C")
	443	.arg(dir)
	444	.args(args)
	445	.output()
	446	.map_err(\|e\| RepoError::Msg(format!("running git: {e}")))?;
79ab8528 Andy McCall 1 month ago ↶	447	if !out.status.success() {
75167672 Andy McCall 1 month ago	448	return fail(format!(
	449	"git {} failed: {}",
	450	args.join(" "),
	451	String::from_utf8_lossy(&out.stderr).trim()
	452	));
79ab8528 Andy McCall 1 month ago ↶	453	}
	454	Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
	455	}
	456
	457	impl Repo {
	458	fn git_marks_dir(&self, git_dir: &Path) -> Result<PathBuf> {
544d0541 Andy McCall 10 days ago ↶	459	let canonical =
544d0541 Andy McCall 10 days ago ↶	460	crate::portable::canonical(git_dir).unwrap_or_else(\|_\| git_dir.to_path_buf());
79ab8528 Andy McCall 1 month ago ↶	461	let key = hash_object(ObjectType::Chunk, canonical.to_string_lossy().as_bytes()).short(16);
	462	let dir = self.oxy.join("git").join(key);
	463	fs::create_dir_all(&dir)?;
	464	Ok(dir)
	465	}
	466
30ea9c1e Andy McCall 1 month ago ↶	467	/// `state ref → Git commit sha` for bookmark states that equal a Git
	468	/// commit's tree (recorded by import and fetch).
	469	fn read_git_states(path: &Path) -> Result<BTreeMap<ObjectRef, String>> {
	470	let mut out = BTreeMap::new();
	471	if let Ok(text) = fs::read_to_string(path) {
	472	for line in text.lines() {
	473	if let Some((h, sha)) = line.split_once(' ') {
	474	if let Some(h) = ObjectRef::from_hex(h) {
	475	out.insert(h, sha.trim().to_string());
	476	}
	477	}
	478	}
	479	}
	480	Ok(out)
	481	}
	482
	483	fn record_git_state(&self, git_dir: &Path, state: &ObjectRef, sha: &str) -> Result<()> {
	484	let path = self.git_marks_dir(git_dir)?.join("git-states");
	485	let mut states = Self::read_git_states(&path)?;
	486	states.insert(*state, sha.to_string());
75167672 Andy McCall 1 month ago	487	let text: String = states
	488	.iter()
	489	.map(\|(h, s)\| format!("{} {s}\n", h.to_hex()))
	490	.collect();
30ea9c1e Andy McCall 1 month ago ↶	491	write_atomic(&path, text.as_bytes())
	492	}
	493
	494	/// The bare mirror this repository keeps for the Git remote `url`.
	495	fn git_mirror(&self, url: &str) -> Result<PathBuf> {
	496	let key = hash_object(ObjectType::Chunk, url.as_bytes()).short(16);
	497	let dir = self.oxy.join("git").join(key).join("mirror.git");
	498	if !dir.join("HEAD").exists() {
	499	fs::create_dir_all(&dir)?;
	500	git(&dir, &["init", "--bare", "-q"])?;
	501	}
	502	Ok(dir)
	503	}
	504
	505	/// Git commits already imported into the bookmark, by sha.
7891e21c Andy McCall 1 month ago ↶	506	pub(crate) fn imported_commits(&self) -> Result<HashMap<String, ChangeHash>> {
30ea9c1e Andy McCall 1 month ago ↶	507	let mut map = HashMap::new();
	508	for e in self.log()? {
	509	if e.is_working_copy {
	510	continue;
	511	}
75167672 Andy McCall 1 month ago	512	if let Some(sha) = e
	513	.change
	514	.message
	515	.lines()
	516	.rev()
	517	.find_map(\|l\| l.strip_prefix("Git-Commit: "))
	518	{
30ea9c1e Andy McCall 1 month ago ↶	519	map.insert(sha.trim().to_string(), e.hash);
	520	}
	521	}
	522	Ok(map)
	523	}
	524
	525	/// Bring the mirror's `branch` up to date with the Git remote `url`.
	526	/// A remote without that branch yet leaves the mirror as it is.
	527	fn git_mirror_fetch(&self, mirror: &Path, url: &str, branch: &str) -> Result<()> {
	528	let spec = format!("+refs/heads/{branch}:refs/heads/{branch}");
	529	match git(mirror, &["fetch", "-q", url, &spec]) {
	530	Ok(_) => Ok(()),
	531	Err(e) => {
	532	let ls = git(mirror, &["ls-remote", "--heads", url, branch]).unwrap_or_default();
75167672 Andy McCall 1 month ago	533	if ls.trim().is_empty() {
	534	Ok(())
	535	} else {
	536	Err(e)
	537	}
30ea9c1e Andy McCall 1 month ago ↶	538	}
	539	}
	540	}
	541
	542	/// Fetch `branch` of the Git repository at `url` (a path or a Git URL)
	543	/// through a mirror and import the commits not yet in the bookmark
	544	/// (spec §23). Works on a repository that already has history: the
	545	/// new changes join the bookmark like a pull.
	546	pub fn git_fetch(&self, url: &str, branch: &str) -> Result<GitImportReport> {
9935d490 Andy McCall 29 days ago ↶	547	// A local path, made absolute: git runs it from inside the mirror.
544d0541 Andy McCall 10 days ago ↶	548	let local = crate::portable::canonical(Path::new(url))
9935d490 Andy McCall 29 days ago ↶	549	.ok()
	550	.filter(\|_\| !url.contains("://"));
	551	let absolute = local.as_ref().map(\|p\| p.to_string_lossy().into_owned());
	552	let url = absolute.as_deref().unwrap_or(url);
30ea9c1e Andy McCall 1 month ago ↶	553	let mirror = self.git_mirror(url)?;
30ea9c1e Andy McCall 1 month ago ↶	554	self.git_mirror_fetch(&mirror, url, branch)?;
75167672 Andy McCall 1 month ago	555	if git(
	556	&mirror,
	557	&[
	558	"rev-parse",
	559	"--verify",
	560	"-q",
	561	&format!("refs/heads/{branch}"),
	562	],
	563	)
	564	.is_err()
	565	{
30ea9c1e Andy McCall 1 month ago ↶	566	return fail(format!("{url} has no branch {branch:?}"));
30ea9c1e Andy McCall 1 month ago ↶	567	}
58dfe0e1 Andy McCall 1 month ago ↶	568	self.import_git_commits(
	569	&mirror,
	570	branch,
	571	&format!("oxy git fetch {url} {branch}"),
	572	&mut \|_\| {},
	573	)
30ea9c1e Andy McCall 1 month ago ↶	574	}
	575
	576	/// Export the bookmark to `branch` of the Git repository at `url`
	577	/// through the mirror and push it. The remote's tip must already have
	578	/// been fetched (`git_fetch`), so the push fast-forwards.
	579	pub fn git_push(&self, url: &str, branch: &str) -> Result<GitExportReport> {
	580	let mirror = self.git_mirror(url)?;
	581	self.git_mirror_fetch(&mirror, url, branch)?;
75167672 Andy McCall 1 month ago	582	if let Ok(tip) = git(
	583	&mirror,
	584	&[
	585	"rev-parse",
	586	"--verify",
	587	"-q",
	588	&format!("refs/heads/{branch}"),
	589	],
	590	) {
30ea9c1e Andy McCall 1 month ago ↶	591	let known = self.git_known(&mirror)?;
30ea9c1e Andy McCall 1 month ago ↶	592	if !known.imported.contains_key(&tip) && !known.exported.contains_key(&tip) {
75167672 Andy McCall 1 month ago	593	return fail(format!(
	594	"{url} {branch} has commits not yet fetched; run `oxy git fetch {url}` first"
	595	));
30ea9c1e Andy McCall 1 month ago ↶	596	}
	597	}
	598	let report = self.git_export_as(&mirror, branch)?;
	599	let spec = format!("refs/heads/{branch}:refs/heads/{branch}");
	600	git(&mirror, &["push", "-q", url, &spec])?;
	601	Ok(report)
	602	}
	603
79ab8528 Andy McCall 1 month ago ↶	604	fn read_oxy_marks(path: &Path) -> Result<BTreeMap<ObjectRef, u64>> {
	605	let mut out = BTreeMap::new();
	606	if let Ok(text) = fs::read_to_string(path) {
	607	for line in text.lines() {
	608	if let Some((h, m)) = line.split_once(' ') {
	609	if let (Some(h), Ok(m)) = (ObjectRef::from_hex(h), m.parse()) {
	610	out.insert(h, m);
	611	}
	612	}
	613	}
	614	}
	615	Ok(out)
	616	}
	617
	618	/// Export the current bookmark to the Git repository at `git_dir`,
	619	/// creating a bare one if nothing is there. Each state the bookmark
	620	/// passed through (per the operation log) becomes one commit whose tree
	621	/// is that state, so the Git branch is a linear history of full trees.
	622	pub fn git_export(&self, git_dir: &Path) -> Result<GitExportReport> {
30ea9c1e Andy McCall 1 month ago ↶	623	let bookmark = self.config.bookmark.clone();
	624	self.git_export_as(git_dir, &bookmark)
	625	}
	626
	627	/// `git_export` to the Git branch `branch`. A state that came from a
	628	/// Git commit (`oxy git fetch`) is not re-exported: later states chain
	629	/// onto that commit, so a push back to the Git remote fast-forwards.
	630	pub fn git_export_as(&self, git_dir: &Path, branch: &str) -> Result<GitExportReport> {
79ab8528 Andy McCall 1 month ago ↶	631	if !git_dir.join("HEAD").exists() && !git_dir.join(".git").exists() {
	632	fs::create_dir_all(git_dir)?;
	633	git(git_dir, &["init", "--bare", "-q"])?;
	634	}
	635	let marks_dir = self.git_marks_dir(git_dir)?;
	636	let git_marks = marks_dir.join("git-marks");
	637	let oxy_marks_path = marks_dir.join("oxy-marks");
	638	let mut marks = Self::read_oxy_marks(&oxy_marks_path)?;
30ea9c1e Andy McCall 1 month ago ↶	639	let git_states = Self::read_git_states(&marks_dir.join("git-states"))?;
79ab8528 Andy McCall 1 month ago ↶	640	let mut next_mark = marks.values().copied().max().unwrap_or(0) + 1;
	641
	642	let bookmark = self.config.bookmark.clone();
	643	let mut stream: Vec<u8> = Vec::new();
75167672 Andy McCall 1 month ago	644	let mut report = GitExportReport {
	645	branch: branch.to_string(),
	646	heads: self.bookmark_heads()?.len(),
	647	..Default::default()
	648	};
79ab8528 Andy McCall 1 month ago ↶	649
	650	// States along the bookmark, oldest first, skipping unchanged and pruned ones.
	651	let mut ops = self.op_log()?;
	652	ops.reverse();
	653	let mut prev: Option<(ObjectRef, Vec<ChangeHash>)> = None;
30ea9c1e Andy McCall 1 month ago ↶	654	// The fast-import parent of the next commit: `:mark` or a commit sha.
30ea9c1e Andy McCall 1 month ago ↶	655	let mut last_parent: Option<String> = None;
79ab8528 Andy McCall 1 month ago ↶	656	for (_, op) in &ops {
75167672 Andy McCall 1 month ago	657	let Some(state_ref) = op.view.bookmarks.get(&bookmark).copied() else {
	658	continue;
	659	};
79ab8528 Andy McCall 1 month ago ↶	660	if prev.as_ref().is_some_and(\|(p, _)\| *p == state_ref) {
	661	continue;
	662	}
75167672 Andy McCall 1 month ago	663	let Ok(state) = self.store.get_state(&state_ref) else {
	664	continue;
	665	};
79ab8528 Andy McCall 1 month ago ↶	666	if state.heads.is_empty() {
	667	prev = Some((state_ref, vec![]));
	668	continue;
	669	}
30ea9c1e Andy McCall 1 month ago ↶	670	if let Some(sha) = git_states.get(&state_ref) {
	671	report.skipped += 1;
	672	last_parent = Some(sha.clone());
	673	prev = Some((state_ref, state.heads.clone()));
	674	continue;
	675	}
79ab8528 Andy McCall 1 month ago ↶	676	if let Some(m) = marks.get(&state_ref) {
79ab8528 Andy McCall 1 month ago ↶	677	report.skipped += 1;
30ea9c1e Andy McCall 1 month ago ↶	678	last_parent = Some(format!(":{m}"));
79ab8528 Andy McCall 1 month ago ↶	679	prev = Some((state_ref, state.heads.clone()));
	680	continue;
	681	}
	682	let prev_heads = prev.as_ref().map(\|(_, h)\| h.clone()).unwrap_or_default();
75167672 Andy McCall 1 month ago	683	let added = self
	684	.negotiate(&prev_heads, &state.heads)
	685	.unwrap_or_default();
79ab8528 Andy McCall 1 month ago ↶	686	let loaded = self.load_state(&state.heads)?;
	687	let snapshot = loaded.materialize();
	688	let (author, secs, mut message) = match added.as_slice() {
	689	[h] => {
	690	let c = self.store.get_change(h)?;
	691	let mut m = c.message.clone();
	692	if !m.ends_with('\n') {
	693	m.push('\n');
	694	}
75167672 Andy McCall 1 month ago	695	m.push_str(&format!(
	696	"\nChange-Id: {}\nOxy-Hash: {}\n",
	697	c.change_id.to_hex(),
	698	h.to_hex()
	699	));
79ab8528 Andy McCall 1 month ago ↶	700	(self.git_author(&c), c.hlc.wall_ms / 1000, m)
79ab8528 Andy McCall 1 month ago ↶	701	}
75167672 Andy McCall 1 month ago	702	[] => (
	703	"oxy <oxy@localhost>".to_string(),
	704	op.hlc.wall_ms / 1000,
	705	format!("oxy: {}\n\n", op.command),
	706	),
79ab8528 Andy McCall 1 month ago ↶	707	many => {
	708	let mut m = format!("oxy: {} ({} changes)\n\n", op.command, many.len());
	709	for h in many {
	710	let c = self.store.get_change(h)?;
75167672 Andy McCall 1 month ago	711	m.push_str(&format!(
	712	"- {} {}\n",
	713	c.change_id.short(8),
	714	c.message.lines().next().unwrap_or("")
	715	));
79ab8528 Andy McCall 1 month ago ↶	716	}
75167672 Andy McCall 1 month ago	717	m.push_str(&format!(
	718	"\nOxy-Changes: {}\n",
	719	many.iter()
	720	.map(\|h\| h.to_hex())
	721	.collect::<Vec<_>>()
	722	.join(" ")
	723	));
79ab8528 Andy McCall 1 month ago ↶	724	("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, m)
	725	}
	726	};
	727	message.push_str(&format!("Oxy-State: {}\n", state_ref.to_hex()));
	728	if snapshot.is_conflicted() {
	729	message.push_str("Oxy-Conflicted: true\n");
	730	}
	731	let mark = next_mark;
	732	next_mark += 1;
30ea9c1e Andy McCall 1 month ago ↶	733	let parents: Vec<String> = last_parent.clone().into_iter().collect();
75167672 Andy McCall 1 month ago	734	self.write_commit(
	735	&mut stream,
	736	"refs/oxy/export",
	737	mark,
	738	&author,
	739	secs,
	740	&message,
	741	&parents,
	742	&snapshot,
	743	)?;
79ab8528 Andy McCall 1 month ago ↶	744	marks.insert(state_ref, mark);
30ea9c1e Andy McCall 1 month ago ↶	745	last_parent = Some(format!(":{mark}"));
79ab8528 Andy McCall 1 month ago ↶	746	prev = Some((state_ref, state.heads.clone()));
	747	report.exported += 1;
	748	}
30ea9c1e Andy McCall 1 month ago ↶	749	if let Some(p) = &last_parent {
30ea9c1e Andy McCall 1 month ago ↶	750	stream.extend_from_slice(format!("reset refs/heads/{branch}\nfrom {p}\n\n").as_bytes());
79ab8528 Andy McCall 1 month ago ↶	751	}
132b884b Andy McCall 1 month ago ↶	752	// Tags: a tag whose state was exported as a bookmark state points at
	753	// that commit; otherwise its state gets a commit of its own.
	754	for t in self.tags()? {
30ea9c1e Andy McCall 1 month ago ↶	755	let target = match (git_states.get(&t.state), marks.get(&t.state)) {
	756	(Some(sha), _) => sha.clone(),
	757	(None, Some(m)) => format!(":{m}"),
	758	(None, None) => {
132b884b Andy McCall 1 month ago ↶	759	let loaded = self.load_state(&t.heads)?;
	760	let snapshot = loaded.materialize();
	761	let mark = next_mark;
	762	next_mark += 1;
75167672 Andy McCall 1 month ago	763	let message = format!(
	764	"oxy: tag {}\n\n{}Oxy-State: {}\n",
	765	t.name,
	766	if t.message.is_empty() {
	767	String::new()
	768	} else {
	769	format!("{}\n\n", t.message)
	770	},
	771	t.state.to_hex()
	772	);
	773	self.write_commit(
	774	&mut stream,
	775	"refs/oxy/export",
	776	mark,
	777	"oxy <oxy@localhost>",
	778	Hlc::now().wall_ms / 1000,
	779	&message,
	780	&[],
	781	&snapshot,
	782	)?;
132b884b Andy McCall 1 month ago ↶	783	marks.insert(t.state, mark);
132b884b Andy McCall 1 month ago ↶	784	report.exported += 1;
30ea9c1e Andy McCall 1 month ago ↶	785	format!(":{mark}")
132b884b Andy McCall 1 month ago ↶	786	}
132b884b Andy McCall 1 month ago ↶	787	};
75167672 Andy McCall 1 month ago	788	stream.extend_from_slice(
	789	format!("reset refs/tags/{}\nfrom {target}\n\n", t.name).as_bytes(),
	790	);
132b884b Andy McCall 1 month ago ↶	791	}
132b884b Andy McCall 1 month ago ↶	792	stream.extend_from_slice(b"done\n");
79ab8528 Andy McCall 1 month ago ↶	793
	794	let mut cmd = Command::new("git");
	795	// `--force`: the scratch ref may jump between unrelated states.
75167672 Andy McCall 1 month ago	796	cmd.arg("-C")
	797	.arg(git_dir)
	798	.args(["fast-import", "--quiet", "--done", "--force"])
	799	.arg(format!("--export-marks={}", git_marks.display()));
79ab8528 Andy McCall 1 month ago ↶	800	if git_marks.exists() {
	801	cmd.arg(format!("--import-marks={}", git_marks.display()));
	802	}
75167672 Andy McCall 1 month ago	803	let mut child = cmd
	804	.stdin(Stdio::piped())
	805	.stdout(Stdio::null())
	806	.stderr(Stdio::piped())
	807	.spawn()
	808	.map_err(\|e\| RepoError::Msg(format!("running git fast-import: {e}")))?;
79ab8528 Andy McCall 1 month ago ↶	809	child.stdin.take().unwrap().write_all(&stream)?;
	810	let out = child.wait_with_output()?;
	811	if !out.status.success() {
75167672 Andy McCall 1 month ago	812	return fail(format!(
	813	"git fast-import failed: {}",
	814	String::from_utf8_lossy(&out.stderr).trim()
	815	));
79ab8528 Andy McCall 1 month ago ↶	816	}
75167672 Andy McCall 1 month ago	817	let text: String = marks
	818	.iter()
	819	.map(\|(h, m)\| format!("{} {m}\n", h.to_hex()))
	820	.collect();
79ab8528 Andy McCall 1 month ago ↶	821	write_atomic(&oxy_marks_path, text.as_bytes())?;
79ab8528 Andy McCall 1 month ago ↶	822	let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]);
4dee277c Andy McCall 1 month ago ↶	823	report.skipped_subrepos = self.git_export_skipped.replace(0);
79ab8528 Andy McCall 1 month ago ↶	824	Ok(report)
	825	}
	826
c3069537 Andy McCall 1 month ago ↶	827	/// Import branch `branch` of the Git repository at `git_dir` into this
	828	/// (empty) repository's bookmark (spec §23): one change per commit, its
	829	/// edits the difference between the union of its parents' states and its
	830	/// tree, so merge commits become resolutions. Submodules are skipped.
	831	pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result<GitImportReport> {
58dfe0e1 Andy McCall 1 month ago ↶	832	self.import_git_with(git_dir, branch, &mut \|_\| {})
	833	}
	834
	835	/// `import_git`, telling `progress` where it is every few hundred
	836	/// commits: the count so far, the merges, and the commit in hand, so a
	837	/// long import is never silent and a run that dies says where.
	838	pub fn import_git_with(
	839	&self,
	840	git_dir: &Path,
	841	branch: &str,
	842	progress: crate::store::Progress<'_>,
	843	) -> Result<GitImportReport> {
c3069537 Andy McCall 1 month ago ↶	844	if !self.bookmark_heads()?.is_empty() {
	845	return fail("git import needs an empty bookmark; import into a fresh repository");
	846	}
75167672 Andy McCall 1 month ago	847	self.import_git_commits(
	848	git_dir,
	849	branch,
	850	&format!("oxy git import {} {branch}", git_dir.display()),
58dfe0e1 Andy McCall 1 month ago ↶	851	progress,
75167672 Andy McCall 1 month ago	852	)
30ea9c1e Andy McCall 1 month ago ↶	853	}
	854
	855	/// Import the commits of `branch` that the bookmark does not have yet,
	856	/// then move the bookmark to the union of its heads and theirs.
75167672 Andy McCall 1 month ago	857	fn import_git_commits(
	858	&self,
	859	git_dir: &Path,
	860	branch: &str,
	861	command: &str,
58dfe0e1 Andy McCall 1 month ago ↶	862	progress: crate::store::Progress<'_>,
75167672 Andy McCall 1 month ago	863	) -> Result<GitImportReport> {
8f10a07a Andy McCall 1 month ago ↶	864	let batch = self.store.begin_batch()?;
58dfe0e1 Andy McCall 1 month ago ↶	865	let result = self.import_git_commits_batched(git_dir, branch, command, progress);
8f10a07a Andy McCall 1 month ago ↶	866	batch.finish()?;
	867	result
	868	}
	869
75167672 Andy McCall 1 month ago	870	fn import_git_commits_batched(
	871	&self,
	872	git_dir: &Path,
	873	branch: &str,
	874	command: &str,
58dfe0e1 Andy McCall 1 month ago ↶	875	progress: crate::store::Progress<'_>,
75167672 Andy McCall 1 month ago	876	) -> Result<GitImportReport> {
c3069537 Andy McCall 1 month ago ↶	877	let commits = list_commits(git_dir, branch)?;
	878	let mut cat = CatFile::start(git_dir)?;
	879	let mut report = GitImportReport::default();
30ea9c1e Andy McCall 1 month ago ↶	880	let mut known = self.git_known(git_dir)?;
30ea9c1e Andy McCall 1 month ago ↶	881	let old_heads = self.bookmark_heads()?;
dfb23a3a Andy McCall 1 month ago ↶	882	// The bookmark's heads as changes arrive: a change stops being a
	883	// head when another depends on it, so no full state of every
	884	// imported change is needed at the end.
	885	let mut heads_now: BTreeSet<ChangeHash> = old_heads.iter().copied().collect();
958fb27d Andy McCall 1 month ago ↶	886	// The state and Git tree of imported commits whose first-parent
	887	// children are still to come, so a child starts from its parent's
	888	// state instead of rebuilding it from history. The last child takes
dfb23a3a Andy McCall 1 month ago ↶	889	// it; earlier ones clone it. At most `max_kept` are kept (a later
958fb27d Andy McCall 1 month ago ↶	890	// child of an evicted one rebuilds).
dfb23a3a Andy McCall 1 month ago ↶	891	// OXY_IMPORT_KEEP overrides it, for measuring.
	892	let max_kept: usize = std::env::var("OXY_IMPORT_KEEP")
	893	.ok()
	894	.and_then(\|v\| v.parse().ok())
	895	.unwrap_or(8);
958fb27d Andy McCall 1 month ago ↶	896	let mut waiting: HashMap<&str, usize> = HashMap::new();
	897	for gc in &commits {
	898	if let Some(p1) = gc.parents.first() {
	899	*waiting.entry(p1.as_str()).or_default() += 1;
	900	}
	901	}
	902	type Kept = (State, BTreeMap<String, (String, String)>);
	903	let mut kept: HashMap<String, Kept> = HashMap::new();
2306a4d2 Andy McCall 1 month ago ↶	904	// Every commit to import that has a parent, diffed against its first.
	905	let pairs: Vec<(String, String)> = commits
	906	.iter()
	907	.filter(\|gc\| {
	908	!known.imported.contains_key(&gc.sha) && !known.exported.contains_key(&gc.sha)
	909	})
	910	.filter_map(\|gc\| Some((gc.sha.clone(), gc.parents.first()?.clone())))
	911	.collect();
	912	let mut diffs = DiffTree::start(git_dir, pairs)?;
	913	let mut rebuilt = 0usize;
958fb27d Andy McCall 1 month ago ↶	914	let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
958fb27d Andy McCall 1 month ago ↶	915	let mut last: Option<String> = None;
c3069537 Andy McCall 1 month ago ↶	916	let mut blob_cache: HashMap<String, Vec<u8>> = HashMap::new();
c3069537 Andy McCall 1 month ago ↶	917	let mut cache_bytes: usize = 0;
958fb27d Andy McCall 1 month ago ↶	918	// Time spent per phase over all commits (`OXY_TIMINGS=1`).
	919	let mut spent = [std::time::Duration::ZERO; 6];
	920	let mut clock = std::time::Instant::now();
fb5b6181 Andy McCall 1 month ago ↶	921	// One line per commit on stderr with OXY_IMPORT_TRACE=1: where its
	922	// base came from, what a merge added, the graphs its state holds,
	923	// the kept states and the process's resident memory.
	924	let trace = std::env::var_os("OXY_IMPORT_TRACE").is_some();
	925	let rss_mb = \|\| -> u64 {
	926	std::fs::read_to_string("/proc/self/statm")
	927	.ok()
	928	.and_then(\|s\| s.split_whitespace().nth(1)?.parse::<u64>().ok())
d2e3c39d Andy McCall 1 month ago ↶	929	.map(\|pages\| (pages * 4096) >> 20)
fb5b6181 Andy McCall 1 month ago ↶	930	.unwrap_or(0)
	931	};
	932	let mut done = 0usize;
d45dab04 Andy McCall 1 month ago ↶	933	// Atoms a kept state's graphs may hold before they are shed;
	934	// OXY_IMPORT_KEPT_ATOMS overrides it, for measuring.
	935	let kept_atoms: usize = std::env::var("OXY_IMPORT_KEPT_ATOMS")
	936	.ok()
	937	.and_then(\|v\| v.parse().ok())
	938	.unwrap_or(KEPT_ATOMS);
fd280228 Andy McCall 1 month ago ↶	939	// Graphs turned back into history (see `State::shed_graphs`).
fd280228 Andy McCall 1 month ago ↶	940	let mut shed = 0usize;
958fb27d Andy McCall 1 month ago ↶	941	let mut lap = \|i: usize, clock: &mut std::time::Instant\| {
	942	let now = std::time::Instant::now();
	943	spent[i] += now - *clock;
	944	*clock = now;
	945	};
c3069537 Andy McCall 1 month ago ↶	946
9935d490 Andy McCall 29 days ago ↶	947	// OXY_IMPORT_CHECKPOINT overrides it, for testing.
	948	let checkpoint_every: usize = std::env::var("OXY_IMPORT_CHECKPOINT")
	949	.ok()
	950	.and_then(\|v\| v.parse().ok())
	951	.filter(\|n\| *n > 0)
	952	.unwrap_or(CHECKPOINT_COMMITS);
810691ca Andy McCall 29 days ago ↶	953	// OXY_IMPORT_STOP_AFTER ends the import cleanly after that many
	954	// commits, its timings reported, for measuring at depth.
	955	let stop_after: Option<usize> = std::env::var("OXY_IMPORT_STOP_AFTER")
	956	.ok()
	957	.and_then(\|v\| v.parse().ok());
d256e826 Andy McCall 27 days ago ↶	958	// Merges built from the paths they can change (oxygen#80): on VS
	959	// Code at 139k commits the same 400 commits came out identical to
	960	// whole-tree merges, with merges 3.2x faster. OXY_IMPORT_WHOLE_MERGES=1
	961	// builds them from the whole tree, for comparing.
	962	let partial_merges = std::env::var_os("OXY_IMPORT_WHOLE_MERGES").is_none_or(\|v\| v == "0");
58dfe0e1 Andy McCall 1 month ago ↶	963	let total = commits.len();
d6e0d5b8 Andy McCall 25 days ago ↶	964	// The importing identity's own commits, claimed (`git.claim`,
	965	// oxygen#97): a commit whose Git author email is its email gets
	966	// its key as author; anyone else's keeps the zero key.
	967	let claim: Option<(KeyId, String)> = if self.config.git_claim {
	968	let Some(key) = self.config.author_key else {
	969	return fail("git.claim authors your own commits with your key, and there is none: run `oxy identity create` first, or `oxy config git.claim false`");
	970	};
	971	Some((key, self.identity(&key)?.email.to_lowercase()))
	972	} else {
	973	None
	974	};
58dfe0e1 Andy McCall 1 month ago ↶	975	for (index, gc) in commits.iter().enumerate() {
30ea9c1e Andy McCall 1 month ago ↶	976	if known.imported.contains_key(&gc.sha) \|\| known.exported.contains_key(&gc.sha) {
30ea9c1e Andy McCall 1 month ago ↶	977	continue;
810691ca Andy McCall 29 days ago ↶	978	}
	979	if stop_after.is_some_and(\|n\| done >= n) {
	980	break;
58dfe0e1 Andy McCall 1 month ago ↶	981	}
810691ca Andy McCall 29 days ago ↶	982	let commit_started = std::time::Instant::now();
58dfe0e1 Andy McCall 1 month ago ↶	983	if index % 500 == 0 {
	984	progress(&format!(
	985	"{index} of {total} commits, {} merges; at {} {}",
	986	report.merges,
	987	&gc.sha[..12.min(gc.sha.len())],
	988	gc.message
	989	.lines()
	990	.next()
	991	.unwrap_or("")
	992	.chars()
	993	.take(60)
	994	.collect::<String>()
	995	));
30ea9c1e Andy McCall 1 month ago ↶	996	}
fb5b6181 Andy McCall 1 month ago ↶	997	let mut base_how = "root";
958fb27d Andy McCall 1 month ago ↶	998	// Base: union of every parent's state, starting from the first
	999	// parent's kept state when there is one; its tree tells which
	1000	// paths this commit leaves alone.
	1001	let (mut base, prev_tree) = match gc.parents.first() {
	1002	Some(p1) => {
	1003	let left = waiting.get_mut(p1.as_str()).map(\|n\| {
	1004	*n -= 1;
	1005	*n
	1006	});
	1007	let found = if left == Some(0) {
	1008	kept_order.retain(\|k\| k != p1);
	1009	kept.remove(p1.as_str())
	1010	} else {
	1011	kept.get(p1.as_str()).cloned()
fb5b6181 Andy McCall 1 month ago ↶	1012	};
	1013	let how = match (&found, left) {
	1014	(Some(_), Some(0)) => "kept",
	1015	(Some(_), _) => "cloned",
	1016	(None, _) => "rebuilt",
958fb27d Andy McCall 1 month ago ↶	1017	};
fb5b6181 Andy McCall 1 month ago ↶	1018	if trace {
	1019	base_how = how;
	1020	}
958fb27d Andy McCall 1 month ago ↶	1021	match found {
	1022	Some((st, tree)) => (st, Some(tree)),
	1023	None => {
b493d993 Andy McCall 1 month ago ↶	1024	// History only: graphs are built for the files a
	1025	// commit changes, so memory follows what changes,
	1026	// not the whole tree.
	1027	let mut st = State::new_history_only();
958fb27d Andy McCall 1 month ago ↶	1028	self.add_git_ancestors(&mut st, git_dir, &[p1.as_str()], &known)?;
2306a4d2 Andy McCall 1 month ago ↶	1029	// The parent's tree, so this commit still
	1030	// renders only what it changes.
	1031	rebuilt += 1;
	1032	(st, Some(list_tree(git_dir, p1)?))
958fb27d Andy McCall 1 month ago ↶	1033	}
958fb27d Andy McCall 1 month ago ↶	1034	}
30ea9c1e Andy McCall 1 month ago ↶	1035	}
b493d993 Andy McCall 1 month ago ↶	1036	None => (State::new_history_only(), None),
c3069537 Andy McCall 1 month ago ↶	1037	};
958fb27d Andy McCall 1 month ago ↶	1038	// Nodes the other parents' changes touch: the first parent's
	1039	// tree cannot vouch for them.
	1040	let mut touched: HashSet<NodeId> = HashSet::new();
fb5b6181 Andy McCall 1 month ago ↶	1041	let mut merged = 0usize;
ce118287 Andy McCall 29 days ago ↶	1042	// Where the first parent has each node, read before the merged
	1043	// changes move anything: a merge's partial build names the paths
	1044	// they touch in the first parent's tree as well as in the merged
	1045	// state (oxygen#80).
	1046	let mut first_parent_view: Option<crate::tree::TreeView> = None;
c3069537 Andy McCall 1 month ago ↶	1047	if gc.parents.len() > 1 {
c3069537 Andy McCall 1 month ago ↶	1048	report.merges += 1;
579afd10 Andy McCall 27 days ago ↶	1049	if prev_tree.is_some() && partial_merges {
ce118287 Andy McCall 29 days ago ↶	1050	first_parent_view = Some(base.tree().view());
ce118287 Andy McCall 29 days ago ↶	1051	}
c3069537 Andy McCall 1 month ago ↶	1052	let mut args: Vec<String> = gc.parents[1..].to_vec();
c3069537 Andy McCall 1 month ago ↶	1053	args.push(format!("^{}", gc.parents[0]));
30ea9c1e Andy McCall 1 month ago ↶	1054	let refs: Vec<&str> = args.iter().map(String::as_str).collect();
fd280228 Andy McCall 1 month ago ↶	1055	// A merge of a long-lived branch brings thousands of
	1056	// changes: added to built graphs, which kept states share,
	1057	// they were copied and applied whole (vscode's import passed
	1058	// 14 GB on such merges). Shed the graphs first, so the
	1059	// changes go in as skeletons and only what this commit
	1060	// touches is built again.
	1061	if base.built_count() > 0 {
	1062	let mut count_args = vec!["rev-list", "--count"];
	1063	count_args.extend(refs.iter().copied());
	1064	let others: usize = git(git_dir, &count_args)?.trim().parse().unwrap_or(0);
33be2409 Andy McCall 1 month ago ↶	1065	let shed_at = std::env::var("OXY_MERGE_SHED")
	1066	.ok()
	1067	.and_then(\|v\| v.parse().ok())
	1068	.unwrap_or(MERGE_SHED_COMMITS);
	1069	if others > shed_at {
fd280228 Andy McCall 1 month ago ↶	1070	shed += base.shed_graphs(\|b\| self.store_blob(b))?;
	1071	}
	1072	}
fb5b6181 Andy McCall 1 month ago ↶	1073	let before = base.len();
958fb27d Andy McCall 1 month ago ↶	1074	self.add_git_ancestors_touching(&mut base, git_dir, &refs, &known, &mut touched)?;
fb5b6181 Andy McCall 1 month ago ↶	1075	merged = base.len() - before;
03571027 Andy McCall 1 month ago ↶	1076	}
958fb27d Andy McCall 1 month ago ↶	1077	lap(0, &mut clock);
2306a4d2 Andy McCall 1 month ago ↶	1078	// The commit's tree: its first parent's with the diff applied,
	1079	// or listed whole when that tree is not kept.
	1080	let diff = match gc.parents.first() {
	1081	Some(_) => Some(diffs.diff(&gc.sha)?),
	1082	None => None,
	1083	};
	1084	let (tree, changed) = match (prev_tree, diff) {
	1085	(Some(mut t), Some(diff)) => {
	1086	let mut changed: HashSet<String> = HashSet::new();
	1087	for (path, entry) in diff {
	1088	match entry {
	1089	Some(v) => {
	1090	t.insert(path.clone(), v);
	1091	}
	1092	None => {
	1093	t.remove(&path);
	1094	}
	1095	}
	1096	changed.insert(path);
	1097	}
	1098	(t, Some(changed))
	1099	}
	1100	_ => (list_tree(git_dir, &gc.sha)?, None),
	1101	};
958fb27d Andy McCall 1 month ago ↶	1102	lap(1, &mut clock);
03571027 Andy McCall 1 month ago ↶	1103	// Text files with the same blob and mode as in the first
	1104	// parent's tree equal the base's rendering: they are neither
	1105	// rendered nor read nor compared (spec §12.3, as for a clean
	1106	// working copy), so a commit costs what it changes.
	1107	let mut same: HashMap<&str, NodeId> = HashMap::new();
2306a4d2 Andy McCall 1 month ago ↶	1108	// Opaque files and symlinks likewise, when the base has one
	1109	// blob for them: they are neither read nor stored again.
	1110	let mut same_opaque: HashSet<&str> = HashSet::new();
	1111	// An ordinary commit in a tree without submodules is built from
	1112	// the paths it changes alone (oxygen#25): with the `.oxyattributes`
	1113	// files that decide how they are stored, and the directories its
	1114	// removals may leave empty. A merge, or a commit whose parent's
	1115	// tree is not at hand, is built from its whole tree.
ce118287 Andy McCall 29 days ago ↶	1116	// A merge can differ from its first parent only at the paths
	1117	// Git's diff names and where the merged changes touched: those
	1118	// paths in the first parent's tree and in the merged state. With
	1119	// them added it is built like an ordinary commit, instead of
	1120	// from its whole tree (18 s for any merge at 118k commits of
d256e826 Andy McCall 27 days ago ↶	1121	// VS Code, oxygen#80). OXY_IMPORT_WHOLE_MERGES=1 takes the
d256e826 Andy McCall 27 days ago ↶	1122	// whole-tree path.
ce118287 Andy McCall 29 days ago ↶	1123	let mut changed = changed;
	1124	// A tree conflict in the merged state (both sides placed a file
	1125	// at one path) is resolved by the whole-tree path.
	1126	let mut merge_how = if gc.parents.len() > 1 {
	1127	if first_parent_view.is_some() {
	1128	"partial"
	1129	} else {
	1130	"whole"
	1131	}
	1132	} else {
	1133	"-"
	1134	};
	1135	// Only a conflict among what the merged changes touched; one
	1136	// elsewhere stays as it was in the first parent either way.
	1137	if first_parent_view.is_some()
	1138	&& base.tree().view().conflicts.iter().any(\|c\| {
	1139	use crate::tree::TreeConflict::*;
	1140	match c {
	1141	Name { nodes, .. } => nodes.iter().any(\|n\| touched.contains(n)),
	1142	MultiPlacement { node, .. } \| Cycle { node } \| Mode { node, .. } => {
	1143	touched.contains(node)
	1144	}
	1145	Orphan { node, parent } => {
	1146	touched.contains(node) \|\| touched.contains(parent)
	1147	}
	1148	}
	1149	})
	1150	{
	1151	first_parent_view = None;
	1152	merge_how = "conflicted";
	1153	}
	1154	if let (Some(ch), Some(fp)) = (changed.as_mut(), &first_parent_view) {
	1155	let union = base.tree().view();
	1156	for v in [fp, &union] {
	1157	for (path, e) in v.entries.iter() {
	1158	if touched.contains(&e.node) {
	1159	ch.insert(path.clone());
	1160	}
	1161	}
	1162	}
	1163	}
2306a4d2 Andy McCall 1 month ago ↶	1164	let partial = changed.is_some()
ce118287 Andy McCall 29 days ago ↶	1165	&& (gc.parents.len() == 1 \|\| first_parent_view.is_some())
2306a4d2 Andy McCall 1 month ago ↶	1166	&& !tree.values().any(\|(mode, _)\| mode == "160000");
2306a4d2 Andy McCall 1 month ago ↶	1167	let mut rel: BTreeSet<&str> = BTreeSet::new();
ce118287 Andy McCall 29 days ago ↶	1168	// Text files whose graphs this commit builds, for the trace.
ce118287 Andy McCall 29 days ago ↶	1169	let need_n: usize;
2306a4d2 Andy McCall 1 month ago ↶	1170	let base_snapshot = if let (true, Some(changed)) = (partial, &changed) {
03571027 Andy McCall 1 month ago ↶	1171	let view = base.tree().view();
2306a4d2 Andy McCall 1 month ago ↶	1172	rel.extend(changed.iter().map(String::as_str));
	1173	rel.extend(
	1174	tree.keys()
	1175	.map(String::as_str)
	1176	.filter(\|p\| *p == ".oxyattributes" \|\| p.ends_with("/.oxyattributes")),
	1177	);
	1178	let mut dirs: BTreeSet<&str> = BTreeSet::new();
	1179	for p in changed.iter().filter(\|p\| !tree.contains_key(*p)) {
	1180	let mut end = p.len();
	1181	while let Some(i) = p[..end].rfind('/') {
	1182	if !dirs.insert(&p[..i]) {
	1183	break;
	1184	}
	1185	end = i;
03571027 Andy McCall 1 month ago ↶	1186	}
2306a4d2 Andy McCall 1 month ago ↶	1187	}
	1188	let need: Vec<NodeId> = rel
	1189	.iter()
	1190	.filter_map(\|p\| view.entries.get(*p))
	1191	.filter(\|e\| e.kind == NodeKind::File && !e.mode.opaque)
	1192	.map(\|e\| e.node)
	1193	.collect();
	1194	crate::timing::add("render: same and need", clock.elapsed());
ce118287 Andy McCall 29 days ago ↶	1195	need_n = need.len();
2306a4d2 Andy McCall 1 month ago ↶	1196	let t = std::time::Instant::now();
	1197	self.load_graphs_of(&mut base, &need)?;
	1198	crate::timing::add("render: load graphs", t.elapsed());
	1199	let t = std::time::Instant::now();
	1200	let snap = base.materialize_paths(view, rel.iter().chain(dirs.iter()).copied());
	1201	crate::timing::add("render: materialize", t.elapsed());
	1202	snap
	1203	} else {
	1204	if let Some(changed) = &changed {
	1205	let view = base.tree().view();
	1206	for path in tree.keys() {
	1207	// Submodule handling reads these two whatever changed.
	1208	if changed.contains(path) \|\| path == ".gitmodules" \|\| path == ".oxymodules"
958fb27d Andy McCall 1 month ago ↶	1209	{
2306a4d2 Andy McCall 1 month ago ↶	1210	continue;
	1211	}
	1212	if let Some(e) = view.entries.get(path) {
	1213	if touched.contains(&e.node) {
	1214	continue;
	1215	}
	1216	let opaque = e.kind == NodeKind::Symlink
	1217	\|\| (e.kind == NodeKind::File && e.mode.opaque);
	1218	if e.kind == NodeKind::File && !e.mode.opaque {
	1219	same.insert(path.as_str(), e.node);
	1220	} else if opaque && base.live_blobs(&e.node).len() == 1 {
	1221	same_opaque.insert(path.as_str());
	1222	}
03571027 Andy McCall 1 month ago ↶	1223	}
	1224	}
	1225	}
2306a4d2 Andy McCall 1 month ago ↶	1226	// Build the graphs of every text file the commit may change.
	1227	let skip: HashSet<NodeId> = same.values().copied().collect();
	1228	let need: Vec<NodeId> = base
	1229	.tree()
	1230	.view()
	1231	.entries
	1232	.values()
	1233	.filter(\|e\| {
	1234	e.kind == NodeKind::File && !e.mode.opaque && !skip.contains(&e.node)
	1235	})
	1236	.map(\|e\| e.node)
	1237	.collect();
	1238	crate::timing::add("render: same and need", clock.elapsed());
ce118287 Andy McCall 29 days ago ↶	1239	need_n = need.len();
2306a4d2 Andy McCall 1 month ago ↶	1240	let t = std::time::Instant::now();
c3d2ef61 Andy McCall 1 month ago ↶	1241	let snap = if need.len() > merge_batch() {
	1242	// A merge onto a base with no built graphs: thousands
	1243	// of files, whose graphs would not fit at once.
	1244	self.materialize_batched(&mut base, &need, &skip)?
	1245	} else {
ce118287 Andy McCall 29 days ago ↶	1246	let g = std::time::Instant::now();
c3d2ef61 Andy McCall 1 month ago ↶	1247	self.load_graphs_of(&mut base, &need)?;
ce118287 Andy McCall 29 days ago ↶	1248	crate::timing::add("render: whole, load graphs", g.elapsed());
	1249	let m = std::time::Instant::now();
	1250	let snap = base.materialize_except(&skip);
	1251	crate::timing::add("render: whole, materialize", m.elapsed());
	1252	snap
c3d2ef61 Andy McCall 1 month ago ↶	1253	};
c3d2ef61 Andy McCall 1 month ago ↶	1254	crate::timing::add("render: whole tree", t.elapsed());
2306a4d2 Andy McCall 1 month ago ↶	1255	snap
2306a4d2 Andy McCall 1 month ago ↶	1256	};
c3069537 Andy McCall 1 month ago ↶	1257
958fb27d Andy McCall 1 month ago ↶	1258	lap(2, &mut clock);
c3069537 Andy McCall 1 month ago ↶	1259	// The commit's tree as in-memory entries.
c3069537 Andy McCall 1 month ago ↶	1260	let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();
4dee277c Andy McCall 1 month ago ↶	1261	let mut gitlinks: Vec<(String, String)> = Vec::new();
2306a4d2 Andy McCall 1 month ago ↶	1262	let listed: Box<dyn Iterator<Item = (&String, &(String, String))>> = if partial {
	1263	Box::new(rel.iter().filter_map(\|p\| tree.get_key_value(*p)))
	1264	} else {
	1265	Box::new(tree.iter())
	1266	};
	1267	for (path, (mode, sha)) in listed {
03571027 Andy McCall 1 month ago ↶	1268	let (path, sha) = (path.clone(), sha.clone());
c3069537 Andy McCall 1 month ago ↶	1269	let (kind, exec) = match mode.as_str() {
	1270	"100644" \| "100664" => (NodeKind::File, false),
	1271	"100755" => (NodeKind::File, true),
	1272	"120000" => (NodeKind::Symlink, false),
	1273	"160000" => {
4dee277c Andy McCall 1 month ago ↶	1274	gitlinks.push((path, sha));
c3069537 Andy McCall 1 month ago ↶	1275	continue;
	1276	}
	1277	_ => continue,
	1278	};
2306a4d2 Andy McCall 1 month ago ↶	1279	if same.contains_key(path.as_str()) \|\| same_opaque.contains(path.as_str()) {
03571027 Andy McCall 1 month ago ↶	1280	let mut e = FsEntry::new(kind, exec, Vec::new());
	1281	e.unchanged = true;
	1282	entries.insert(path, e);
	1283	continue;
	1284	}
c3069537 Andy McCall 1 month ago ↶	1285	let content = match blob_cache.get(&sha) {
	1286	Some(c) => c.clone(),
	1287	None => {
	1288	let c = cat.read(&sha)?;
dfb23a3a Andy McCall 1 month ago ↶	1289	if cache_bytes + c.len() > 64 * 1024 * 1024 {
c3069537 Andy McCall 1 month ago ↶	1290	blob_cache.clear();
	1291	cache_bytes = 0;
	1292	}
	1293	cache_bytes += c.len();
	1294	blob_cache.insert(sha.clone(), c.clone());
	1295	c
	1296	}
	1297	};
b7f102fc Andy McCall 1 month ago ↶	1298	entries.insert(path, FsEntry::new(kind, exec, content));
c3069537 Andy McCall 1 month ago ↶	1299	}
c3069537 Andy McCall 1 month ago ↶	1300
4dee277c Andy McCall 1 month ago ↶	1301	// Submodules (spec §23): gitlinks become subrepo nodes with Git
	1302	// pins, and `.gitmodules` also yields `.oxymodules`.
	1303	if !gitlinks.is_empty() {
	1304	report.submodules += gitlinks.len();
75167672 Andy McCall 1 month ago	1305	let gm = entries
	1306	.get(".gitmodules")
	1307	.map(\|e\| {
	1308	crate::subrepo::Modules::from_gitmodules(&String::from_utf8_lossy(
	1309	&e.content,
	1310	))
	1311	})
	1312	.unwrap_or_default();
	1313	let mut modules = entries
	1314	.get(".oxymodules")
	1315	.map(\|e\| crate::subrepo::Modules::parse(&String::from_utf8_lossy(&e.content)))
	1316	.unwrap_or_default();
4dee277c Andy McCall 1 month ago ↶	1317	for (path, sha) in &gitlinks {
4dee277c Andy McCall 1 month ago ↶	1318	let m = gm.entries.get(path);
75167672 Andy McCall 1 month ago	1319	let branch = m
	1320	.and_then(\|m\| m.bookmark.clone())
	1321	.unwrap_or_else(\|\| "HEAD".into());
	1322	entries.insert(
	1323	path.clone(),
	1324	FsEntry::new(
	1325	NodeKind::Subrepo,
	1326	false,
	1327	crate::subrepo::Pin::Git {
	1328	sha: sha.clone(),
	1329	branch,
	1330	}
	1331	.encode(),
	1332	),
	1333	);
4dee277c Andy McCall 1 month ago ↶	1334	if let Some(m) = m {
	1335	modules.entries.insert(path.clone(), m.clone());
	1336	}
	1337	}
	1338	if !modules.entries.is_empty() {
75167672 Andy McCall 1 month ago	1339	entries.insert(
	1340	".oxymodules".into(),
	1341	FsEntry::new(NodeKind::File, false, modules.to_text().into_bytes()),
	1342	);
4dee277c Andy McCall 1 month ago ↶	1343	}
4dee277c Andy McCall 1 month ago ↶	1344	}
958fb27d Andy McCall 1 month ago ↶	1345	lap(3, &mut clock);
dfb23a3a Andy McCall 1 month ago ↶	1346	// New nodes' identifiers derive from the commit, so importing the
	1347	// same Git history twice gives the same changes (spec §23).
	1348	let seed = *blake3::hash(format!("oxy git import\0{}", gc.sha).as_bytes()).as_bytes();
2306a4d2 Andy McCall 1 month ago ↶	1349	let mut built = if partial {
	1350	// Whether the new tree has anything under `dir`.
	1351	let dir_needed = \|dir: &str\| {
	1352	let prefix = format!("{dir}/");
	1353	tree.range(prefix.clone()..)
	1354	.next()
	1355	.is_some_and(\|(k, _)\| k.starts_with(&prefix))
	1356	};
	1357	self.build_from_entries_seeded(
	1358	&base,
	1359	&base_snapshot,
	1360	&entries,
	1361	seed,
	1362	Some(&dir_needed),
	1363	)?
	1364	} else {
	1365	self.build_from_entries_seeded(&base, &base_snapshot, &entries, seed, None)?
	1366	};
958fb27d Andy McCall 1 month ago ↶	1367	lap(4, &mut clock);
62794ecc Andy McCall 1 month ago ↶	1368	// A file or submodule with several live blobs (branches that
	1369	// replaced it independently) is resolved to the commit's blob:
	1370	// Git's merge is the authority here, unlike a working copy,
	1371	// where a candidate on disk is not yet a decision (spec §23).
	1372	for (path, f) in &base_snapshot.files {
	1373	let (FileContent::Opaque { blobs }, Some(e)) = (&f.content, entries.get(path))
	1374	else {
	1375	continue;
	1376	};
	1377	if blobs.len() < 2 \|\| f.kind != e.kind {
	1378	continue;
	1379	}
	1380	let chosen = self.store_blob(&e.content)?;
	1381	for b in blobs {
	1382	built.edits.push(Edit::Replace {
	1383	node: f.node,
	1384	old: Some(*b),
	1385	new: Some(chosen),
	1386	});
	1387	}
	1388	built.resolves = true;
	1389	built
	1390	.contributors
	1391	.extend(base.live_replaces(&f.node).into_iter().map(\|(c, _)\| c));
	1392	}
e83354cd Andy McCall 1 month ago ↶	1393	// It shares the base's tree view, which applying the change
	1394	// then updates in place rather than copying.
	1395	drop(base_snapshot);
	1396	drop(entries);
c3069537 Andy McCall 1 month ago ↶	1397	let mut deps: BTreeSet<ChangeHash> = built.contributors.iter().copied().collect();
	1398	let mut message = gc.message.clone();
	1399	if !message.ends_with('\n') {
	1400	message.push('\n');
	1401	}
75167672 Andy McCall 1 month ago	1402	message.push_str(&format!(
	1403	"\nGit-Commit: {}\nGit-Author: {}\n",
	1404	gc.sha, gc.author
	1405	));
c3069537 Andy McCall 1 month ago ↶	1406	let mut change = Change {
6f8aeb99 Andy McCall 1 month ago ↶	1407	version: crate::objects::CHANGE_VERSION,
c3069537 Andy McCall 1 month ago ↶	1408	change_id: change_id_for(&gc.sha, &gc.message),
c3069537 Andy McCall 1 month ago ↶	1409	deps: vec![],
d6e0d5b8 Andy McCall 25 days ago ↶	1410	author: claim
	1411	.as_ref()
	1412	.filter(\|(_, email)\| git_email(&gc.author).is_some_and(\|e\| e == *email))
	1413	.map_or(ObjectRef::ZERO, \|(key, _)\| *key),
75167672 Andy McCall 1 month ago	1414	hlc: Hlc {
	1415	wall_ms: gc.author_secs * 1000,
	1416	logical: 0,
	1417	},
c3069537 Andy McCall 1 month ago ↶	1418	message,
	1419	tree_ops: built.tree_ops,
	1420	edits: built.edits,
75167672 Andy McCall 1 month ago	1421	kind: if built.resolves {
	1422	ChangeKind::Resolution
	1423	} else {
	1424	ChangeKind::Normal
	1425	},
c3069537 Andy McCall 1 month ago ↶	1426	};
2306a4d2 Andy McCall 1 month ago ↶	1427	let t = std::time::Instant::now();
73d592e7 Andy McCall 1 month ago ↶	1428	// A deletion depends on whoever inserted the live atoms after
	1429	// it, which only the file's graph knows: build the few the
	1430	// batched rendering left deferred.
	1431	let need: Vec<NodeId> = change
	1432	.edits
	1433	.iter()
	1434	.filter(\|e\| matches!(e, Edit::Delete { .. }))
	1435	.map(\|e\| e.node())
	1436	.filter(\|n\| !base.is_built(n))
	1437	.collect();
	1438	if !need.is_empty() {
	1439	self.load_graphs_of(&mut base, &need)?;
	1440	}
c3069537 Andy McCall 1 month ago ↶	1441	deps.extend(referenced_changes(&change, &base));
2306a4d2 Andy McCall 1 month ago ↶	1442	crate::timing::add("store: referenced", t.elapsed());
c3069537 Andy McCall 1 month ago ↶	1443	change.deps = deps.into_iter().collect();
	1444	change.normalize();
	1445	let hash = self.store.put_change(&change)?;
dfb23a3a Andy McCall 1 month ago ↶	1446	for d in &change.deps {
	1447	heads_now.remove(d);
	1448	}
	1449	heads_now.insert(hash);
ab4d4e31 Andy McCall 1 month ago ↶	1450	// Built graphs take its edits whole; files not built keep only
ab4d4e31 Andy McCall 1 month ago ↶	1451	// where its text is, read back from the store.
2306a4d2 Andy McCall 1 month ago ↶	1452	let t = std::time::Instant::now();
ab4d4e31 Andy McCall 1 month ago ↶	1453	let (skeleton, texts) = self.store.get_change_lazy(&hash)?;
ab4d4e31 Andy McCall 1 month ago ↶	1454	base.add_known_mixed(hash, change, skeleton, texts);
2306a4d2 Andy McCall 1 month ago ↶	1455	crate::timing::add("store: add to state", t.elapsed());
30ea9c1e Andy McCall 1 month ago ↶	1456	known.imported.insert(gc.sha.clone(), hash);
958fb27d Andy McCall 1 month ago ↶	1457	if waiting.get(gc.sha.as_str()).is_some_and(\|n\| *n > 0) {
fd280228 Andy McCall 1 month ago ↶	1458	// A kept state holds its graphs until its children come:
fd280228 Andy McCall 1 month ago ↶	1459	// past a few thousand, they go back into its history.
d45dab04 Andy McCall 1 month ago ↶	1460	// Past a few thousand graphs, or past a budget of atoms: a
	1461	// long-lived file's graph keeps every line it ever had, so a
	1462	// handful of them can hold gigabytes (#25).
	1463	if base.built_count() > KEPT_GRAPHS \|\| base.built_atoms() > kept_atoms {
fd280228 Andy McCall 1 month ago ↶	1464	shed += base.shed_graphs(\|b\| self.store_blob(b))?;
fd280228 Andy McCall 1 month ago ↶	1465	}
958fb27d Andy McCall 1 month ago ↶	1466	kept.insert(gc.sha.clone(), (base, tree));
958fb27d Andy McCall 1 month ago ↶	1467	kept_order.push_back(gc.sha.clone());
dfb23a3a Andy McCall 1 month ago ↶	1468	if kept_order.len() > max_kept {
958fb27d Andy McCall 1 month ago ↶	1469	if let Some(old) = kept_order.pop_front() {
	1470	kept.remove(&old);
	1471	}
	1472	}
	1473	}
	1474	last = Some(gc.sha.clone());
c3069537 Andy McCall 1 month ago ↶	1475	report.commits += 1;
958fb27d Andy McCall 1 month ago ↶	1476	lap(5, &mut clock);
fb5b6181 Andy McCall 1 month ago ↶	1477	done += 1;
9935d490 Andy McCall 29 days ago ↶	1478	if done.is_multiple_of(checkpoint_every) {
	1479	self.checkpoint_import(&heads_now, done)?;
	1480	}
d45dab04 Andy McCall 1 month ago ↶	1481	// A merge renders thousands of files on many threads and frees
	1482	// it all; the allocator keeps those pages unless asked to give
	1483	// them back, and the next merge piles on top (#25).
	1484	let before_trim = if trace { rss_mb() } else { 0 };
	1485	if gc.parents.len() > 1 {
	1486	crate::alloc_stats::release_freed_memory();
	1487	}
fb5b6181 Andy McCall 1 month ago ↶	1488	if trace {
	1489	let built = kept
	1490	.get(gc.sha.as_str())
	1491	.map(\|(st, _)\| st.built_count())
d45dab04 Andy McCall 1 month ago ↶	1492	.unwrap_or(0);
	1493	let (edits, most) = kept
	1494	.get(gc.sha.as_str())
	1495	.map(\|(st, _)\| st.deferred_edit_count())
	1496	.unwrap_or((0, 0));
	1497	let atoms = kept
	1498	.get(gc.sha.as_str())
	1499	.map(\|(st, _)\| st.built_atoms())
fb5b6181 Andy McCall 1 month ago ↶	1500	.unwrap_or(0);
fb5b6181 Andy McCall 1 month ago ↶	1501	eprintln!(
ce118287 Andy McCall 29 days ago ↶	1502	"trace {done} {} parents={} base={base_how} merged={merged} built={built} atoms={atoms} kept={} shed={shed} rss_mb={} before_trim_mb={before_trim} deferred_edits={edits} most={most} need={need_n} merge={merge_how} ms={}",
fb5b6181 Andy McCall 1 month ago ↶	1503	&gc.sha[..12.min(gc.sha.len())],
	1504	gc.parents.len(),
	1505	kept.len(),
810691ca Andy McCall 29 days ago ↶	1506	rss_mb(),
810691ca Andy McCall 29 days ago ↶	1507	commit_started.elapsed().as_millis()
fb5b6181 Andy McCall 1 month ago ↶	1508	);
fb5b6181 Andy McCall 1 month ago ↶	1509	}
958fb27d Andy McCall 1 month ago ↶	1510	}
	1511	for (name, d) in [
	1512	"git import: bases",
2306a4d2 Andy McCall 1 month ago ↶	1513	"git import: tree",
958fb27d Andy McCall 1 month ago ↶	1514	"git import: render base",
	1515	"git import: read blobs",
	1516	"git import: build change",
	1517	"git import: store change",
	1518	]
	1519	.iter()
	1520	.zip(spent)
	1521	{
	1522	crate::timing::record(name, d);
2306a4d2 Andy McCall 1 month ago ↶	1523	}
	1524	crate::timing::report_totals();
	1525	if crate::timing::enabled() {
fd280228 Andy McCall 1 month ago ↶	1526	eprintln!(
	1527	"timing git import: {rebuilt} bases rebuilt from history, {shed} graphs shed"
	1528	);
c3069537 Andy McCall 1 month ago ↶	1529	}
c3069537 Andy McCall 1 month ago ↶	1530
30ea9c1e Andy McCall 1 month ago ↶	1531	// The bookmark: heads of the old state plus every new change.
	1532	report.heads = old_heads.len();
	1533	if last.is_none() {
	1534	return Ok(report);
	1535	}
dfb23a3a Andy McCall 1 month ago ↶	1536	let heads: Vec<ChangeHash> = heads_now.into_iter().collect();
c3069537 Andy McCall 1 month ago ↶	1537	report.heads = heads.len();
	1538	let (op_ref, op) = self.head_op()?;
	1539	let mut view = op.view.clone();
30ea9c1e Andy McCall 1 month ago ↶	1540	let old_state_ref = view.bookmarks.get(&self.config.bookmark).copied();
30ea9c1e Andy McCall 1 month ago ↶	1541	let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?;
75167672 Andy McCall 1 month ago	1542	view.bookmarks
75167672 Andy McCall 1 month ago	1543	.insert(self.config.bookmark.clone(), state_ref);
30ea9c1e Andy McCall 1 month ago ↶	1544	self.record_op(&[op_ref], command, view)?;
	1545	// The new state equals the tip's tree when the old one was itself a
	1546	// Git commit's tree (fetched or exported), or there was none.
75167672 Andy McCall 1 month ago	1547	let old_known = old_heads.is_empty()
75167672 Andy McCall 1 month ago	1548	\|\| old_state_ref.is_some_and(\|s\| known.exported.values().any(\|e\| *e == s));
30ea9c1e Andy McCall 1 month ago ↶	1549	if let (Some(tip), true) = (commits.last(), old_known) {
	1550	self.record_git_state(git_dir, &state_ref, &tip.sha)?;
	1551	}
	1552	// The working copy follows when it was clean on the old heads.
	1553	if old_heads.is_empty() {
	1554	self.checkout(&heads, true, &format!("{command} (checkout)"))?;
	1555	} else {
	1556	let snap = self.snapshot(command)?;
	1557	let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
	1558	let mut base = snap.base.maximal(&snap.wc.deps);
	1559	base.sort();
	1560	let mut sorted_old = old_heads.clone();
	1561	sorted_old.sort();
9935d490 Andy McCall 29 days ago ↶	1562	// An empty base too: an import that stopped after a checkpoint
	1563	// never checked anything out, and resuming it should.
	1564	if clean && (base == sorted_old \|\| base.is_empty()) {
30ea9c1e Andy McCall 1 month ago ↶	1565	self.checkout(&heads, false, &format!("{command} (checkout)"))?;
	1566	}
	1567	}
c3069537 Andy McCall 1 month ago ↶	1568	Ok(report)
	1569	}
	1570
30ea9c1e Andy McCall 1 month ago ↶	1571	/// Add to `st` the changes of every Git commit `rev-list args` lists:
	1572	/// an imported commit's change, or the whole state an exported commit
	1573	/// was made from (which may hold changes Git never saw).
75167672 Andy McCall 1 month ago	1574	fn add_git_ancestors(
958fb27d Andy McCall 1 month ago ↶	1575	&self,
	1576	st: &mut State,
	1577	git_dir: &Path,
	1578	args: &[&str],
	1579	known: &GitKnown,
	1580	) -> Result<()> {
	1581	self.add_git_ancestors_touching(st, git_dir, args, known, &mut HashSet::new())
	1582	}
	1583
	1584	/// `add_git_ancestors`, also collecting in `touched` every node the
	1585	/// changes it adds edit, place or remove.
	1586	fn add_git_ancestors_touching(
75167672 Andy McCall 1 month ago	1587	&self,
	1588	st: &mut State,
	1589	git_dir: &Path,
	1590	args: &[&str],
	1591	known: &GitKnown,
958fb27d Andy McCall 1 month ago ↶	1592	touched: &mut HashSet<NodeId>,
75167672 Andy McCall 1 month ago	1593	) -> Result<()> {
30ea9c1e Andy McCall 1 month ago ↶	1594	let mut full = vec!["rev-list"];
	1595	full.extend(args);
	1596	let out = git(git_dir, &full)?;
958fb27d Andy McCall 1 month ago ↶	1597	let mut add = \|st: &mut State, h: &ChangeHash\| -> Result<()> {
958fb27d Andy McCall 1 month ago ↶	1598	if st.contains(h) {
e83354cd Andy McCall 1 month ago ↶	1599	return Ok(());
	1600	}
	1601	if !st.is_history_only() {
	1602	let c = self.store.get_change(h)?;
	1603	touched.extend(c.edits.iter().map(\|e\| e.node()));
	1604	touched.extend(c.tree_ops.iter().map(\|o\| o.node()));
	1605	st.add_unchecked(c);
958fb27d Andy McCall 1 month ago ↶	1606	return Ok(());
e83354cd Andy McCall 1 month ago ↶	1607	}
	1608	// History only: the text of an edit waits unread unless its
	1609	// file's graph is built.
	1610	let (skeleton, texts) = self.store.get_change_lazy(h)?;
	1611	touched.extend(skeleton.edits.iter().map(\|e\| e.node()));
	1612	touched.extend(skeleton.tree_ops.iter().map(\|o\| o.node()));
	1613	let built = skeleton
	1614	.edits
	1615	.iter()
	1616	.any(\|e\| !matches!(e, Edit::Replace { .. }) && st.is_built(&e.node()));
	1617	if built {
	1618	let full = self.store.get_change(h)?;
	1619	st.add_known_mixed(*h, full, skeleton, texts);
	1620	} else {
	1621	st.add_known(*h, skeleton, texts);
958fb27d Andy McCall 1 month ago ↶	1622	}
	1623	Ok(())
	1624	};
c3069537 Andy McCall 1 month ago ↶	1625	for sha in out.lines() {
30ea9c1e Andy McCall 1 month ago ↶	1626	if let Some(h) = known.imported.get(sha) {
958fb27d Andy McCall 1 month ago ↶	1627	add(st, h)?;
30ea9c1e Andy McCall 1 month ago ↶	1628	} else if let Some(state_ref) = known.exported.get(sha) {
	1629	let heads = self.store.get_state(state_ref)?.heads;
	1630	for h in self.negotiate(&[], &heads)? {
958fb27d Andy McCall 1 month ago ↶	1631	add(st, &h)?;
30ea9c1e Andy McCall 1 month ago ↶	1632	}
	1633	}
	1634	}
9935d490 Andy McCall 29 days ago ↶	1635	Ok(())
	1636	}
	1637
54e2e630 Andy McCall 29 days ago ↶	1638	/// Recover an import that stopped before it moved the bookmark: its
	1639	/// unfinished pack indexed (`Store::adopt_unfinished_packs`), then every
	1640	/// change in the store that carries a `Git-Commit:` trailer, its heads
	1641	/// (those no other such change depends on) made the bookmark's heads,
	1642	/// so `oxy git fetch` carries on from there. Reads every object once.
	1643	/// Returns how many imported changes it found and the heads.
	1644	pub fn recover_git_import(&self, progress: &dyn Fn(&str)) -> Result<(usize, usize)> {
	1645	// A stopped import's objects are in a pack it never indexed.
	1646	// OXY_RECOVER_MIN_AGE (seconds) overrides the five minutes it must
	1647	// have been left alone, for testing.
	1648	let min_age = std::env::var("OXY_RECOVER_MIN_AGE")
	1649	.ok()
	1650	.and_then(\|v\| v.parse().ok())
	1651	.unwrap_or(300);
	1652	self.store
	1653	.adopt_unfinished_packs(std::time::Duration::from_secs(min_age), progress)?;
	1654	let all = self
	1655	.store
	1656	.list()
	1657	.map_err(\|e\| RepoError::Msg(format!("listing the store: {e}")))?;
	1658	let total = all.len();
	1659	let mut found: HashMap<ChangeHash, Vec<ChangeHash>> = HashMap::new();
	1660	for (i, r) in all.iter().enumerate() {
	1661	if i.is_multiple_of(100_000) {
	1662	progress(&format!(
	1663	"{i} of {total} objects read, {} imported changes",
	1664	found.len()
	1665	));
	1666	}
	1667	let Ok(meta) = self.store.get_change_meta(r) else {
	1668	continue;
	1669	};
	1670	if meta.message.lines().any(\|l\| l.starts_with("Git-Commit: ")) {
	1671	found.insert(*r, meta.deps.clone());
	1672	}
	1673	}
	1674	if found.is_empty() {
	1675	return fail("no imported Git commit's change in this store");
	1676	}
	1677	let depended: HashSet<ChangeHash> = found.values().flatten().copied().collect();
	1678	let mut heads: Vec<ChangeHash> = found
	1679	.keys()
	1680	.filter(\|h\| !depended.contains(h))
	1681	.copied()
	1682	.collect();
	1683	// The bookmark's present heads stay: a recovery only adds.
	1684	heads.extend(self.bookmark_heads().unwrap_or_default());
	1685	let st = self.load_state_history(&heads)?;
	1686	let mut heads = st.maximal(&heads);
	1687	heads.sort();
	1688	heads.dedup();
	1689	let (op_ref, op) = self.head_op()?;
	1690	let mut view = op.view.clone();
	1691	let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?;
	1692	view.bookmarks
	1693	.insert(self.config.bookmark.clone(), state_ref);
	1694	self.record_op(&[op_ref], "oxy git recover", view)?;
	1695	Ok((found.len(), heads.len()))
	1696	}
	1697
9935d490 Andy McCall 29 days ago ↶	1698	/// Point the bookmark at an import's heads so far (see
	1699	/// `CHECKPOINT_COMMITS`); the working copy is left alone until the end.
	1700	fn checkpoint_import(&self, heads: &BTreeSet<ChangeHash>, done: usize) -> Result<()> {
	1701	let heads: Vec<ChangeHash> = heads.iter().copied().collect();
	1702	let (op_ref, op) = self.head_op()?;
	1703	let mut view = op.view.clone();
	1704	let state_ref = self.store.put_state(&StateObject::new(heads))?;
	1705	view.bookmarks
	1706	.insert(self.config.bookmark.clone(), state_ref);
	1707	self.record_op(
	1708	&[op_ref],
	1709	&format!("oxy git import (checkpoint after {done} commits)"),
	1710	view,
	1711	)?;
30ea9c1e Andy McCall 1 month ago ↶	1712	Ok(())
	1713	}
	1714
	1715	/// Commits this repository can relate to Oxygen history: imported
	1716	/// ones by change, exported ones (through `git_dir`'s marks) by state.
	1717	fn git_known(&self, git_dir: &Path) -> Result<GitKnown> {
	1718	let imported = self.imported_commits()?;
	1719	let marks_dir = self.git_marks_dir(git_dir)?;
	1720	let oxy_marks = Self::read_oxy_marks(&marks_dir.join("oxy-marks"))?;
	1721	let mut sha_of_mark: HashMap<u64, String> = HashMap::new();
	1722	if let Ok(text) = fs::read_to_string(marks_dir.join("git-marks")) {
	1723	for line in text.lines() {
	1724	if let Some((m, sha)) = line.split_once(' ') {
	1725	if let Ok(m) = m.trim_start_matches(':').parse::<u64>() {
	1726	sha_of_mark.insert(m, sha.trim().to_string());
	1727	}
	1728	}
	1729	}
	1730	}
	1731	let mut exported = HashMap::new();
	1732	for (state_ref, m) in oxy_marks {
	1733	if let Some(sha) = sha_of_mark.get(&m) {
	1734	exported.insert(sha.clone(), state_ref);
c3069537 Andy McCall 1 month ago ↶	1735	}
c3069537 Andy McCall 1 month ago ↶	1736	}
30ea9c1e Andy McCall 1 month ago ↶	1737	for (state_ref, sha) in Self::read_git_states(&marks_dir.join("git-states"))? {
	1738	exported.entry(sha).or_insert(state_ref);
	1739	}
	1740	Ok(GitKnown { imported, exported })
c3069537 Andy McCall 1 month ago ↶	1741	}
c3069537 Andy McCall 1 month ago ↶	1742
79ab8528 Andy McCall 1 month ago ↶	1743	fn git_author(&self, c: &Change) -> String {
6f4226bf Andy McCall 4 days ago ↶	1744	self.git_author_key(&c.author)
79ab8528 Andy McCall 1 month ago ↶	1745	}
	1746
	1747	#[allow(clippy::too_many_arguments)]
75167672 Andy McCall 1 month ago	1748	fn write_commit(
	1749	&self,
	1750	out: &mut Vec<u8>,
	1751	reference: &str,
	1752	mark: u64,
	1753	author: &str,
	1754	secs: u64,
	1755	message: &str,
	1756	parents: &[String],
	1757	snapshot: &Snapshot,
	1758	) -> Result<()> {
79ab8528 Andy McCall 1 month ago ↶	1759	out.extend_from_slice(format!("commit {reference}\nmark :{mark}\nauthor {author} {secs} +0000\ncommitter {author} {secs} +0000\ndata {}\n{message}\n", message.len()).as_bytes());
79ab8528 Andy McCall 1 month ago ↶	1760	for (i, p) in parents.iter().enumerate() {
75167672 Andy McCall 1 month ago	1761	out.extend_from_slice(
	1762	format!("{} {p}\n", if i == 0 { "from" } else { "merge" }).as_bytes(),
	1763	);
79ab8528 Andy McCall 1 month ago ↶	1764	}
79ab8528 Andy McCall 1 month ago ↶	1765	out.extend_from_slice(b"deleteall\n");
4dee277c Andy McCall 1 month ago ↶	1766	let has_gitmodules = snapshot.files.contains_key(".gitmodules");
79ab8528 Andy McCall 1 month ago ↶	1767	for (path, f) in &snapshot.files {
4dee277c Andy McCall 1 month ago ↶	1768	if path == ".oxymodules" {
	1769	// Git reads `.gitmodules`; derive it unless the tree has one.
	1770	if !has_gitmodules {
	1771	if let FileContent::Text { bytes, .. } = &f.content {
75167672 Andy McCall 1 month ago	1772	let gm = crate::subrepo::Modules::parse(&String::from_utf8_lossy(bytes))
75167672 Andy McCall 1 month ago	1773	.to_gitmodules();
4dee277c Andy McCall 1 month ago ↶	1774	if !gm.is_empty() {
75167672 Andy McCall 1 month ago	1775	out.extend_from_slice(
	1776	format!("M 100644 inline .gitmodules\ndata {}\n", gm.len())
	1777	.as_bytes(),
	1778	);
4dee277c Andy McCall 1 month ago ↶	1779	out.extend_from_slice(gm.as_bytes());
	1780	out.push(b'\n');
	1781	}
	1782	}
	1783	}
	1784	continue;
	1785	}
	1786	if f.kind == NodeKind::Subrepo {
	1787	if let FileContent::Opaque { blobs } = &f.content {
	1788	if let Some(b) = blobs.first() {
	1789	match crate::subrepo::Pin::decode(&self.read_blob(b)?)? {
75167672 Andy McCall 1 month ago	1790	crate::subrepo::Pin::Git { sha, .. } => {
	1791	out.extend_from_slice(format!("M 160000 {sha} {path}\n").as_bytes())
	1792	}
	1793	crate::subrepo::Pin::Oxy { .. } => self
	1794	.git_export_skipped
	1795	.set(self.git_export_skipped.get() + 1),
4dee277c Andy McCall 1 month ago ↶	1796	}
	1797	}
	1798	}
	1799	continue;
	1800	}
79ab8528 Andy McCall 1 month ago ↶	1801	let (mode, bytes): (&str, Vec<u8>) = match (&f.kind, &f.content) {
79ab8528 Andy McCall 1 month ago ↶	1802	(NodeKind::Dir, _) => continue,
75167672 Andy McCall 1 month ago	1803	(NodeKind::Symlink, FileContent::Opaque { blobs }) => (
	1804	"120000",
	1805	blobs
	1806	.first()
	1807	.map(\|b\| self.read_blob(b))
	1808	.transpose()?
	1809	.unwrap_or_default(),
	1810	),
	1811	(_, FileContent::Text { bytes, .. }) => {
	1812	(if f.mode.exec { "100755" } else { "100644" }, bytes.clone())
	1813	}
	1814	(_, FileContent::Opaque { blobs }) => (
	1815	if f.mode.exec { "100755" } else { "100644" },
	1816	blobs
	1817	.first()
	1818	.map(\|b\| self.read_blob(b))
	1819	.transpose()?
	1820	.unwrap_or_default(),
	1821	),
79ab8528 Andy McCall 1 month ago ↶	1822	(_, FileContent::Dir) => continue,
79ab8528 Andy McCall 1 month ago ↶	1823	};
75167672 Andy McCall 1 month ago	1824	out.extend_from_slice(
	1825	format!("M {mode} inline {path}\ndata {}\n", bytes.len()).as_bytes(),
	1826	);
79ab8528 Andy McCall 1 month ago ↶	1827	out.extend_from_slice(&bytes);
	1828	out.push(b'\n');
	1829	}
	1830	out.push(b'\n');
	1831	Ok(())
	1832	}
	1833	}
62794ecc Andy McCall 1 month ago ↶	1834
	1835	/// What `verify_git_import` found (spec §23: import must reproduce every tree).
	1836	#[derive(Debug, Clone, Default, PartialEq, Eq)]
	1837	pub struct GitVerifyReport {
	1838	pub commits: usize,
7eac74ab Andy McCall 1 month ago ↶	1839	/// Commits followed but not rebuilt (all but the last few, when asked).
7eac74ab Andy McCall 1 month ago ↶	1840	pub skipped: usize,
62794ecc Andy McCall 1 month ago ↶	1841	pub files_compared: usize,
	1842	/// `(commit, what differs)`; empty when every commit's tree is reproduced.
	1843	pub mismatches: Vec<(String, String)>,
f0406c3b Andy McCall 27 days ago ↶	1844	/// Commits of the branch after the last imported one: an import not
	1845	/// finished yet. Not checked.
	1846	pub not_yet: usize,
62794ecc Andy McCall 1 month ago ↶	1847	}
	1848
	1849	impl Repo {
	1850	/// Check an import against its Git source: for every commit of
	1851	/// `branch`, the state formed by the changes imported for it and all
	1852	/// its ancestors must materialize to exactly that commit's tree, byte
	1853	/// for byte, with the same executable bits, symlink targets and
	1854	/// submodule pins. Reads only; stops listing after 50 differences.
	1855	pub fn verify_git_import(&self, git_dir: &Path, branch: &str) -> Result<GitVerifyReport> {
8b7986dd Andy McCall 27 days ago ↶	1856	self.verify_git_import_last(git_dir, branch, None, 50)
7eac74ab Andy McCall 1 month ago ↶	1857	}
	1858
	1859	/// [`Repo::verify_git_import`] of the last `last` commits only (all
	1860	/// when `None`): earlier ones are only followed, not rebuilt, for
	1861	/// looking again at where an import went wrong.
8b7986dd Andy McCall 27 days ago ↶	1862	/// It stops after `max` differences (0: never), so one that repeats
8b7986dd Andy McCall 27 days ago ↶	1863	/// through history does not hide the commits after it.
7eac74ab Andy McCall 1 month ago ↶	1864	pub fn verify_git_import_last(
	1865	&self,
	1866	git_dir: &Path,
	1867	branch: &str,
	1868	last: Option<usize>,
8b7986dd Andy McCall 27 days ago ↶	1869	max: usize,
7eac74ab Andy McCall 1 month ago ↶	1870	) -> Result<GitVerifyReport> {
f0406c3b Andy McCall 27 days ago ↶	1871	let mut rep = GitVerifyReport::default();
f0406c3b Andy McCall 27 days ago ↶	1872	let mut commits = list_commits(git_dir, branch)?;
62794ecc Andy McCall 1 month ago ↶	1873	let imported = self.imported_commits()?;
f0406c3b Andy McCall 27 days ago ↶	1874	// An import still going (or stopped) has reached part of the
	1875	// branch: what follows its last imported commit is not checked,
	1876	// only counted, and --last counts back from there (oxygen#81).
	1877	if let Some(end) = commits.iter().rposition(\|c\| imported.contains_key(&c.sha)) {
	1878	let rest = commits.len() - end - 1;
	1879	if rest > 0 {
	1880	commits.truncate(end + 1);
	1881	}
	1882	rep.not_yet = rest;
	1883	}
62794ecc Andy McCall 1 month ago ↶	1884	let mut cat = CatFile::start(git_dir)?;
	1885	// Changes of each commit and all its ancestors, built up in topological order.
	1886	let mut closure: HashMap<String, Arc<BTreeSet<ChangeHash>>> = HashMap::new();
a87c7b9d Andy McCall 1 month ago ↶	1887	// Graphs (by key) and blobs already found equal to a Git blob.
	1888	let mut verified_graphs: HashMap<[u8; 32], String> = HashMap::new();
	1889	let mut verified_blobs: HashMap<ObjectRef, String> = HashMap::new();
57d8dc05 Andy McCall 1 month ago ↶	1890	// The state of a commit whose first-parent children are still to
	1891	// come, so a child adds its own changes instead of loading all its
	1892	// history again (as the import does).
	1893	let mut waiting: HashMap<&str, usize> = HashMap::new();
ee6aa3e8 Andy McCall 28 days ago ↶	1894	// Children still to come per commit, through any parent: a commit's
	1895	// closure is dropped after its last child, so only the open
	1896	// frontier is held. Kept for every commit, the closures grew with
	1897	// the square of the history and VS Code's verify ran out of memory
	1898	// (oxygen#81).
	1899	let mut children: HashMap<&str, usize> = HashMap::new();
62794ecc Andy McCall 1 month ago ↶	1900	for c in &commits {
57d8dc05 Andy McCall 1 month ago ↶	1901	if let Some(p1) = c.parents.first() {
	1902	*waiting.entry(p1.as_str()).or_default() += 1;
	1903	}
ee6aa3e8 Andy McCall 28 days ago ↶	1904	for p in &c.parents {
	1905	*children.entry(p.as_str()).or_default() += 1;
	1906	}
57d8dc05 Andy McCall 1 month ago ↶	1907	}
	1908	let mut kept: HashMap<String, State> = HashMap::new();
	1909	let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
	1910	for c in &commits {
62794ecc Andy McCall 1 month ago ↶	1911	rep.commits += 1;
ee6aa3e8 Andy McCall 28 days ago ↶	1912	// The first parent's closure, taken over when this is its last
ee6aa3e8 Andy McCall 28 days ago ↶	1913	// child (no copy), then the other parents' added.
62794ecc Andy McCall 1 month ago ↶	1914	let mut set: BTreeSet<ChangeHash> = BTreeSet::new();
ee6aa3e8 Andy McCall 28 days ago ↶	1915	for (i, p) in c.parents.iter().enumerate() {
	1916	let left = children.get_mut(p.as_str()).map(\|n\| {
	1917	*n -= 1;
	1918	*n
	1919	});
	1920	let ps = if left == Some(0) {
	1921	closure.remove(p)
	1922	} else {
	1923	closure.get(p).cloned()
	1924	};
	1925	if let Some(ps) = ps {
	1926	if i == 0 && set.is_empty() {
	1927	set = Arc::try_unwrap(ps).unwrap_or_else(\|a\| (*a).clone());
	1928	} else {
	1929	set.extend(ps.iter().copied());
	1930	}
62794ecc Andy McCall 1 month ago ↶	1931	}
	1932	}
	1933	match imported.get(&c.sha) {
	1934	Some(h) => {
	1935	set.insert(*h);
	1936	}
	1937	None => {
	1938	rep.mismatches.push((
	1939	c.sha.clone(),
	1940	"no imported change carries this commit".into(),
	1941	));
	1942	}
	1943	}
	1944	let set = Arc::new(set);
ee6aa3e8 Andy McCall 28 days ago ↶	1945	if children.get(c.sha.as_str()).is_some_and(\|n\| *n > 0) {
	1946	closure.insert(c.sha.clone(), set.clone());
	1947	}
8b7986dd Andy McCall 27 days ago ↶	1948	if max > 0 && rep.mismatches.len() >= max {
62794ecc Andy McCall 1 month ago ↶	1949	break;
7eac74ab Andy McCall 1 month ago ↶	1950	}
	1951	if last.is_some_and(\|n\| rep.commits + n <= commits.len()) {
	1952	rep.skipped += 1;
	1953	continue;
62794ecc Andy McCall 1 month ago ↶	1954	}
62794ecc Andy McCall 1 month ago ↶	1955	let heads: Vec<ChangeHash> = set.iter().copied().collect();
a87c7b9d Andy McCall 1 month ago ↶	1956	// History only: a file whose graph (by key) or blob was already
	1957	// found equal to this Git blob is not built, rendered or read
	1958	// again; the rest are.
57d8dc05 Andy McCall 1 month ago ↶	1959	let from_parent = c.parents.first().and_then(\|p1\| {
	1960	let left = waiting.get_mut(p1.as_str()).map(\|n\| {
	1961	*n -= 1;
	1962	*n
	1963	});
	1964	if left == Some(0) {
	1965	kept_order.retain(\|k\| k != p1);
	1966	kept.remove(p1.as_str())
	1967	} else {
	1968	kept.get(p1.as_str()).cloned()
	1969	}
	1970	});
	1971	let state = match from_parent {
	1972	Some(mut st) => {
	1973	// The changes the parent's state lacks. The state never
	1974	// builds a graph (they are built beside it), so they
	1975	// are read without their text.
	1976	for h in set.iter() {
	1977	if !st.contains(h) {
	1978	let (change, texts) = self.store.get_change_lazy(h)?;
	1979	st.add_known(*h, change, texts);
	1980	}
	1981	}
	1982	st
	1983	}
	1984	None => self.load_state_history(&heads)?,
	1985	};
a87c7b9d Andy McCall 1 month ago ↶	1986	let view = state.tree().view();
62794ecc Andy McCall 1 month ago ↶	1987	let tree = list_tree(git_dir, &c.sha)?;
	1988	let short = &c.sha[..12.min(c.sha.len())];
	1989	let mut differ = \|what: String\| rep.mismatches.push((c.sha.clone(), what));
a87c7b9d Andy McCall 1 month ago ↶	1990	let mut conflicted: Vec<String> =
	1991	view.conflicts.iter().map(\|c\| format!("{c:?}")).collect();
	1992	let mut need: Vec<NodeId> = Vec::new();
	1993	for (path, (mode, sha)) in &tree {
	1994	let Some(e) = view.entries.get(path) else {
	1995	continue;
	1996	};
	1997	if mode != "160000"
	1998	&& mode != "120000"
	1999	&& e.kind == NodeKind::File
	2000	&& !e.mode.opaque
	2001	{
	2002	let known = state
	2003	.graph_key(&e.node)
	2004	.is_some_and(\|k\| verified_graphs.get(&k) == Some(sha));
	2005	if !known {
	2006	need.push(e.node);
62794ecc Andy McCall 1 month ago ↶	2007	}
	2008	}
	2009	}
a87c7b9d Andy McCall 1 month ago ↶	2010	// A graph's key is known only before it is built.
	2011	let keys: HashMap<NodeId, [u8; 32]> = need
	2012	.iter()
	2013	.filter_map(\|n\| Some((*n, state.graph_key(n)?)))
	2014	.collect();
	2015	let need_set: HashSet<NodeId> = need.iter().copied().collect();
57d8dc05 Andy McCall 1 month ago ↶	2016	let graphs = self.build_graphs_detached(&state, &need)?;
62794ecc Andy McCall 1 month ago ↶	2017	for (path, (mode, sha)) in &tree {
a87c7b9d Andy McCall 1 month ago ↶	2018	let Some(e) = view.entries.get(path) else {
62794ecc Andy McCall 1 month ago ↶	2019	differ(format!("{short}: {path} is missing"));
	2020	continue;
	2021	};
	2022	rep.files_compared += 1;
	2023	match mode.as_str() {
a87c7b9d Andy McCall 1 month ago ↶	2024	"160000" => {
	2025	let blobs = state.live_blobs(&e.node);
	2026	match (e.kind, blobs.as_slice()) {
	2027	(NodeKind::Subrepo, [b]) => {
	2028	let pin = crate::subrepo::Pin::decode(&self.read_blob(b)?)?;
	2029	if !matches!(&pin, crate::subrepo::Pin::Git { sha: s, .. } if s == sha)
	2030	{
	2031	differ(format!("{short}: {path} pins another commit"));
	2032	}
62794ecc Andy McCall 1 month ago ↶	2033	}
a87c7b9d Andy McCall 1 month ago ↶	2034	(kind, _) => differ(format!(
	2035	"{short}: {path} is not one submodule pin: {kind:?} with {} pins",
	2036	blobs.len()
	2037	)),
62794ecc Andy McCall 1 month ago ↶	2038	}
a87c7b9d Andy McCall 1 month ago ↶	2039	}
62794ecc Andy McCall 1 month ago ↶	2040	"120000" => {
62794ecc Andy McCall 1 month ago ↶	2041	let target = cat.read(sha)?;
a87c7b9d Andy McCall 1 month ago ↶	2042	let blobs = state.live_blobs(&e.node);
	2043	match (e.kind, blobs.as_slice()) {
	2044	(NodeKind::Symlink, [b]) if self.read_blob(b)? == target => {}
62794ecc Andy McCall 1 month ago ↶	2045	_ => differ(format!("{short}: symlink {path} differs")),
	2046	}
	2047	}
	2048	_ => {
a87c7b9d Andy McCall 1 month ago ↶	2049	if e.mode.exec != (mode == "100755") {
	2050	differ(format!("{short}: {path} executable bit differs"));
	2051	}
	2052	if e.kind == NodeKind::File && !e.mode.opaque {
	2053	if !need_set.contains(&e.node) {
	2054	// Known equal: skipped when the graphs were chosen.
62794ecc Andy McCall 1 month ago ↶	2055	continue;
62794ecc Andy McCall 1 month ago ↶	2056	}
57d8dc05 Andy McCall 1 month ago ↶	2057	let (have, m) = match graphs.get(&e.node) {
	2058	Some(g) => state.render_graph(g),
	2059	None => (Vec::new(), Default::default()),
	2060	};
a87c7b9d Andy McCall 1 month ago ↶	2061	if m.is_conflicted() {
57d8dc05 Andy McCall 1 month ago ↶	2062	// Who conflicts: a copy of the state with this graph built.
	2063	let mut detail = state.clone();
	2064	self.load_graphs_of(&mut detail, &[e.node])?;
	2065	let who: Vec<String> = detail
a87c7b9d Andy McCall 1 month ago ↶	2066	.conflict_regions(&e.node)
	2067	.contributors
	2068	.iter()
	2069	.filter_map(\|h\| state.get(h))
	2070	.map(\|m\| {
	2071	let sha = m
	2072	.message
	2073	.lines()
	2074	.rev()
	2075	.find_map(\|l\| l.strip_prefix("Git-Commit: "))
	2076	.unwrap_or("?");
	2077	format!(
	2078	"{} {:?}",
	2079	&sha[..12.min(sha.len())],
	2080	m.message.lines().next().unwrap_or("")
	2081	)
	2082	})
	2083	.collect();
	2084	conflicted.push(format!("{path} between {}", who.join(" and ")));
	2085	}
	2086	let want = cat.read(sha)?;
	2087	if have != want {
	2088	differ(format!(
	2089	"{short}: {path} differs: {}",
	2090	first_difference(&have, &want)
	2091	));
	2092	} else if !m.is_conflicted() {
	2093	if let Some(k) = keys.get(&e.node) {
	2094	verified_graphs.insert(*k, sha.clone());
	2095	}
	2096	}
	2097	continue;
	2098	}
	2099	let blobs = state.live_blobs(&e.node);
	2100	let [b] = blobs.as_slice() else {
	2101	differ(format!("{short}: {path} is not one file"));
	2102	continue;
62794ecc Andy McCall 1 month ago ↶	2103	};
a87c7b9d Andy McCall 1 month ago ↶	2104	if verified_blobs.get(b) == Some(sha) {
a87c7b9d Andy McCall 1 month ago ↶	2105	continue;
62794ecc Andy McCall 1 month ago ↶	2106	}
a87c7b9d Andy McCall 1 month ago ↶	2107	if self.read_blob(b)? != cat.read(sha)? {
	2108	differ(format!("{short}: {path} differs"));
	2109	} else {
	2110	verified_blobs.insert(*b, sha.clone());
62794ecc Andy McCall 1 month ago ↶	2111	}
	2112	}
	2113	}
	2114	}
a87c7b9d Andy McCall 1 month ago ↶	2115	if !conflicted.is_empty() {
	2116	differ(format!(
	2117	"{short}: the state is conflicted: {}",
	2118	conflicted.join("; ")
	2119	));
	2120	}
62794ecc Andy McCall 1 month ago ↶	2121	// Import records submodules in `.oxymodules` as well (spec §23),
	2122	// and export leaves it out again.
	2123	let has_submodules = tree.values().any(\|(m, _)\| m == "160000");
e83354cd Andy McCall 1 month ago ↶	2124	for (path, e) in view.entries.iter() {
a87c7b9d Andy McCall 1 month ago ↶	2125	if e.kind != NodeKind::Dir
62794ecc Andy McCall 1 month ago ↶	2126	&& !tree.contains_key(path)
	2127	&& !(has_submodules && path == ".oxymodules")
	2128	{
	2129	differ(format!("{short}: {path} is extra"));
57d8dc05 Andy McCall 1 month ago ↶	2130	}
	2131	}
	2132	if waiting.get(c.sha.as_str()).is_some_and(\|n\| *n > 0) {
	2133	kept.insert(c.sha.clone(), state);
	2134	kept_order.push_back(c.sha.clone());
	2135	if kept_order.len() > 8 {
	2136	if let Some(old) = kept_order.pop_front() {
	2137	kept.remove(&old);
	2138	}
62794ecc Andy McCall 1 month ago ↶	2139	}
62794ecc Andy McCall 1 month ago ↶	2140	}
8b7986dd Andy McCall 27 days ago ↶	2141	if max > 0 && rep.mismatches.len() >= max {
62794ecc Andy McCall 1 month ago ↶	2142	break;
	2143	}
	2144	}
	2145	Ok(rep)
	2146	}
	2147	}
	2148
	2149	/// Where two versions of a file first part, for a verification report.
	2150	fn first_difference(have: &[u8], want: &[u8]) -> String {
	2151	let (h, w): (Vec<&[u8]>, Vec<&[u8]>) = (
	2152	have.split(\|b\| *b == b'\n').collect(),
	2153	want.split(\|b\| *b == b'\n').collect(),
	2154	);
	2155	let i = h
	2156	.iter()
	2157	.zip(&w)
	2158	.position(\|(a, b)\| a != b)
	2159	.unwrap_or(h.len().min(w.len()));
	2160	let show = \|l: Option<&&[u8]>\| {
	2161	l.map(\|l\| String::from_utf8_lossy(&l[..l.len().min(60)]).into_owned())
	2162	.unwrap_or_else(\|\| "(end)".into())
	2163	};
	2164	format!(
	2165	"line {}: has {:?}, Git has {:?} ({} lines, Git has {})",
	2166	i + 1,
	2167	show(h.get(i)),
	2168	show(w.get(i)),
	2169	h.len(),
	2170	w.len()
	2171	)
6f4226bf Andy McCall 4 days ago ↶	2172	}
	2173
	2174	/// What `git_export_graph` wrote.
	2175	#[derive(Debug, Clone, Default, PartialEq, Eq)]
	2176	pub struct GitGraphReport {
	2177	/// Changes written as commits this time (the rest were exported before).
	2178	pub commits: usize,
	2179	/// Branches and tags pointed at their commits.
	2180	pub branches: usize,
	2181	pub tags: usize,
	2182	/// Bookmarks or tags whose names Git does not take as refs.
	2183	pub skipped_refs: Vec<String>,
	2184	/// Subrepo nodes with Oxygen pins, which Git cannot represent.
	2185	pub skipped_subrepos: usize,
	2186	}
	2187
	2188	/// A bookmark or tag name Git takes as a ref without escaping.
	2189	fn git_ref_ok(name: &str) -> bool {
	2190	!name.is_empty()
	2191	&& !name.starts_with(['-', '/', '.'])
	2192	&& !name.ends_with(['/', '.'])
	2193	&& !name.contains("..")
	2194	&& !name.contains("//")
	2195	&& !name.ends_with(".lock")
	2196	&& name
	2197	.bytes()
	2198	.all(\|b\| b.is_ascii_alphanumeric() \|\| b"-_./+".contains(&b))
	2199	}
	2200
	2201	impl Repo {
	2202	/// Export every bookmark and tag to the bare Git repository at
	2203	/// `git_dir` from the change graph (spec §23, oxygenhub D1), for a hub
	2204	/// to serve to Git clients. Unlike `git_export`, which follows one
	2205	/// checkout's operation log, this depends only on the changes, so the
	2206	/// same changes give the same Git commits wherever it runs:
	2207	///
	2208	/// - each change is a commit whose tree is the state of the change and
	2209	/// everything it depends on, whose parents are its dependencies'
	2210	/// commits, with the change's author, time and message and its
	2211	/// `Change-Id:` and `Oxy-Hash:` trailers (a change that depends on
	2212	/// nothing is a root, so Git sees Oxygen's dependencies, not the order
	2213	/// changes were made in);
	2214	/// - a bookmark with one head is a branch at that head's commit; with
	2215	/// several, a merge of their commits whose tree is the bookmark's
	2216	/// state, at the newest head's time;
	2217	/// - a tag points at its heads' commit, or such a merge.
	2218	///
	2219	/// Commits already exported are remembered (marks), so a later run
	2220	/// writes only new changes. Branches for bookmarks that are gone are
	2221	/// deleted.
	2222	pub fn git_export_graph(&self, git_dir: &Path) -> Result<GitGraphReport> {
	2223	if !git_dir.join("HEAD").exists() {
	2224	fs::create_dir_all(git_dir)?;
	2225	git(git_dir, &["init", "--bare", "-q"])?;
	2226	}
	2227	let marks_dir = self.git_marks_dir(git_dir)?;
	2228	let git_marks = marks_dir.join("git-marks");
	2229	let graph_marks = marks_dir.join("graph-marks");
	2230	let mut marks = Self::read_oxy_marks(&graph_marks)?;
	2231	let mut next = marks.values().copied().max().unwrap_or(0) + 1;
	2232	let mut report = GitGraphReport::default();
	2233	let mut stream: Vec<u8> = Vec::new();
	2234
	2235	let mut refs: Vec<(String, Vec<ChangeHash>)> = Vec::new();
2236	for b in self.bookmarks()? {
2237	if git_ref_ok(&b.name) {
2238	refs.push((format!("refs/heads/{}", b.name), b.heads));
2239	} else {
2240	report.skipped_refs.push(b.name);
2241	}
2242	}
2243	let tags = self.tags()?;
2244	for t in &tags {
2245	if git_ref_ok(&t.name) {
2246	refs.push((format!("refs/tags/{}", t.name), t.heads.clone()));
2247	} else {
2248	report.skipped_refs.push(t.name.clone());
2249	}
2250	}
2251
2252	// Every change the refs reach, dependencies before dependents: a
2253	// depth-first walk in hash order, so the order is the same everywhere.
2254	let mut order: Vec<ChangeHash> = Vec::new();
2255	let mut seen: HashSet<ChangeHash> = HashSet::new();
2256	let mut metas: HashMap<ChangeHash, crate::state::ChangeMeta> = HashMap::new();
2257	let mut roots: Vec<ChangeHash> = refs.iter().flat_map(\|(_, h)\| h.iter().copied()).collect();
2258	roots.sort();
2259	roots.dedup();
2260	for root in roots {
2261	if !seen.insert(root) {
2262	continue;
2263	}
2264	let mut stack: Vec<(ChangeHash, bool)> = vec![(root, false)];
2265	while let Some((h, done)) = stack.pop() {
2266	if done {
2267	order.push(h);
2268	continue;
2269	}
2270	stack.push((h, true));
2271	let meta = self.store.get_change_meta(&h)?;
2272	let mut deps = meta.deps.clone();
2273	deps.sort();
2274	for d in deps.into_iter().rev() {
2275	if seen.insert(d) {
2276	stack.push((d, false));
2277	}
2278	}
2279	metas.insert(h, meta);
2280	}
2281	}
2282
2283	for h in &order {
2284	if marks.contains_key(h) {
2285	continue;
2286	}
2287	let meta = &metas[h];
2288	let snapshot = self.load_state(&[*h])?.materialize();
2289	let mut message = meta.message.clone();
2290	if !message.ends_with('\n') {
2291	message.push('\n');
2292	}
2293	message.push_str(&format!(
2294	"\nChange-Id: {}\nOxy-Hash: {}\n",
2295	meta.change_id.to_hex(),
2296	h.to_hex()
2297	));
2298	if snapshot.is_conflicted() {
2299	message.push_str("Oxy-Conflicted: true\n");
2300	}
2301	let mut deps = meta.deps.clone();
2302	deps.sort();
2303	let parents: Vec<String> = deps
2304	.iter()
2305	.filter_map(\|d\| marks.get(d))
2306	.map(\|m\| format!(":{m}"))
2307	.collect();
2308	let mark = next;
2309	next += 1;
2310	if parents.is_empty() {
2311	// Without `from`, fast-import would make the scratch ref's
2312	// tip its parent; a change that depends on nothing is a root.
2313	stream.extend_from_slice(b"reset refs/oxy/export\n\n");
2314	}
2315	self.write_commit(
2316	&mut stream,
2317	"refs/oxy/export",
2318	mark,
2319	&self.git_author_key(&meta.author),
2320	meta.hlc.wall_ms / 1000,
2321	&message,
2322	&parents,
2323	&snapshot,
2324	)?;
2325	marks.insert(*h, mark);
2326	report.commits += 1;
2327	}
2328
2329	let mut wanted: HashSet<String> = HashSet::new();
2330	for (name, heads) in &refs {
2331	let mut heads = heads.clone();
2332	heads.sort();
2333	heads.dedup();
2334	let target = match heads.as_slice() {
2335	[] => continue,
2336	[one] => format!(":{}", marks[one]),
2337	many => {
2338	// A merge of the heads, named by them so it is made once.
2339	let key = hash_object(
2340	ObjectType::Chunk,
2341	many.iter()
2342	.map(\|h\| h.to_hex())
2343	.collect::<Vec<_>>()
2344	.join(" ")
2345	.as_bytes(),
2346	);
2347	if let Some(m) = marks.get(&key) {
2348	format!(":{m}")
2349	} else {
2350	let snapshot = self.load_state(many)?.materialize();
2351	let newest = many.iter().map(\|h\| metas[h].hlc.wall_ms).max().unwrap_or(0);
2352	let mut message = format!(
2353	"Merge of {} heads\n\nOxy-Heads: {}\n",
2354	many.len(),
2355	many.iter()
2356	.map(\|h\| h.to_hex())
2357	.collect::<Vec<_>>()
2358	.join(" ")
2359	);
2360	if snapshot.is_conflicted() {
2361	message.push_str("Oxy-Conflicted: true\n");
2362	}
2363	let parents: Vec<String> =
2364	many.iter().map(\|h\| format!(":{}", marks[h])).collect();
2365	let mark = next;
2366	next += 1;
2367	self.write_commit(
2368	&mut stream,
2369	"refs/oxy/export",
2370	mark,
2371	"oxy <oxy@localhost>",
2372	newest / 1000,
2373	&message,
2374	&parents,
2375	&snapshot,
2376	)?;
2377	marks.insert(key, mark);
2378	format!(":{mark}")
2379	}
2380	}
2381	};
2382	stream.extend_from_slice(format!("reset {name}\nfrom {target}\n\n").as_bytes());
2383	wanted.insert(name.clone());
2384	if name.starts_with("refs/tags/") {
2385	report.tags += 1;
2386	} else {
2387	report.branches += 1;
2388	}
2389	}
2390	stream.extend_from_slice(b"done\n");
2391	run_fast_import(git_dir, &stream, &git_marks)?;
2392	let text: String = marks
2393	.iter()
2394	.map(\|(h, m)\| format!("{} {m}\n", h.to_hex()))
2395	.collect();
2396	write_atomic(&graph_marks, text.as_bytes())?;
2397	let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]);
2398	// Branches and tags of bookmarks and tags that are gone.
2399	let listed = git(
2400	git_dir,
2401	&[
2402	"for-each-ref",
2403	"--format=%(refname)",
2404	"refs/heads",
2405	"refs/tags",
2406	],
2407	)?;
2408	for r in listed
2409	.lines()
2410	.map(str::trim)
2411	.filter(\|r\| !r.is_empty() && !wanted.contains(*r))
2412	{
2413	git(git_dir, &["update-ref", "-d", r])?;
2414	}
2415	// HEAD names the main bookmark when there is one, so a clone checks it out.
2416	let main = ["main", "master", "trunk", "develop"]
2417	.into_iter()
2418	.find(\|b\| wanted.contains(&format!("refs/heads/{b}")));
2419	if let Some(b) = main {
2420	git(
2421	git_dir,
2422	&["symbolic-ref", "HEAD", &format!("refs/heads/{b}")],
2423	)?;
2424	}
2425	report.skipped_subrepos = self.git_export_skipped.replace(0);
2426	Ok(report)
2427	}
2428
2429	/// `Name <email>` for a Git commit, from the author's identity.
2430	fn git_author_key(&self, key: &KeyId) -> String {
2431	match self.store.get_identity(key) {
2432	Ok(id) if !id.name.is_empty() => format!(
2433	"{} <{}>",
2434	id.name.replace(['<', '>', '\n'], " "),
2435	id.email.replace(['<', '>', '\n', ' '], "")
2436	),
2437	_ => "oxy <oxy@localhost>".to_string(),
2438	}
2439	}
2440	}
2441
2442	/// Feed a fast-import stream to `git fast-import` in `git_dir`, keeping its
2443	/// marks in `git_marks` so a later run can refer to commits it wrote.
2444	fn run_fast_import(git_dir: &Path, stream: &[u8], git_marks: &Path) -> Result<()> {
2445	let mut cmd = Command::new("git");
2446	// `--force`: a ref may move to an unrelated commit (a bookmark rewritten).
2447	cmd.arg("-C")
2448	.arg(git_dir)
2449	.args(["fast-import", "--quiet", "--done", "--force"])
2450	.arg(format!("--export-marks={}", git_marks.display()));
2451	if git_marks.exists() {
2452	cmd.arg(format!("--import-marks={}", git_marks.display()));
2453	}
2454	let mut child = cmd
2455	.stdin(Stdio::piped())
2456	.stdout(Stdio::null())
2457	.stderr(Stdio::piped())
2458	.spawn()
2459	.map_err(\|e\| RepoError::Msg(format!("running git fast-import: {e}")))?;
2460	child.stdin.take().unwrap().write_all(stream)?;
2461	let out = child.wait_with_output()?;
2462	if !out.status.success() {
2463	return fail(format!(
2464	"git fast-import failed: {}",
2465	String::from_utf8_lossy(&out.stderr).trim()
2466	));
2467	}
2468	Ok(())
62794ecc Andy McCall 1 month ago ↶	2469	}
c397a8c3 Andy McCall 4 days ago ↶	2470
	2471	impl Repo {
	2472	/// Bring the side-by-side Git repository up to date (`git.colocate`):
	2473	/// a `.git` beside `.oxy`, made with `git init` when missing and told to
	2474	/// leave `.oxy/` alone, holding the current bookmark as exported by
	2475	/// `git_export_as` (one commit per state the bookmark passed through),
	2476	/// with HEAD on that branch and Git's index at its tree, so `git status`
	2477	/// shows only the working copy's uncommitted work. Files on disk are
	2478	/// never touched. Does nothing when the bookmark has not moved since the
	2479	/// last time; `Ok(true)` when it exported.
	2480	pub fn git_colocate_sync(&self) -> Result<bool> {
	2481	if self.bare {
	2482	return fail("a bare repository has no working copy for a .git beside it");
	2483	}
	2484	let git_dir = self.root.join(".git");
	2485	if git_dir.is_file() {
	2486	return fail(".git is a file (a Git worktree or submodule link); remove it to keep a side-by-side Git repository here");
	2487	}
	2488	if !git_dir.join("HEAD").exists() {
	2489	git(&self.root, &["init", "-q"])?;
	2490	}
	2491	let exclude = git_dir.join("info").join("exclude");
	2492	let have = fs::read_to_string(&exclude).unwrap_or_default();
	2493	if !have.lines().any(\|l\| l.trim() == "/.oxy/") {
	2494	fs::create_dir_all(exclude.parent().expect("info/"))?;
	2495	let sep = if have.is_empty() \|\| have.ends_with('\n') {
	2496	""
	2497	} else {
	2498	"\n"
	2499	};
	2500	fs::write(
	2501	&exclude,
	2502	format!("{have}{sep}# Oxygen's own repository (oxy git colocate)\n/.oxy/\n"),
	2503	)?;
	2504	}
	2505	let bookmark = self.config.bookmark.clone();
	2506	let state = self
	2507	.view()?
	2508	.bookmarks
	2509	.get(&bookmark)
	2510	.map(\|s\| s.to_hex())
	2511	.unwrap_or_default();
	2512	let stamp_file = self.oxy.join("git").join("colocate-stamp");
	2513	let stamp = format!("{bookmark} {state}");
	2514	if fs::read_to_string(&stamp_file).is_ok_and(\|s\| s.trim() == stamp) {
	2515	return Ok(false);
	2516	}
	2517	if !state.is_empty() {
	2518	self.git_export_as(&git_dir, &bookmark)?;
	2519	git(
	2520	&self.root,
	2521	&["symbolic-ref", "HEAD", &format!("refs/heads/{bookmark}")],
	2522	)?;
	2523	git(&self.root, &["read-tree", "HEAD"])?;
	2524	}
	2525	fs::create_dir_all(stamp_file.parent().expect("git/"))?;
	2526	fs::write(&stamp_file, stamp)?;
	2527	Ok(!state.is_empty())
	2528	}
	2529	}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

Keyboard