//! Git bridge (spec §23), export side: each state the current bookmark //! passed through becomes a Git commit through `git fast-import`, with the //! state's full tree and the previous state's commit as parent. A state //! that added exactly one change carries that change's message, author, //! and `Change-Id`/`Oxy-Hash` trailers; every commit carries `Oxy-State`. //! //! Marks make re-exports incremental: `.oxy/git//git-marks` is //! git's own marks file and `oxy-marks` maps state refs to marks. use super::*; use std::process::{Command, Stdio}; use std::sync::Arc; #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct GitImportReport { pub commits: usize, pub merges: usize, /// Gitlink entries imported as subrepo nodes with Git pins. pub submodules: usize, pub heads: usize, } /// A running `git cat-file --batch`, for reading many blobs cheaply. struct CatFile { child: std::process::Child, stdin: std::process::ChildStdin, stdout: std::io::BufReader, } impl CatFile { fn start(git_dir: &Path) -> Result { let mut child = Command::new("git") .arg("-C") .arg(git_dir) .args(["cat-file", "--batch"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()) .spawn() .map_err(|e| RepoError::Msg(format!("running git cat-file: {e}")))?; let stdin = child.stdin.take().unwrap(); let stdout = std::io::BufReader::new(child.stdout.take().unwrap()); Ok(CatFile { child, stdin, stdout, }) } fn read(&mut self, sha: &str) -> Result> { use std::io::{BufRead, Read}; writeln!(self.stdin, "{sha}")?; self.stdin.flush()?; let mut header = String::new(); self.stdout.read_line(&mut header)?; let parts: Vec<&str> = header.trim_end().split(' ').collect(); if parts.len() != 3 { return fail(format!("git cat-file: object {sha} not found")); } let size: usize = parts[2] .parse() .map_err(|_| RepoError::Msg("git cat-file: bad size".into()))?; let mut body = vec![0u8; size]; self.stdout.read_exact(&mut body)?; let mut nl = [0u8; 1]; self.stdout.read_exact(&mut nl)?; Ok(body) } } impl Drop for CatFile { fn drop(&mut self) { let _ = self.child.kill(); let _ = self.child.wait(); } } /// One commit as `git log` describes it. struct GitCommit { sha: String, parents: Vec, author_secs: u64, author: String, message: String, } fn list_commits(git_dir: &Path, branch: &str) -> Result> { let out = Command::new("git") .arg("-C") .arg(git_dir) .args([ "log", "--reverse", "--topo-order", "--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e", branch, ]) .output() .map_err(|e| RepoError::Msg(format!("running git log: {e}")))?; if !out.status.success() { return fail(format!( "git log {branch} failed: {}", String::from_utf8_lossy(&out.stderr).trim() )); } let text = String::from_utf8_lossy(&out.stdout); let mut commits = Vec::new(); for record in text.split('\x1e') { let record = record.trim_start_matches('\n'); if record.trim().is_empty() { continue; } let f: Vec<&str> = record.splitn(5, '\0').collect(); if f.len() != 5 { return fail("unexpected git log record"); } commits.push(GitCommit { sha: f[0].to_string(), parents: f[1].split_whitespace().map(str::to_string).collect(), author_secs: f[2].parse().unwrap_or(0), author: f[3].to_string(), message: f[4].trim_end().to_string(), }); } Ok(commits) } /// `path → (mode, blob sha)` for a commit's tree. fn list_tree(git_dir: &Path, sha: &str) -> Result> { let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?; let mut out = BTreeMap::new(); for entry in text.split('\0') { if entry.is_empty() { continue; } let Some((meta, path)) = entry.split_once('\t') else { continue; }; let m: Vec<&str> = meta.split(' ').collect(); if m.len() == 3 { out.insert(path.to_string(), (m[0].to_string(), m[2].to_string())); } } Ok(out) } /// The ChangeId for a Git commit: its `Change-Id` trailer if it has a /// 32-hex one, else BLAKE3-128 of the commit hash (spec §23). fn change_id_for(sha: &str, message: &str) -> ChangeId { for line in message.lines().rev() { if let Some(v) = line.strip_prefix("Change-Id:") { if let Some(id) = ChangeId::from_hex(v.trim()) { return id; } } } let h = blake3::hash(sha.as_bytes()); ChangeId(h.as_bytes()[..16].try_into().unwrap()) } /// Git commits with a known Oxygen counterpart. struct GitKnown { /// Imported commit → its change. imported: HashMap, /// Exported commit (or a fetched tip) → the bookmark state it equals. exported: HashMap, } #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct GitExportReport { pub exported: usize, pub skipped: usize, /// Subrepo nodes with Oxygen pins, which Git cannot represent. pub skipped_subrepos: usize, pub branch: String, pub heads: usize, } fn git(dir: &Path, args: &[&str]) -> Result { let out = Command::new("git") .arg("-C") .arg(dir) .args(args) .output() .map_err(|e| RepoError::Msg(format!("running git: {e}")))?; if !out.status.success() { return fail(format!( "git {} failed: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim() )); } Ok(String::from_utf8_lossy(&out.stdout).trim().to_string()) } impl Repo { fn git_marks_dir(&self, git_dir: &Path) -> Result { let canonical = git_dir .canonicalize() .unwrap_or_else(|_| git_dir.to_path_buf()); let key = hash_object(ObjectType::Chunk, canonical.to_string_lossy().as_bytes()).short(16); let dir = self.oxy.join("git").join(key); fs::create_dir_all(&dir)?; Ok(dir) } /// `state ref → Git commit sha` for bookmark states that equal a Git /// commit's tree (recorded by import and fetch). fn read_git_states(path: &Path) -> Result> { let mut out = BTreeMap::new(); if let Ok(text) = fs::read_to_string(path) { for line in text.lines() { if let Some((h, sha)) = line.split_once(' ') { if let Some(h) = ObjectRef::from_hex(h) { out.insert(h, sha.trim().to_string()); } } } } Ok(out) } fn record_git_state(&self, git_dir: &Path, state: &ObjectRef, sha: &str) -> Result<()> { let path = self.git_marks_dir(git_dir)?.join("git-states"); let mut states = Self::read_git_states(&path)?; states.insert(*state, sha.to_string()); let text: String = states .iter() .map(|(h, s)| format!("{} {s}\n", h.to_hex())) .collect(); write_atomic(&path, text.as_bytes()) } /// The bare mirror this repository keeps for the Git remote `url`. fn git_mirror(&self, url: &str) -> Result { let key = hash_object(ObjectType::Chunk, url.as_bytes()).short(16); let dir = self.oxy.join("git").join(key).join("mirror.git"); if !dir.join("HEAD").exists() { fs::create_dir_all(&dir)?; git(&dir, &["init", "--bare", "-q"])?; } Ok(dir) } /// Git commits already imported into the bookmark, by sha. fn imported_commits(&self) -> Result> { let mut map = HashMap::new(); for e in self.log()? { if e.is_working_copy { continue; } if let Some(sha) = e .change .message .lines() .rev() .find_map(|l| l.strip_prefix("Git-Commit: ")) { map.insert(sha.trim().to_string(), e.hash); } } Ok(map) } /// Bring the mirror's `branch` up to date with the Git remote `url`. /// A remote without that branch yet leaves the mirror as it is. fn git_mirror_fetch(&self, mirror: &Path, url: &str, branch: &str) -> Result<()> { let spec = format!("+refs/heads/{branch}:refs/heads/{branch}"); match git(mirror, &["fetch", "-q", url, &spec]) { Ok(_) => Ok(()), Err(e) => { let ls = git(mirror, &["ls-remote", "--heads", url, branch]).unwrap_or_default(); if ls.trim().is_empty() { Ok(()) } else { Err(e) } } } } /// Fetch `branch` of the Git repository at `url` (a path or a Git URL) /// through a mirror and import the commits not yet in the bookmark /// (spec §23). Works on a repository that already has history: the /// new changes join the bookmark like a pull. pub fn git_fetch(&self, url: &str, branch: &str) -> Result { let mirror = self.git_mirror(url)?; self.git_mirror_fetch(&mirror, url, branch)?; if git( &mirror, &[ "rev-parse", "--verify", "-q", &format!("refs/heads/{branch}"), ], ) .is_err() { return fail(format!("{url} has no branch {branch:?}")); } self.import_git_commits(&mirror, branch, &format!("oxy git fetch {url} {branch}")) } /// Export the bookmark to `branch` of the Git repository at `url` /// through the mirror and push it. The remote's tip must already have /// been fetched (`git_fetch`), so the push fast-forwards. pub fn git_push(&self, url: &str, branch: &str) -> Result { let mirror = self.git_mirror(url)?; self.git_mirror_fetch(&mirror, url, branch)?; if let Ok(tip) = git( &mirror, &[ "rev-parse", "--verify", "-q", &format!("refs/heads/{branch}"), ], ) { let known = self.git_known(&mirror)?; if !known.imported.contains_key(&tip) && !known.exported.contains_key(&tip) { return fail(format!( "{url} {branch} has commits not yet fetched; run `oxy git fetch {url}` first" )); } } let report = self.git_export_as(&mirror, branch)?; let spec = format!("refs/heads/{branch}:refs/heads/{branch}"); git(&mirror, &["push", "-q", url, &spec])?; Ok(report) } fn read_oxy_marks(path: &Path) -> Result> { let mut out = BTreeMap::new(); if let Ok(text) = fs::read_to_string(path) { for line in text.lines() { if let Some((h, m)) = line.split_once(' ') { if let (Some(h), Ok(m)) = (ObjectRef::from_hex(h), m.parse()) { out.insert(h, m); } } } } Ok(out) } /// Export the current bookmark to the Git repository at `git_dir`, /// creating a bare one if nothing is there. Each state the bookmark /// passed through (per the operation log) becomes one commit whose tree /// is that state, so the Git branch is a linear history of full trees. pub fn git_export(&self, git_dir: &Path) -> Result { let bookmark = self.config.bookmark.clone(); self.git_export_as(git_dir, &bookmark) } /// `git_export` to the Git branch `branch`. A state that came from a /// Git commit (`oxy git fetch`) is not re-exported: later states chain /// onto that commit, so a push back to the Git remote fast-forwards. pub fn git_export_as(&self, git_dir: &Path, branch: &str) -> Result { if !git_dir.join("HEAD").exists() && !git_dir.join(".git").exists() { fs::create_dir_all(git_dir)?; git(git_dir, &["init", "--bare", "-q"])?; } let marks_dir = self.git_marks_dir(git_dir)?; let git_marks = marks_dir.join("git-marks"); let oxy_marks_path = marks_dir.join("oxy-marks"); let mut marks = Self::read_oxy_marks(&oxy_marks_path)?; let git_states = Self::read_git_states(&marks_dir.join("git-states"))?; let mut next_mark = marks.values().copied().max().unwrap_or(0) + 1; let bookmark = self.config.bookmark.clone(); let mut stream: Vec = Vec::new(); let mut report = GitExportReport { branch: branch.to_string(), heads: self.bookmark_heads()?.len(), ..Default::default() }; // States along the bookmark, oldest first, skipping unchanged and pruned ones. let mut ops = self.op_log()?; ops.reverse(); let mut prev: Option<(ObjectRef, Vec)> = None; // The fast-import parent of the next commit: `:mark` or a commit sha. let mut last_parent: Option = None; for (_, op) in &ops { let Some(state_ref) = op.view.bookmarks.get(&bookmark).copied() else { continue; }; if prev.as_ref().is_some_and(|(p, _)| *p == state_ref) { continue; } let Ok(state) = self.store.get_state(&state_ref) else { continue; }; if state.heads.is_empty() { prev = Some((state_ref, vec![])); continue; } if let Some(sha) = git_states.get(&state_ref) { report.skipped += 1; last_parent = Some(sha.clone()); prev = Some((state_ref, state.heads.clone())); continue; } if let Some(m) = marks.get(&state_ref) { report.skipped += 1; last_parent = Some(format!(":{m}")); prev = Some((state_ref, state.heads.clone())); continue; } let prev_heads = prev.as_ref().map(|(_, h)| h.clone()).unwrap_or_default(); let added = self .negotiate(&prev_heads, &state.heads) .unwrap_or_default(); let loaded = self.load_state(&state.heads)?; let snapshot = loaded.materialize(); let (author, secs, mut message) = match added.as_slice() { [h] => { let c = self.store.get_change(h)?; let mut m = c.message.clone(); if !m.ends_with('\n') { m.push('\n'); } m.push_str(&format!( "\nChange-Id: {}\nOxy-Hash: {}\n", c.change_id.to_hex(), h.to_hex() )); (self.git_author(&c), c.hlc.wall_ms / 1000, m) } [] => ( "oxy ".to_string(), op.hlc.wall_ms / 1000, format!("oxy: {}\n\n", op.command), ), many => { let mut m = format!("oxy: {} ({} changes)\n\n", op.command, many.len()); for h in many { let c = self.store.get_change(h)?; m.push_str(&format!( "- {} {}\n", c.change_id.short(8), c.message.lines().next().unwrap_or("") )); } m.push_str(&format!( "\nOxy-Changes: {}\n", many.iter() .map(|h| h.to_hex()) .collect::>() .join(" ") )); ("oxy ".to_string(), op.hlc.wall_ms / 1000, m) } }; message.push_str(&format!("Oxy-State: {}\n", state_ref.to_hex())); if snapshot.is_conflicted() { message.push_str("Oxy-Conflicted: true\n"); } let mark = next_mark; next_mark += 1; let parents: Vec = last_parent.clone().into_iter().collect(); self.write_commit( &mut stream, "refs/oxy/export", mark, &author, secs, &message, &parents, &snapshot, )?; marks.insert(state_ref, mark); last_parent = Some(format!(":{mark}")); prev = Some((state_ref, state.heads.clone())); report.exported += 1; } if let Some(p) = &last_parent { stream.extend_from_slice(format!("reset refs/heads/{branch}\nfrom {p}\n\n").as_bytes()); } // Tags: a tag whose state was exported as a bookmark state points at // that commit; otherwise its state gets a commit of its own. for t in self.tags()? { let target = match (git_states.get(&t.state), marks.get(&t.state)) { (Some(sha), _) => sha.clone(), (None, Some(m)) => format!(":{m}"), (None, None) => { let loaded = self.load_state(&t.heads)?; let snapshot = loaded.materialize(); let mark = next_mark; next_mark += 1; let message = format!( "oxy: tag {}\n\n{}Oxy-State: {}\n", t.name, if t.message.is_empty() { String::new() } else { format!("{}\n\n", t.message) }, t.state.to_hex() ); self.write_commit( &mut stream, "refs/oxy/export", mark, "oxy ", Hlc::now().wall_ms / 1000, &message, &[], &snapshot, )?; marks.insert(t.state, mark); report.exported += 1; format!(":{mark}") } }; stream.extend_from_slice( format!("reset refs/tags/{}\nfrom {target}\n\n", t.name).as_bytes(), ); } stream.extend_from_slice(b"done\n"); let mut cmd = Command::new("git"); // `--force`: the scratch ref may jump between unrelated states. cmd.arg("-C") .arg(git_dir) .args(["fast-import", "--quiet", "--done", "--force"]) .arg(format!("--export-marks={}", git_marks.display())); if git_marks.exists() { cmd.arg(format!("--import-marks={}", git_marks.display())); } let mut child = cmd .stdin(Stdio::piped()) .stdout(Stdio::null()) .stderr(Stdio::piped()) .spawn() .map_err(|e| RepoError::Msg(format!("running git fast-import: {e}")))?; child.stdin.take().unwrap().write_all(&stream)?; let out = child.wait_with_output()?; if !out.status.success() { return fail(format!( "git fast-import failed: {}", String::from_utf8_lossy(&out.stderr).trim() )); } let text: String = marks .iter() .map(|(h, m)| format!("{} {m}\n", h.to_hex())) .collect(); write_atomic(&oxy_marks_path, text.as_bytes())?; let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]); report.skipped_subrepos = self.git_export_skipped.replace(0); Ok(report) } /// Import branch `branch` of the Git repository at `git_dir` into this /// (empty) repository's bookmark (spec §23): one change per commit, its /// edits the difference between the union of its parents' states and its /// tree, so merge commits become resolutions. Submodules are skipped. pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result { if !self.bookmark_heads()?.is_empty() { return fail("git import needs an empty bookmark; import into a fresh repository"); } self.import_git_commits( git_dir, branch, &format!("oxy git import {} {branch}", git_dir.display()), ) } /// Import the commits of `branch` that the bookmark does not have yet, /// then move the bookmark to the union of its heads and theirs. fn import_git_commits( &self, git_dir: &Path, branch: &str, command: &str, ) -> Result { let batch = self.store.begin_batch()?; let result = self.import_git_commits_batched(git_dir, branch, command); batch.finish()?; result } fn import_git_commits_batched( &self, git_dir: &Path, branch: &str, command: &str, ) -> Result { let commits = list_commits(git_dir, branch)?; let mut cat = CatFile::start(git_dir)?; let mut report = GitImportReport::default(); let mut known = self.git_known(git_dir)?; let old_heads = self.bookmark_heads()?; let mut new_changes: Vec = Vec::new(); // The state after the most recently imported commit, reused when the // next commit's first parent is that commit. let mut last: Option<(String, State)> = None; let mut blob_cache: HashMap> = HashMap::new(); let mut cache_bytes: usize = 0; for gc in &commits { if known.imported.contains_key(&gc.sha) || known.exported.contains_key(&gc.sha) { continue; } // Base: union of every parent's state. let mut base = match (&gc.parents.first(), &last) { (Some(p1), Some((sha, st))) if *p1 == sha => st.clone(), (Some(p1), _) => { let mut st = State::new(); self.add_git_ancestors(&mut st, git_dir, &[p1.as_str()], &known)?; st } (None, _) => State::new(), }; if gc.parents.len() > 1 { report.merges += 1; let mut args: Vec = gc.parents[1..].to_vec(); args.push(format!("^{}", gc.parents[0])); let refs: Vec<&str> = args.iter().map(String::as_str).collect(); self.add_git_ancestors(&mut base, git_dir, &refs, &known)?; } let base_snapshot = base.materialize(); // The commit's tree as in-memory entries. let mut entries: BTreeMap = BTreeMap::new(); let mut gitlinks: Vec<(String, String)> = Vec::new(); for (path, (mode, sha)) in list_tree(git_dir, &gc.sha)? { let (kind, exec) = match mode.as_str() { "100644" | "100664" => (NodeKind::File, false), "100755" => (NodeKind::File, true), "120000" => (NodeKind::Symlink, false), "160000" => { gitlinks.push((path, sha)); continue; } _ => continue, }; let content = match blob_cache.get(&sha) { Some(c) => c.clone(), None => { let c = cat.read(&sha)?; if cache_bytes + c.len() > 256 * 1024 * 1024 { blob_cache.clear(); cache_bytes = 0; } cache_bytes += c.len(); blob_cache.insert(sha.clone(), c.clone()); c } }; entries.insert(path, FsEntry::new(kind, exec, content)); } // Submodules (spec §23): gitlinks become subrepo nodes with Git // pins, and `.gitmodules` also yields `.oxymodules`. if !gitlinks.is_empty() { report.submodules += gitlinks.len(); let gm = entries .get(".gitmodules") .map(|e| { crate::subrepo::Modules::from_gitmodules(&String::from_utf8_lossy( &e.content, )) }) .unwrap_or_default(); let mut modules = entries .get(".oxymodules") .map(|e| crate::subrepo::Modules::parse(&String::from_utf8_lossy(&e.content))) .unwrap_or_default(); for (path, sha) in &gitlinks { let m = gm.entries.get(path); let branch = m .and_then(|m| m.bookmark.clone()) .unwrap_or_else(|| "HEAD".into()); entries.insert( path.clone(), FsEntry::new( NodeKind::Subrepo, false, crate::subrepo::Pin::Git { sha: sha.clone(), branch, } .encode(), ), ); if let Some(m) = m { modules.entries.insert(path.clone(), m.clone()); } } if !modules.entries.is_empty() { entries.insert( ".oxymodules".into(), FsEntry::new(NodeKind::File, false, modules.to_text().into_bytes()), ); } } let mut built = self.build_from_entries(&base, &base_snapshot, HashMap::new(), &entries)?; // A file or submodule with several live blobs (branches that // replaced it independently) is resolved to the commit's blob: // Git's merge is the authority here, unlike a working copy, // where a candidate on disk is not yet a decision (spec §23). for (path, f) in &base_snapshot.files { let (FileContent::Opaque { blobs }, Some(e)) = (&f.content, entries.get(path)) else { continue; }; if blobs.len() < 2 || f.kind != e.kind { continue; } let chosen = self.store_blob(&e.content)?; for b in blobs { built.edits.push(Edit::Replace { node: f.node, old: Some(*b), new: Some(chosen), }); } built.resolves = true; built .contributors .extend(base.live_replaces(&f.node).into_iter().map(|(c, _)| c)); } let mut deps: BTreeSet = built.contributors.iter().copied().collect(); let mut message = gc.message.clone(); if !message.ends_with('\n') { message.push('\n'); } message.push_str(&format!( "\nGit-Commit: {}\nGit-Author: {}\n", gc.sha, gc.author )); let mut change = Change { change_id: change_id_for(&gc.sha, &gc.message), deps: vec![], author: ObjectRef::ZERO, hlc: Hlc { wall_ms: gc.author_secs * 1000, logical: 0, }, message, tree_ops: built.tree_ops, edits: built.edits, kind: if built.resolves { ChangeKind::Resolution } else { ChangeKind::Normal }, }; deps.extend(referenced_changes(&change, &base)); change.deps = deps.into_iter().collect(); change.normalize(); let hash = self.store.put_change(&change)?; new_changes.push(change.clone()); base.add_unchecked(change); known.imported.insert(gc.sha.clone(), hash); last = Some((gc.sha.clone(), base)); report.commits += 1; } // The bookmark: heads of the old state plus every new change. report.heads = old_heads.len(); if last.is_none() { return Ok(report); } let mut all = self.load_state(&old_heads)?; for c in new_changes { all.add_unchecked(c); } let mut heads = all.heads(); heads.sort(); report.heads = heads.len(); let (op_ref, op) = self.head_op()?; let mut view = op.view.clone(); let old_state_ref = view.bookmarks.get(&self.config.bookmark).copied(); let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?; view.bookmarks .insert(self.config.bookmark.clone(), state_ref); self.record_op(&[op_ref], command, view)?; // The new state equals the tip's tree when the old one was itself a // Git commit's tree (fetched or exported), or there was none. let old_known = old_heads.is_empty() || old_state_ref.is_some_and(|s| known.exported.values().any(|e| *e == s)); if let (Some(tip), true) = (commits.last(), old_known) { self.record_git_state(git_dir, &state_ref, &tip.sha)?; } // The working copy follows when it was clean on the old heads. if old_heads.is_empty() { self.checkout(&heads, true, &format!("{command} (checkout)"))?; } else { let snap = self.snapshot(command)?; let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty(); let mut base = snap.base.maximal(&snap.wc.deps); base.sort(); let mut sorted_old = old_heads.clone(); sorted_old.sort(); if clean && base == sorted_old { self.checkout(&heads, false, &format!("{command} (checkout)"))?; } } Ok(report) } /// Add to `st` the changes of every Git commit `rev-list args` lists: /// an imported commit's change, or the whole state an exported commit /// was made from (which may hold changes Git never saw). fn add_git_ancestors( &self, st: &mut State, git_dir: &Path, args: &[&str], known: &GitKnown, ) -> Result<()> { let mut full = vec!["rev-list"]; full.extend(args); let out = git(git_dir, &full)?; for sha in out.lines() { if let Some(h) = known.imported.get(sha) { st.add_unchecked(self.store.get_change(h)?); } else if let Some(state_ref) = known.exported.get(sha) { let heads = self.store.get_state(state_ref)?.heads; for h in self.negotiate(&[], &heads)? { st.add_unchecked(self.store.get_change(&h)?); } } } Ok(()) } /// Commits this repository can relate to Oxygen history: imported /// ones by change, exported ones (through `git_dir`'s marks) by state. fn git_known(&self, git_dir: &Path) -> Result { let imported = self.imported_commits()?; let marks_dir = self.git_marks_dir(git_dir)?; let oxy_marks = Self::read_oxy_marks(&marks_dir.join("oxy-marks"))?; let mut sha_of_mark: HashMap = HashMap::new(); if let Ok(text) = fs::read_to_string(marks_dir.join("git-marks")) { for line in text.lines() { if let Some((m, sha)) = line.split_once(' ') { if let Ok(m) = m.trim_start_matches(':').parse::() { sha_of_mark.insert(m, sha.trim().to_string()); } } } } let mut exported = HashMap::new(); for (state_ref, m) in oxy_marks { if let Some(sha) = sha_of_mark.get(&m) { exported.insert(sha.clone(), state_ref); } } for (state_ref, sha) in Self::read_git_states(&marks_dir.join("git-states"))? { exported.entry(sha).or_insert(state_ref); } Ok(GitKnown { imported, exported }) } fn git_author(&self, c: &Change) -> String { match self.store.get_identity(&c.author) { Ok(id) if !id.name.is_empty() => format!( "{} <{}>", id.name.replace(['<', '>', '\n'], " "), id.email.replace(['<', '>', '\n', ' '], "") ), _ => "oxy ".to_string(), } } #[allow(clippy::too_many_arguments)] fn write_commit( &self, out: &mut Vec, reference: &str, mark: u64, author: &str, secs: u64, message: &str, parents: &[String], snapshot: &Snapshot, ) -> Result<()> { out.extend_from_slice(format!("commit {reference}\nmark :{mark}\nauthor {author} {secs} +0000\ncommitter {author} {secs} +0000\ndata {}\n{message}\n", message.len()).as_bytes()); for (i, p) in parents.iter().enumerate() { out.extend_from_slice( format!("{} {p}\n", if i == 0 { "from" } else { "merge" }).as_bytes(), ); } out.extend_from_slice(b"deleteall\n"); let has_gitmodules = snapshot.files.contains_key(".gitmodules"); for (path, f) in &snapshot.files { if path == ".oxymodules" { // Git reads `.gitmodules`; derive it unless the tree has one. if !has_gitmodules { if let FileContent::Text { bytes, .. } = &f.content { let gm = crate::subrepo::Modules::parse(&String::from_utf8_lossy(bytes)) .to_gitmodules(); if !gm.is_empty() { out.extend_from_slice( format!("M 100644 inline .gitmodules\ndata {}\n", gm.len()) .as_bytes(), ); out.extend_from_slice(gm.as_bytes()); out.push(b'\n'); } } } continue; } if f.kind == NodeKind::Subrepo { if let FileContent::Opaque { blobs } = &f.content { if let Some(b) = blobs.first() { match crate::subrepo::Pin::decode(&self.read_blob(b)?)? { crate::subrepo::Pin::Git { sha, .. } => { out.extend_from_slice(format!("M 160000 {sha} {path}\n").as_bytes()) } crate::subrepo::Pin::Oxy { .. } => self .git_export_skipped .set(self.git_export_skipped.get() + 1), } } } continue; } let (mode, bytes): (&str, Vec) = match (&f.kind, &f.content) { (NodeKind::Dir, _) => continue, (NodeKind::Symlink, FileContent::Opaque { blobs }) => ( "120000", blobs .first() .map(|b| self.read_blob(b)) .transpose()? .unwrap_or_default(), ), (_, FileContent::Text { bytes, .. }) => { (if f.mode.exec { "100755" } else { "100644" }, bytes.clone()) } (_, FileContent::Opaque { blobs }) => ( if f.mode.exec { "100755" } else { "100644" }, blobs .first() .map(|b| self.read_blob(b)) .transpose()? .unwrap_or_default(), ), (_, FileContent::Dir) => continue, }; out.extend_from_slice( format!("M {mode} inline {path}\ndata {}\n", bytes.len()).as_bytes(), ); out.extend_from_slice(&bytes); out.push(b'\n'); } out.push(b'\n'); Ok(()) } } /// What `verify_git_import` found (spec §23: import must reproduce every tree). #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct GitVerifyReport { pub commits: usize, pub files_compared: usize, /// `(commit, what differs)`; empty when every commit's tree is reproduced. pub mismatches: Vec<(String, String)>, } impl Repo { /// Check an import against its Git source: for every commit of /// `branch`, the state formed by the changes imported for it and all /// its ancestors must materialize to exactly that commit's tree, byte /// for byte, with the same executable bits, symlink targets and /// submodule pins. Reads only; stops listing after 50 differences. pub fn verify_git_import(&self, git_dir: &Path, branch: &str) -> Result { let commits = list_commits(git_dir, branch)?; let imported = self.imported_commits()?; let mut cat = CatFile::start(git_dir)?; let mut rep = GitVerifyReport::default(); // Changes of each commit and all its ancestors, built up in topological order. let mut closure: HashMap>> = HashMap::new(); for c in &commits { rep.commits += 1; let mut set: BTreeSet = BTreeSet::new(); for p in &c.parents { if let Some(ps) = closure.get(p) { set.extend(ps.iter().copied()); } } match imported.get(&c.sha) { Some(h) => { set.insert(*h); } None => { rep.mismatches.push(( c.sha.clone(), "no imported change carries this commit".into(), )); } } let set = Arc::new(set); closure.insert(c.sha.clone(), set.clone()); if rep.mismatches.len() >= 50 { break; } let heads: Vec = set.iter().copied().collect(); let state = self.load_state(&heads)?; let snap = state.materialize(); let tree = list_tree(git_dir, &c.sha)?; let short = &c.sha[..12.min(c.sha.len())]; let mut differ = |what: String| rep.mismatches.push((c.sha.clone(), what)); if snap.is_conflicted() { // Name the changes behind each conflicted file, by their Git commits. let mut detail = Vec::new(); for (path, f) in &snap.files { if !f.is_conflicted() { continue; } let who: Vec = state .conflict_regions(&f.node) .contributors .iter() .filter_map(|h| state.get(h)) .map(|m| { let sha = m .message .lines() .rev() .find_map(|l| l.strip_prefix("Git-Commit: ")) .unwrap_or("?"); format!( "{} {:?}", &sha[..12.min(sha.len())], m.message.lines().next().unwrap_or("") ) }) .collect(); detail.push(format!("{path} between {}", who.join(" and "))); } differ(format!( "{short}: the state is conflicted: {}", detail.join("; ") )); } for (path, (mode, sha)) in &tree { let Some(f) = snap.files.get(path) else { differ(format!("{short}: {path} is missing")); continue; }; rep.files_compared += 1; match mode.as_str() { "160000" => match (&f.kind, &f.content) { (NodeKind::Subrepo, FileContent::Opaque { blobs }) if blobs.len() == 1 => { let pin = crate::subrepo::Pin::decode(&self.read_blob(&blobs[0])?)?; if !matches!(&pin, crate::subrepo::Pin::Git { sha: s, .. } if s == sha) { differ(format!("{short}: {path} pins another commit")); } } (kind, content) => differ(format!( "{short}: {path} is not one submodule pin: {kind:?} with {}", match content { FileContent::Opaque { blobs } => format!("{} pins", blobs.len()), FileContent::Text { .. } => "text".into(), FileContent::Dir => "a directory".into(), } )), }, "120000" => { let target = cat.read(sha)?; match (&f.kind, &f.content) { (NodeKind::Symlink, FileContent::Opaque { blobs }) if blobs.len() == 1 && self.read_blob(&blobs[0])? == target => {} _ => differ(format!("{short}: symlink {path} differs")), } } _ => { let want = cat.read(sha)?; let have = match &f.content { FileContent::Text { bytes, .. } => bytes.clone(), FileContent::Opaque { blobs } if blobs.len() == 1 => { self.read_blob(&blobs[0])? } _ => { differ(format!("{short}: {path} is not one file")); continue; } }; if have != want { differ(format!( "{short}: {path} differs: {}", first_difference(&have, &want) )); } if f.mode.exec != (mode == "100755") { differ(format!("{short}: {path} executable bit differs")); } } } } // Import records submodules in `.oxymodules` as well (spec §23), // and export leaves it out again. let has_submodules = tree.values().any(|(m, _)| m == "160000"); for (path, f) in &snap.files { if f.kind != NodeKind::Dir && !tree.contains_key(path) && !(has_submodules && path == ".oxymodules") { differ(format!("{short}: {path} is extra")); } } if rep.mismatches.len() >= 50 { break; } } Ok(rep) } } /// Where two versions of a file first part, for a verification report. fn first_difference(have: &[u8], want: &[u8]) -> String { let (h, w): (Vec<&[u8]>, Vec<&[u8]>) = ( have.split(|b| *b == b'\n').collect(), want.split(|b| *b == b'\n').collect(), ); let i = h .iter() .zip(&w) .position(|(a, b)| a != b) .unwrap_or(h.len().min(w.len())); let show = |l: Option<&&[u8]>| { l.map(|l| String::from_utf8_lossy(&l[..l.len().min(60)]).into_owned()) .unwrap_or_else(|| "(end)".into()) }; format!( "line {}: has {:?}, Git has {:?} ({} lines, Git has {})", i + 1, show(h.get(i)), show(w.get(i)), h.len(), w.len() ) }