Git bridge, import side: oxy git import with merge commits as resolutions, Change-Id trailers, batched blob reads c3069537d7c5

pub struct GitImportReport {

    pub commits: usize,

    pub merges: usize,

    pub skipped_submodules: usize,

    pub heads: usize,

}

/// A running `git cat-file --batch`, for reading many blobs cheaply.

struct CatFile {

    child: std::process::Child,

    stdin: std::process::ChildStdin,

    stdout: std::io::BufReader<std::process::ChildStdout>,

}

impl CatFile {

    fn start(git_dir: &Path) -> Result<CatFile> {

        let mut child = Command::new("git")

            .arg("-C")

            .arg(git_dir)

            .args(["cat-file", "--batch"])

            .stdin(Stdio::piped())

            .stdout(Stdio::piped())

            .stderr(Stdio::null())

            .spawn()

            .map_err(|e| RepoError::Msg(format!("running git cat-file: {e}")))?;

        let stdin = child.stdin.take().unwrap();

        let stdout = std::io::BufReader::new(child.stdout.take().unwrap());

        Ok(CatFile { child, stdin, stdout })

    }

    fn read(&mut self, sha: &str) -> Result<Vec<u8>> {

        use std::io::{BufRead, Read};

        writeln!(self.stdin, "{sha}")?;

        self.stdin.flush()?;

        let mut header = String::new();

        self.stdout.read_line(&mut header)?;

        let parts: Vec<&str> = header.trim_end().split(' ').collect();

        if parts.len() != 3 {

            return fail(format!("git cat-file: object {sha} not found"));

        }

        let size: usize = parts[2].parse().map_err(|_| RepoError::Msg("git cat-file: bad size".into()))?;

        let mut body = vec![0u8; size];

        self.stdout.read_exact(&mut body)?;

        let mut nl = [0u8; 1];

        self.stdout.read_exact(&mut nl)?;

        Ok(body)

    }

}

impl Drop for CatFile {

    fn drop(&mut self) {

        let _ = self.child.kill();

        let _ = self.child.wait();

    }

}

/// One commit as `git log` describes it.

struct GitCommit {

    sha: String,

    parents: Vec<String>,

    author_secs: u64,

    author: String,

    message: String,

}

fn list_commits(git_dir: &Path, branch: &str) -> Result<Vec<GitCommit>> {

    let out = Command::new("git")

        .arg("-C")

        .arg(git_dir)

        .args(["log", "--reverse", "--topo-order", "--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e", branch])

        .output()

        .map_err(|e| RepoError::Msg(format!("running git log: {e}")))?;

    if !out.status.success() {

        return fail(format!("git log {branch} failed: {}", String::from_utf8_lossy(&out.stderr).trim()));

    }

    let text = String::from_utf8_lossy(&out.stdout);

    let mut commits = Vec::new();

    for record in text.split('\x1e') {

        let record = record.trim_start_matches('\n');

        if record.trim().is_empty() {

            continue;

        }

        let f: Vec<&str> = record.splitn(5, '\0').collect();

        if f.len() != 5 {

            return fail("unexpected git log record");

        }

        commits.push(GitCommit {

            sha: f[0].to_string(),

            parents: f[1].split_whitespace().map(str::to_string).collect(),

            author_secs: f[2].parse().unwrap_or(0),

            author: f[3].to_string(),

            message: f[4].trim_end().to_string(),

        });

    }

    Ok(commits)

}

/// `path → (mode, blob sha)` for a commit's tree.

fn list_tree(git_dir: &Path, sha: &str) -> Result<BTreeMap<String, (String, String)>> {

    let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?;

    let mut out = BTreeMap::new();

    for entry in text.split('\0') {

        if entry.is_empty() {

            continue;

        }

        let Some((meta, path)) = entry.split_once('\t') else { continue };

        let m: Vec<&str> = meta.split(' ').collect();

        if m.len() == 3 {

            out.insert(path.to_string(), (m[0].to_string(), m[2].to_string()));

        }

    }

    Ok(out)

}

/// The ChangeId for a Git commit: its `Change-Id` trailer if it has a

/// 32-hex one, else BLAKE3-128 of the commit hash (spec §23).

fn change_id_for(sha: &str, message: &str) -> ChangeId {

    for line in message.lines().rev() {

        if let Some(v) = line.strip_prefix("Change-Id:") {

            if let Some(id) = ChangeId::from_hex(v.trim()) {

                return id;

            }

        }

    }

    let h = blake3::hash(sha.as_bytes());

    ChangeId(h.as_bytes()[..16].try_into().unwrap())

}

#[derive(Debug, Clone, Default, PartialEq, Eq)]

    /// Import branch `branch` of the Git repository at `git_dir` into this

    /// (empty) repository's bookmark (spec §23): one change per commit, its

    /// edits the difference between the union of its parents' states and its

    /// tree, so merge commits become resolutions. Submodules are skipped.

    pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result<GitImportReport> {

        if !self.bookmark_heads()?.is_empty() {

            return fail("git import needs an empty bookmark; import into a fresh repository");

        }

        let commits = list_commits(git_dir, branch)?;

        let mut cat = CatFile::start(git_dir)?;

        let mut report = GitImportReport::default();

        let mut change_of: HashMap<String, ChangeHash> = HashMap::new();

        // The state after the most recently imported commit, reused when the

        // next commit's first parent is that commit.

        let mut last: Option<(String, State)> = None;

        let mut blob_cache: HashMap<String, Vec<u8>> = HashMap::new();

        let mut cache_bytes: usize = 0;

        for gc in &commits {

            // Base: union of every parent's state.

            let mut base = match (&gc.parents.first(), &last) {

                (Some(p1), Some((sha, st))) if *p1 == sha => st.clone(),

                (Some(p1), _) => self.git_ancestor_state(git_dir, &[p1.as_str()], &change_of)?,

                (None, _) => State::new(),

            };

            if gc.parents.len() > 1 {

                report.merges += 1;

                let mut args: Vec<String> = gc.parents[1..].to_vec();

                args.push(format!("^{}", gc.parents[0]));

                let extra = git(git_dir, &["rev-list".to_string()].iter().map(String::as_str).chain(args.iter().map(String::as_str)).collect::<Vec<_>>())?;

                for sha in extra.lines() {

                    if let Some(h) = change_of.get(sha) {

                        base.add_unchecked(self.store.get_change(h)?);

                    }

                }

            }

            let base_snapshot = base.materialize();

            // The commit's tree as in-memory entries.

            let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();

            for (path, (mode, sha)) in list_tree(git_dir, &gc.sha)? {

                let (kind, exec) = match mode.as_str() {

                    "100644" | "100664" => (NodeKind::File, false),

                    "100755" => (NodeKind::File, true),

                    "120000" => (NodeKind::Symlink, false),

                    "160000" => {

                        report.skipped_submodules += 1;

                        continue;

                    }

                    _ => continue,

                };

                let content = match blob_cache.get(&sha) {

                    Some(c) => c.clone(),

                    None => {

                        let c = cat.read(&sha)?;

                        if cache_bytes + c.len() > 256 * 1024 * 1024 {

                            blob_cache.clear();

                            cache_bytes = 0;

                        }

                        cache_bytes += c.len();

                        blob_cache.insert(sha.clone(), c.clone());

                        c

                    }

                };

                entries.insert(path, FsEntry { kind, exec, content });

            }

            let built = self.build_from_entries(&base, &base_snapshot, HashMap::new(), &entries)?;

            let mut deps: BTreeSet<ChangeHash> = built.contributors.iter().copied().collect();

            let mut message = gc.message.clone();

            if !message.ends_with('\n') {

                message.push('\n');

            }

            message.push_str(&format!("\nGit-Commit: {}\nGit-Author: {}\n", gc.sha, gc.author));

            let mut change = Change {

                change_id: change_id_for(&gc.sha, &gc.message),

                deps: vec![],

                author: ObjectRef::ZERO,

                hlc: Hlc { wall_ms: gc.author_secs * 1000, logical: 0 },

                message,

                tree_ops: built.tree_ops,

                edits: built.edits,

                kind: if built.resolves { ChangeKind::Resolution } else { ChangeKind::Normal },

            };

            deps.extend(referenced_changes(&change, &base));

            change.deps = deps.into_iter().collect();

            change.normalize();

            let hash = self.store.put_change(&change)?;

            base.add_unchecked(change);

            change_of.insert(gc.sha.clone(), hash);

            last = Some((gc.sha.clone(), base));

            report.commits += 1;

        }

        // The bookmark: heads of the tip's state; the working copy follows.

        let Some((_, tip_state)) = last else { return Ok(report) };

        let heads = tip_state.heads();

        report.heads = heads.len();

        let (op_ref, op) = self.head_op()?;

        let mut view = op.view.clone();

        view.bookmarks.insert(self.config.bookmark.clone(), self.store.put_state(&StateObject::new(heads.clone()))?);

        self.record_op(&[op_ref], &format!("oxy git import {} {branch}", git_dir.display()), view)?;

        self.checkout(&heads, true, "oxy git import (checkout)")?;

        Ok(report)

    }

    /// The state holding the changes of every Git ancestor of `shas`.

    fn git_ancestor_state(&self, git_dir: &Path, shas: &[&str], change_of: &HashMap<String, ChangeHash>) -> Result<State> {

        let mut args = vec!["rev-list"];

        args.extend(shas);

        let out = git(git_dir, &args)?;

        let mut st = State::new();

        for sha in out.lines() {

            if let Some(h) = change_of.get(sha) {

                st.add_unchecked(self.store.get_change(h)?);

            }

        }

        Ok(st)

    }

pub use git::{GitExportReport, GitImportReport};

/// A file as found on disk or in an imported tree.

pub(crate) struct FsEntry {

    pub kind: NodeKind,

    pub exec: bool,

    /// File content, or a symlink's target.

    pub content: Vec<u8>,

        self.build_from_entries(base, base_snapshot, reuse, &fs_entries)

    }

    /// Like `build_against`, for files given in memory (imports).

    pub(crate) fn build_from_entries(&self, base: &State, base_snapshot: &Snapshot, reuse: HashMap<String, (NodeId, NodeKind)>, fs_entries: &BTreeMap<String, FsEntry>) -> Result<BuiltChange> {

        for (path, fs_entry) in fs_entries {

#[test]

fn git_import_reproduces_trees() {

    hermetic();

    if std::process::Command::new("git").arg("--version").output().is_err() {

        eprintln!("git not installed; skipping");

        return;

    }

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let g = root.join("src.git");

    fs::create_dir_all(&g).unwrap();

    let git = |args: &[&str]| -> String {

        let out = std::process::Command::new("git")

            .arg("-C")

            .arg(&g)

            .args(["-c", "user.name=Grace", "-c", "[email protected]", "-c", "commit.gpgsign=false"])

            .args(args)

            .output()

            .unwrap();

        assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr));

        String::from_utf8_lossy(&out.stdout).trim().to_string()

    };

    git(&["init", "-q", "-b", "main"]);

    write(&g, "README.md", b"# Hi\n");

    write(&g, "src/lib.rs", b"pub fn a() {}\n");

    git(&["add", "."]);

    git(&["commit", "-q", "-m", "initial"]);

    write(&g, "src/lib.rs", b"pub fn a() {}\npub fn b() {}\n");

    write(&g, "run.sh", b"#!/bin/sh\n");

    #[cfg(unix)]

    {

        use std::os::unix::fs::PermissionsExt;

        fs::set_permissions(g.join("run.sh"), fs::Permissions::from_mode(0o755)).unwrap();

    }

    write(&g, "logo.png", b"\x89PNG\0\0\x01");

    git(&["add", "."]);

    git(&["commit", "-q", "-m", "second\n\nChange-Id: 0123456789abcdef0123456789abcdef"]);

    // A branch that edits README, merged back with a conflict-free merge.

    git(&["checkout", "-q", "-b", "topic", "HEAD~1"]);

    write(&g, "README.md", b"# Hi\n\nMore.\n");

    git(&["commit", "-q", "-am", "readme"]);

    git(&["checkout", "-q", "main"]);

    git(&["merge", "-q", "--no-ff", "-m", "merge topic", "topic"]);

    // A branch from the initial commit inserting at the same place as

    // "second": concurrent in Oxygen too, so the merge commit is a resolution.

    let initial = git(&["rev-list", "--max-parents=0", "HEAD"]);

    git(&["checkout", "-q", "-b", "clash", &initial]);

    write(&g, "src/lib.rs", b"pub fn a() {}\npub fn c() {}\n");

    git(&["commit", "-q", "-am", "clash"]);

    git(&["checkout", "-q", "main"]);

    let _ = std::process::Command::new("git").arg("-C").arg(&g).args(["merge", "-q", "clash"]).output();

    write(&g, "src/lib.rs", b"pub fn a() {}\npub fn b() {}\npub fn c() {}\n");

    git(&["add", "."]);

    git(&["commit", "-q", "-m", "resolve clash"]);

    git(&["rm", "-q", "logo.png"]);

    git(&["commit", "-q", "-m", "drop logo"]);

    let expected_count: usize = git(&["rev-list", "--count", "main"]).parse().unwrap();

    let dest = root.join("oxy");

    let repo = Repo::init(&dest).unwrap().with_home(root.join("home"));

    let rep = repo.import_git(&g, "main").unwrap();

    assert_eq!(rep.commits, expected_count);

    assert_eq!(rep.merges, 2);

    // The working copy equals the Git tree at main.

    assert_eq!(fs::read(dest.join("src/lib.rs")).unwrap(), b"pub fn a() {}\npub fn b() {}\npub fn c() {}\n");

    assert_eq!(fs::read(dest.join("README.md")).unwrap(), b"# Hi\n\nMore.\n");

    assert!(!dest.join("logo.png").exists());

    #[cfg(unix)]

    {

        use std::os::unix::fs::PermissionsExt;

        assert!(fs::metadata(dest.join("run.sh")).unwrap().permissions().mode() & 0o111 != 0);

    }

    assert!(!repo.snapshot("status").unwrap().changed, "clean after import");

    let heads = repo.bookmark_heads().unwrap();

    let st = repo.load_state(&heads).unwrap();

    assert!(!st.materialize().is_conflicted());

    assert_eq!(st.len(), expected_count);

    // Messages carry Git trailers; the Change-Id trailer was honoured.

    let log = repo.log().unwrap();

    let second = log.iter().find(|e| e.change.message.starts_with("second")).unwrap();

    assert_eq!(second.change.change_id, ChangeId::from_hex("0123456789abcdef0123456789abcdef").unwrap());

    assert!(second.change.message.contains("Git-Author: Grace <[email protected]>"));

    let resolve = log.iter().find(|e| e.change.message.starts_with("resolve clash")).unwrap();

    assert_eq!(resolve.change.kind, ChangeKind::Resolution, "a conflicting Git merge becomes a resolution");

    // Re-importing into another fresh repository gives the same ChangeIds.

    let repo2 = Repo::init(&root.join("oxy2")).unwrap().with_home(root.join("home"));

    repo2.import_git(&g, "main").unwrap();

    let ids: BTreeSet<ChangeId> = repo.log().unwrap().iter().filter(|e| !e.is_working_copy).map(|e| e.change.change_id).collect();

    let ids2: BTreeSet<ChangeId> = repo2.log().unwrap().iter().filter(|e| !e.is_working_copy).map(|e| e.change.change_id).collect();

    assert_eq!(ids, ids2);

    // Round trip back out to Git reproduces the tip tree.

    let mirror = root.join("mirror.git");

    repo.git_export(&mirror).unwrap();

    let out = std::process::Command::new("git").arg("-C").arg(&mirror).args(["show", "main:src/lib.rs"]).output().unwrap();

    assert_eq!(String::from_utf8_lossy(&out.stdout), "pub fn a() {}\npub fn b() {}\npub fn c() {}\n");

}

#[derive(Default, Clone)]

    /// Import a Git branch into a new Oxygen repository at DEST

    Import {

        git_dir: PathBuf,

        dest: PathBuf,

        #[arg(long, default_value = "main")]

        branch: String,

    },

        Cmd::Git { cmd: GitCmd::Import { git_dir, dest, branch } } => sync::git_import(&git_dir, &dest, &branch),

pub fn git_import(git_dir: &std::path::Path, dest: &std::path::Path, branch: &str) -> Result<i32> {

    let repo = Repo::init(dest)?;

    let started = std::time::Instant::now();

    let r = repo.import_git(git_dir, branch)?;

    println!(

        "Imported {} ({} merges) from {} {branch} into {} in {:.1?}: {} now has {}.{}",

        plural(r.commits, "commit"),

        r.merges,

        git_dir.display(),

        repo.root().display(),

        started.elapsed(),

        repo.config.bookmark,

        plural(r.heads, "head"),

        if r.skipped_submodules > 0 { format!(" Skipped {} submodule entries.", r.skipped_submodules) } else { String::new() }

    );

    Ok(0)

}