OXYGENHUB

Coverage / crates/oxy-core/src/repo/git.rs 79.7%

1352 of 1696 lines run, 93 of 278 functions · run #256 on main · run missed

1
//! Git bridge (spec §23), export side: each state the current bookmark
2
//! passed through becomes a Git commit through `git fast-import`, with the
3
//! state's full tree and the previous state's commit as parent. A state
4
//! that added exactly one change carries that change's message, author,
5
//! and `Change-Id`/`Oxy-Hash` trailers; every commit carries `Oxy-State`.
6
//!
7
//! Marks make re-exports incremental: `.oxy/git/<target>/git-marks` is
8
//! git's own marks file and `oxy-marks` maps state refs to marks.
9
10
use super::*;
11
use std::process::{Command, Stdio};
12
use std::sync::Arc;
13
14
#[derive(Debug, Clone, Default, PartialEq, Eq)]
15
pub struct GitImportReport {
16
    pub commits: usize,
17
    pub merges: usize,
18
    /// Gitlink entries imported as subrepo nodes with Git pins.
19
    pub submodules: usize,
20
    pub heads: usize,
21
}
22
23
/// A running `git cat-file --batch`, for reading many blobs cheaply.
24
struct CatFile {
25
    child: std::process::Child,
26
    stdin: std::process::ChildStdin,
27
    stdout: std::io::BufReader<std::process::ChildStdout>,
28
}
29
30
impl CatFile {
3122×
    fn start(git_dir: &Path) -> Result<CatFile> {
3222×
        let mut child = Command::new("git")
3322×
            .arg("-C")
3422×
            .arg(git_dir)
3522×
            .args(["cat-file", "--batch"])
3622×
            .stdin(Stdio::piped())
3722×
            .stdout(Stdio::piped())
3822×
            .stderr(Stdio::null())
3922×
            .spawn()
4022×
            .map_err(|e| RepoError::Msg(format!("running git cat-file: {e}")))?;
4122×
        let stdin = child.stdin.take().unwrap();
4222×
        let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
4322×
        Ok(CatFile {
4422×
            child,
4522×
            stdin,
4622×
            stdout,
4722×
        })
4822×
    }
49
50123×
    fn read(&mut self, sha: &str) -> Result<Vec<u8>> {
51
        use std::io::{BufRead, Read};
52123×
        writeln!(self.stdin, "{sha}")?;
53123×
        self.stdin.flush()?;
54123×
        let mut header = String::new();
55123×
        self.stdout.read_line(&mut header)?;
56123×
        let parts: Vec<&str> = header.trim_end().split(' ').collect();
57123×
        if parts.len() != 3 {
580×
            return fail(format!("git cat-file: object {sha} not found"));
59123×
        }
60123×
        let size: usize = parts[2]
61123×
            .parse()
62123×
            .map_err(|_| RepoError::Msg("git cat-file: bad size".into()))?;
63123×
        let mut body = vec![0u8; size];
64123×
        self.stdout.read_exact(&mut body)?;
65123×
        let mut nl = [0u8; 1];
66123×
        self.stdout.read_exact(&mut nl)?;
67123×
        Ok(body)
68123×
    }
69
}
70
71
impl Drop for CatFile {
7222×
    fn drop(&mut self) {
7322×
        let _ = self.child.kill();
7422×
        let _ = self.child.wait();
7522×
    }
76
}
77
78
/// One path a commit changes against its first parent: its new mode and
79
/// blob, or `None` when the commit deletes it.
80
type TreeDiff = Vec<(String, Option<(String, String)>)>;
81
82
/// A running `git diff-tree --stdin`, fed every `commit parent` pair up
83
/// front, whose diffs are read in the same order as they are needed: a
84
/// commit's tree is its first parent's plus its diff, at a cost that
85
/// follows what it changes rather than the whole tree (oxygen#25).
86
struct DiffTree {
87
    child: std::process::Child,
88
    stdout: std::io::BufReader<std::process::ChildStdout>,
89
    feeder: Option<std::thread::JoinHandle<()>>,
90
    /// The header read at the end of the previous diff.
91
    next: Option<String>,
92
}
93
94
impl DiffTree {
9514×
    fn start(git_dir: &Path, pairs: Vec<(String, String)>) -> Result<DiffTree> {
9614×
        let mut child = Command::new("git")
9714×
            .arg("-C")
9814×
            .arg(git_dir)
9914×
            .args([
10014×
                "diff-tree",
10114×
                "--stdin",
10214×
                "--always",
10314×
                "-r",
10414×
                "-z",
10514×
                "--no-renames",
10614×
            ])
10714×
            .stdin(Stdio::piped())
10814×
            .stdout(Stdio::piped())
10914×
            .stderr(Stdio::null())
11014×
            .spawn()
11114×
            .map_err(|e| RepoError::Msg(format!("running git diff-tree: {e}")))?;
11214×
        let mut stdin = child.stdin.take().unwrap();
11314×
        let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
114
        // A thread writes, so git never waits on a full pipe for us to read.
11514×
        let feeder = std::thread::spawn(move || {
116
            // An import that stops before reading every diff kills git
117
            // while this thread may still be writing. oxy restores
118
            // SIGPIPE's default (to end quietly under `| head`), so that
119
            // write would kill the whole process (signal 13, oxygen#81);
120
            // blocked here, it fails with EPIPE and the thread ends.
121
            #[cfg(unix)]
12214×
            unsafe {
12314×
                let mut set: libc::sigset_t = std::mem::zeroed();
12414×
                libc::sigemptyset(&mut set);
12514×
                libc::sigaddset(&mut set, libc::SIGPIPE);
12614×
                libc::pthread_sigmask(libc::SIG_BLOCK, &set, std::ptr::null_mut());
12714×
            }
12814×
            let mut w = std::io::BufWriter::new(&mut stdin);
12966×
            for (c, p) in pairs {
13066×
                if writeln!(w, "{c} {p}").is_err() {
1310×
                    return;
13266×
                }
133
            }
13414×
            let _ = w.flush();
13514×
        });
13614×
        Ok(DiffTree {
13714×
            child,
13814×
            stdout,
13914×
            feeder: Some(feeder),
14014×
            next: None,
14114×
        })
14214×
    }
143
144243×
    fn field(&mut self) -> Result<Option<String>> {
145
        use std::io::BufRead;
146243×
        let mut buf = Vec::new();
147243×
        if self.stdout.read_until(0, &mut buf)? == 0 {
14813×
            return Ok(None);
149230×
        }
150230×
        if buf.last() == Some(&0) {
151230×
            buf.pop();
152230×
        }
153230×
        Ok(Some(String::from_utf8_lossy(&buf).into_owned()))
154243×
    }
155
156
    /// The diff of `sha` against the parent it was fed with.
15766×
    fn diff(&mut self, sha: &str) -> Result<TreeDiff> {
15866×
        let header = match self.next.take() {
15953×
            Some(h) => Some(h),
16013×
            None => self.field()?,
161
        };
16266×
        if header.as_deref() != Some(sha) {
1630×
            return fail(format!("git diff-tree: expected {sha}, got {header:?}"));
16466×
        }
16566×
        let mut out = Vec::new();
166148×
        while let Some(f) = self.field()? {
167135×
            let Some(meta) = f.strip_prefix(':') else {
16853×
                self.next = Some(f);
16953×
                break;
170
            };
17182×
            let Some(path) = self.field()? else {
1720×
                return fail("git diff-tree: record without a path");
173
            };
174
            // `oldmode newmode oldsha newsha status`
17582×
            let m: Vec<&str> = meta.split(' ').collect();
17682×
            if m.len() != 5 {
1770×
                return fail(format!("git diff-tree: bad record {f:?}"));
17882×
            }
17982×
            let entry = if m[4].starts_with('D') {
18014×
                None
181
            } else {
18268×
                Some((m[1].to_string(), m[3].to_string()))
183
            };
18482×
            out.push((path, entry));
185
        }
18666×
        Ok(out)
18766×
    }
188
}
189
190
impl Drop for DiffTree {
19114×
    fn drop(&mut self) {
19214×
        let _ = self.child.kill();
19314×
        let _ = self.child.wait();
19414×
        if let Some(f) = self.feeder.take() {
19514×
            let _ = f.join();
19614×
        }
19714×
    }
198
}
199
200
/// One commit as `git log` describes it.
201
struct GitCommit {
202
    sha: String,
203
    parents: Vec<String>,
204
    author_secs: u64,
205
    author: String,
206
    message: String,
207
}
208
20922×
fn list_commits(git_dir: &Path, branch: &str) -> Result<Vec<GitCommit>> {
21022×
    let out = Command::new("git")
21122×
        .arg("-C")
21222×
        .arg(git_dir)
21322×
        .args([
21422×
            "log",
21522×
            "--reverse",
21622×
            "--topo-order",
21722×
            "--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e",
21822×
            branch,
21922×
        ])
22022×
        .output()
22122×
        .map_err(|e| RepoError::Msg(format!("running git log: {e}")))?;
22222×
    if !out.status.success() {
2230×
        return fail(format!(
224
            "git log {branch} failed: {}",
2250×
            String::from_utf8_lossy(&out.stderr).trim()
226
        ));
22722×
    }
22822×
    let text = String::from_utf8_lossy(&out.stdout);
22922×
    let mut commits = Vec::new();
230162×
    for record in text.split('\x1e') {
231162×
        let record = record.trim_start_matches('\n');
232162×
        if record.trim().is_empty() {
23322×
            continue;
234140×
        }
235140×
        let f: Vec<&str> = record.splitn(5, '\0').collect();
236140×
        if f.len() != 5 {
2370×
            return fail("unexpected git log record");
238140×
        }
239140×
        commits.push(GitCommit {
240140×
            sha: f[0].to_string(),
241140×
            parents: f[1].split_whitespace().map(str::to_string).collect(),
242140×
            author_secs: f[2].parse().unwrap_or(0),
243140×
            author: f[3].to_string(),
244140×
            message: f[4].trim_end().to_string(),
245140×
        });
246
    }
24722×
    Ok(commits)
24822×
}
249
250
/// `path → (mode, blob sha)` for a commit's tree.
25170×
fn list_tree(git_dir: &Path, sha: &str) -> Result<BTreeMap<String, (String, String)>> {
25270×
    let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?;
25370×
    let mut out = BTreeMap::new();
254229×
    for entry in text.split('\0') {
255229×
        if entry.is_empty() {
25670×
            continue;
257159×
        }
258159×
        let Some((meta, path)) = entry.split_once('\t') else {
2590×
            continue;
260
        };
261159×
        let m: Vec<&str> = meta.split(' ').collect();
262159×
        if m.len() == 3 {
263159×
            out.insert(path.to_string(), (m[0].to_string(), m[2].to_string()));
264159×
        }
265
    }
26670×
    Ok(out)
26770×
}
268
269
/// The ChangeId for a Git commit: its `Change-Id` trailer if it has a
270
/// 32-hex one, else BLAKE3-128 of the commit hash (spec §23).
27180×
fn change_id_for(sha: &str, message: &str) -> ChangeId {
27280×
    for line in message.lines().rev() {
27380×
        if let Some(v) = line.strip_prefix("Change-Id:") {
2744×
            if let Some(id) = ChangeId::from_hex(v.trim()) {
2754×
                return id;
2760×
            }
27776×
        }
278
    }
27976×
    let h = blake3::hash(sha.as_bytes());
28076×
    ChangeId(h.as_bytes()[..16].try_into().unwrap())
28180×
}
282
283
/// Git commits with a known Oxygen counterpart.
284
struct GitKnown {
285
    /// Imported commit → its change.
286
    imported: HashMap<String, ChangeHash>,
287
    /// Exported commit (or a fetched tip) → the bookmark state it equals.
288
    exported: HashMap<String, ObjectRef>,
289
}
290
291
#[derive(Debug, Clone, Default, PartialEq, Eq)]
292
pub struct GitExportReport {
293
    pub exported: usize,
294
    pub skipped: usize,
295
    /// Subrepo nodes with Oxygen pins, which Git cannot represent.
296
    pub skipped_subrepos: usize,
297
    pub branch: String,
298
    pub heads: usize,
299
}
300
301
/// A merge bringing more commits than this sheds its base's graphs first.
302
const MERGE_SHED_COMMITS: usize = 100;
303
304
/// The atoms a kept state's built graphs may hold before they go back into
305
/// its history (#25): about a few hundred megabytes.
306
const KEPT_ATOMS: usize = 1_500_000;
307
/// A kept state with more built graphs than this sheds them.
308
const KEPT_GRAPHS: usize = 2000;
309
/// An import moves its bookmark to the commits done so far every this
310
/// many commits, so one that stops part way resumes from there: imported
311
/// commits are found from the bookmark's history (`imported_commits`).
312
const CHECKPOINT_COMMITS: usize = 2000;
313
314
/// Text files of a merge rendered this many at a time (oxygen#25): a
315
/// batch's graphs are built beside the state and dropped once rendered,
316
/// so what is held for the whole tree is the renderings, not the graphs
317
/// with every deleted line of their history.
318
const MERGE_BATCH: usize = 500;
319
320
thread_local! {
321
    /// A smaller batch for this thread's imports: tests set 1, so a small
322
    /// merge takes the batched path.
323
    static MERGE_BATCH_OVERRIDE: std::cell::Cell<Option<usize>> = const { std::cell::Cell::new(None) };
324
}
325
326
#[cfg(test)]
3274×
pub(crate) fn set_merge_batch_for_tests(n: Option<usize>) {
3284×
    MERGE_BATCH_OVERRIDE.with(|c| c.set(n));
3294×
}
330
33127×
fn merge_batch() -> usize {
33227×
    MERGE_BATCH_OVERRIDE
33327×
        .with(|c| c.get())
33427×
        .filter(|n| *n > 0)
33527×
        .or_else(|| {
33611×
            std::env::var("OXY_MERGE_BATCH")
33711×
                .ok()
33811×
                .and_then(|v| v.parse().ok())
33911×
        })
34027×
        .unwrap_or(MERGE_BATCH)
34127×
}
342
343
impl Repo {
344
    /// As `State::materialize_except`, for a tree whose graphs would not
345
    /// fit in memory at once. The few files the change builder reads
346
    /// graphs for (conflicted ones, and renderings whose lines are not
347
    /// one per live atom) are built in the state afterwards.
3485×
    fn materialize_batched(
3495×
        &self,
3505×
        base: &mut State,
3515×
        need: &[crate::ids::NodeId],
3525×
        skip: &HashSet<crate::ids::NodeId>,
3535×
    ) -> Result<crate::state::Snapshot> {
354
        use crate::graph::Materialized;
355
        use crate::state::{FileContent, FileState};
356
        use rayon::prelude::*;
3575×
        let view = base.tree().view();
3585×
        let mut rendered: HashMap<crate::ids::NodeId, (Vec<u8>, Materialized)> =
3595×
            HashMap::with_capacity(need.len());
3605×
        let mut keep_built: Vec<crate::ids::NodeId> = Vec::new();
36114×
        for batch in need.chunks(merge_batch()) {
36214×
            let t = std::time::Instant::now();
36314×
            let graphs = self.build_graphs_detached(base, batch)?;
36414×
            crate::timing::add("render: build graphs, batch", t.elapsed());
36514×
            let t = std::time::Instant::now();
36614×
            let batch_rendered: Vec<(crate::ids::NodeId, Vec<u8>, Materialized)> = graphs
36714×
                .par_iter()
36814×
                .map(|(node, g)| {
36914×
                    let (bytes, m) = base.render_graph(g);
37014×
                    (*node, bytes, m)
37114×
                })
37214×
                .collect();
37314×
            drop(graphs);
37414×
            crate::timing::add("render: render, batch", t.elapsed());
37514×
            for (node, bytes, m) in batch_rendered {
37614×
                if m.is_conflicted()
37714×
                    || m.live_atoms().len() != crate::diff::split_lines(&bytes).len()
3780×
                {
3790×
                    keep_built.push(node);
38014×
                }
38114×
                rendered.insert(node, (bytes, m));
382
            }
383
        }
3845×
        if !keep_built.is_empty() {
3850×
            self.load_graphs_of(base, &keep_built)?;
3865×
        }
3875×
        let mut files = BTreeMap::new();
3885×
        let mut unrendered = BTreeSet::new();
38916×
        for (path, e) in view.entries.iter() {
39016×
            let f = if e.kind == NodeKind::File && !e.mode.opaque {
39116×
                match rendered.remove(&e.node) {
39214×
                    Some((bytes, materialized)) => FileState {
39314×
                        node: e.node,
39414×
                        kind: e.kind,
39514×
                        mode: e.mode,
39614×
                        content: FileContent::Text {
39714×
                            bytes,
39814×
                            materialized,
39914×
                        },
40014×
                    },
4012×
                    None if skip.contains(&e.node) => {
4022×
                        unrendered.insert(path.clone());
4032×
                        FileState {
4042×
                            node: e.node,
4052×
                            kind: e.kind,
4062×
                            mode: e.mode,
4072×
                            content: FileContent::Text {
4082×
                                bytes: Vec::new(),
4092×
                                materialized: Materialized::default(),
4102×
                            },
4112×
                        }
412
                    }
413
                    None => {
414
                        // Neither asked for nor known unchanged: render it
415
                        // from a built graph, as the whole-tree path does.
4160×
                        self.load_graphs_of(base, &[e.node])?;
4170×
                        base.entry_state(e)
418
                    }
419
                }
420
            } else {
4210×
                base.entry_state(e)
422
            };
42316×
            files.insert(path.clone(), f);
424
        }
4255×
        Ok(crate::state::Snapshot {
4265×
            tree: view,
4275×
            files,
4285×
            unrendered,
4295×
        })
4305×
    }
431
}
432
433146×
fn git(dir: &Path, args: &[&str]) -> Result<String> {
434146×
    let out = Command::new("git")
435146×
        .arg("-C")
436146×
        .arg(dir)
437146×
        .args(args)
438146×
        .output()
439146×
        .map_err(|e| RepoError::Msg(format!("running git: {e}")))?;
440146×
    if !out.status.success() {
4410×
        return fail(format!(
442
            "git {} failed: {}",
4430×
            args.join(" "),
4440×
            String::from_utf8_lossy(&out.stderr).trim()
445
        ));
446146×
    }
447146×
    Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
448146×
}
449
450
impl Repo {
45141×
    fn git_marks_dir(&self, git_dir: &Path) -> Result<PathBuf> {
45241×
        let canonical = git_dir
45341×
            .canonicalize()
45441×
            .unwrap_or_else(|_| git_dir.to_path_buf());
45541×
        let key = hash_object(ObjectType::Chunk, canonical.to_string_lossy().as_bytes()).short(16);
45641×
        let dir = self.oxy.join("git").join(key);
45741×
        fs::create_dir_all(&dir)?;
45841×
        Ok(dir)
45941×
    }
460
461
    /// `state ref → Git commit sha` for bookmark states that equal a Git
462
    /// commit's tree (recorded by import and fetch).
46341×
    fn read_git_states(path: &Path) -> Result<BTreeMap<ObjectRef, String>> {
46441×
        let mut out = BTreeMap::new();
46541×
        if let Ok(text) = fs::read_to_string(path) {
46612×
            for line in text.lines() {
46712×
                if let Some((h, sha)) = line.split_once(' ') {
46812×
                    if let Some(h) = ObjectRef::from_hex(h) {
46912×
                        out.insert(h, sha.trim().to_string());
47012×
                    }
4710×
                }
472
            }
47331×
        }
47441×
        Ok(out)
47541×
    }
476
47713×
    fn record_git_state(&self, git_dir: &Path, state: &ObjectRef, sha: &str) -> Result<()> {
47813×
        let path = self.git_marks_dir(git_dir)?.join("git-states");
47913×
        let mut states = Self::read_git_states(&path)?;
48013×
        states.insert(*state, sha.to_string());
48113×
        let text: String = states
48213×
            .iter()
48314×
            .map(|(h, s)| format!("{} {s}\n", h.to_hex()))
48413×
            .collect();
48513×
        write_atomic(&path, text.as_bytes())
48613×
    }
487
488
    /// The bare mirror this repository keeps for the Git remote `url`.
4897×
    fn git_mirror(&self, url: &str) -> Result<PathBuf> {
4907×
        let key = hash_object(ObjectType::Chunk, url.as_bytes()).short(16);
4917×
        let dir = self.oxy.join("git").join(key).join("mirror.git");
4927×
        if !dir.join("HEAD").exists() {
4931×
            fs::create_dir_all(&dir)?;
4941×
            git(&dir, &["init", "--bare", "-q"])?;
4956×
        }
4967×
        Ok(dir)
4977×
    }
498
499
    /// Git commits already imported into the bookmark, by sha.
50026×
    pub(crate) fn imported_commits(&self) -> Result<HashMap<String, ChangeHash>> {
50126×
        let mut map = HashMap::new();
502100×
        for e in self.log()? {
503100×
            if e.is_working_copy {
50426×
                continue;
50574×
            }
50674×
            if let Some(sha) = e
50774×
                .change
50874×
                .message
50974×
                .lines()
51074×
                .rev()
511142×
                .find_map(|l| l.strip_prefix("Git-Commit: "))
51268×
            {
51368×
                map.insert(sha.trim().to_string(), e.hash);
51468×
            }
515
        }
51626×
        Ok(map)
51726×
    }
518
519
    /// Bring the mirror's `branch` up to date with the Git remote `url`.
520
    /// A remote without that branch yet leaves the mirror as it is.
5217×
    fn git_mirror_fetch(&self, mirror: &Path, url: &str, branch: &str) -> Result<()> {
5227×
        let spec = format!("+refs/heads/{branch}:refs/heads/{branch}");
5237×
        match git(mirror, &["fetch", "-q", url, &spec]) {
5247×
            Ok(_) => Ok(()),
5250×
            Err(e) => {
5260×
                let ls = git(mirror, &["ls-remote", "--heads", url, branch]).unwrap_or_default();
5270×
                if ls.trim().is_empty() {
5280×
                    Ok(())
529
                } else {
5300×
                    Err(e)
531
                }
532
            }
533
        }
5347×
    }
535
536
    /// Fetch `branch` of the Git repository at `url` (a path or a Git URL)
537
    /// through a mirror and import the commits not yet in the bookmark
538
    /// (spec §23). Works on a repository that already has history: the
539
    /// new changes join the bookmark like a pull.
5403×
    pub fn git_fetch(&self, url: &str, branch: &str) -> Result<GitImportReport> {
541
        // A local path, made absolute: git runs it from inside the mirror.
5423×
        let local = Path::new(url)
5433×
            .canonicalize()
5443×
            .ok()
5453×
            .filter(|_| !url.contains("://"));
5463×
        let absolute = local.as_ref().map(|p| p.to_string_lossy().into_owned());
5473×
        let url = absolute.as_deref().unwrap_or(url);
5483×
        let mirror = self.git_mirror(url)?;
5493×
        self.git_mirror_fetch(&mirror, url, branch)?;
5503×
        if git(
5513×
            &mirror,
5523×
            &[
5533×
                "rev-parse",
5543×
                "--verify",
5553×
                "-q",
5563×
                &format!("refs/heads/{branch}"),
5573×
            ],
5583×
        )
5593×
        .is_err()
560
        {
5610×
            return fail(format!("{url} has no branch {branch:?}"));
5623×
        }
5633×
        self.import_git_commits(
5643×
            &mirror,
5653×
            branch,
5663×
            &format!("oxy git fetch {url} {branch}"),
5671×
            &mut |_| {},
568
        )
5693×
    }
570
571
    /// Export the bookmark to `branch` of the Git repository at `url`
572
    /// through the mirror and push it. The remote's tip must already have
573
    /// been fetched (`git_fetch`), so the push fast-forwards.
5744×
    pub fn git_push(&self, url: &str, branch: &str) -> Result<GitExportReport> {
5754×
        let mirror = self.git_mirror(url)?;
5764×
        self.git_mirror_fetch(&mirror, url, branch)?;
5774×
        if let Ok(tip) = git(
5784×
            &mirror,
5794×
            &[
5804×
                "rev-parse",
5814×
                "--verify",
5824×
                "-q",
5834×
                &format!("refs/heads/{branch}"),
5844×
            ],
5854×
        ) {
5864×
            let known = self.git_known(&mirror)?;
5874×
            if !known.imported.contains_key(&tip) && !known.exported.contains_key(&tip) {
5881×
                return fail(format!(
589
                    "{url} {branch} has commits not yet fetched; run `oxy git fetch {url}` first"
590
                ));
5913×
            }
5920×
        }
5933×
        let report = self.git_export_as(&mirror, branch)?;
5943×
        let spec = format!("refs/heads/{branch}:refs/heads/{branch}");
5953×
        git(&mirror, &["push", "-q", url, &spec])?;
5963×
        Ok(report)
5974×
    }
598
59928×
    fn read_oxy_marks(path: &Path) -> Result<BTreeMap<ObjectRef, u64>> {
60028×
        let mut out = BTreeMap::new();
60128×
        if let Ok(text) = fs::read_to_string(path) {
60213×
            for line in text.lines() {
60313×
                if let Some((h, m)) = line.split_once(' ') {
60413×
                    if let (Some(h), Ok(m)) = (ObjectRef::from_hex(h), m.parse()) {
60513×
                        out.insert(h, m);
60613×
                    }
6070×
                }
608
            }
60919×
        }
61028×
        Ok(out)
61128×
    }
612
613
    /// Export the current bookmark to the Git repository at `git_dir`,
614
    /// creating a bare one if nothing is there. Each state the bookmark
615
    /// passed through (per the operation log) becomes one commit whose tree
616
    /// is that state, so the Git branch is a linear history of full trees.
6177×
    pub fn git_export(&self, git_dir: &Path) -> Result<GitExportReport> {
6187×
        let bookmark = self.config.bookmark.clone();
6197×
        self.git_export_as(git_dir, &bookmark)
6207×
    }
621
622
    /// `git_export` to the Git branch `branch`. A state that came from a
623
    /// Git commit (`oxy git fetch`) is not re-exported: later states chain
624
    /// onto that commit, so a push back to the Git remote fast-forwards.
62510×
    pub fn git_export_as(&self, git_dir: &Path, branch: &str) -> Result<GitExportReport> {
62610×
        if !git_dir.join("HEAD").exists() && !git_dir.join(".git").exists() {
6274×
            fs::create_dir_all(git_dir)?;
6284×
            git(git_dir, &["init", "--bare", "-q"])?;
6296×
        }
63010×
        let marks_dir = self.git_marks_dir(git_dir)?;
63110×
        let git_marks = marks_dir.join("git-marks");
63210×
        let oxy_marks_path = marks_dir.join("oxy-marks");
63310×
        let mut marks = Self::read_oxy_marks(&oxy_marks_path)?;
63410×
        let git_states = Self::read_git_states(&marks_dir.join("git-states"))?;
63510×
        let mut next_mark = marks.values().copied().max().unwrap_or(0) + 1;
636
63710×
        let bookmark = self.config.bookmark.clone();
63810×
        let mut stream: Vec<u8> = Vec::new();
63910×
        let mut report = GitExportReport {
64010×
            branch: branch.to_string(),
64110×
            heads: self.bookmark_heads()?.len(),
64210×
            ..Default::default()
643
        };
644
645
        // States along the bookmark, oldest first, skipping unchanged and pruned ones.
64610×
        let mut ops = self.op_log()?;
64710×
        ops.reverse();
64810×
        let mut prev: Option<(ObjectRef, Vec<ChangeHash>)> = None;
649
        // The fast-import parent of the next commit: `:mark` or a commit sha.
65010×
        let mut last_parent: Option<String> = None;
65144×
        for (_, op) in &ops {
65244×
            let Some(state_ref) = op.view.bookmarks.get(&bookmark).copied() else {
6530×
                continue;
654
            };
65544×
            if prev.as_ref().is_some_and(|(p, _)| *p == state_ref) {
65612×
                continue;
65732×
            }
65832×
            let Ok(state) = self.store.get_state(&state_ref) else {
6590×
                continue;
660
            };
66132×
            if state.heads.is_empty() {
66210×
                prev = Some((state_ref, vec![]));
66310×
                continue;
66422×
            }
66522×
            if let Some(sha) = git_states.get(&state_ref) {
6664×
                report.skipped += 1;
6674×
                last_parent = Some(sha.clone());
6684×
                prev = Some((state_ref, state.heads.clone()));
6694×
                continue;
67018×
            }
67118×
            if let Some(m) = marks.get(&state_ref) {
6729×
                report.skipped += 1;
6739×
                last_parent = Some(format!(":{m}"));
6749×
                prev = Some((state_ref, state.heads.clone()));
6759×
                continue;
6769×
            }
6779×
            let prev_heads = prev.as_ref().map(|(_, h)| h.clone()).unwrap_or_default();
6789×
            let added = self
6799×
                .negotiate(&prev_heads, &state.heads)
6809×
                .unwrap_or_default();
6819×
            let loaded = self.load_state(&state.heads)?;
6829×
            let snapshot = loaded.materialize();
6839×
            let (author, secs, mut message) = match added.as_slice() {
6846×
                [h] => {
6856×
                    let c = self.store.get_change(h)?;
6866×
                    let mut m = c.message.clone();
6876×
                    if !m.ends_with('\n') {
6886×
                        m.push('\n');
6896×
                    }
6906×
                    m.push_str(&format!(
6916×
                        "\nChange-Id: {}\nOxy-Hash: {}\n",
6926×
                        c.change_id.to_hex(),
6936×
                        h.to_hex()
6946×
                    ));
6956×
                    (self.git_author(&c), c.hlc.wall_ms / 1000, m)
696
                }
6973×
                [] => (
6980×
                    "oxy <oxy@localhost>".to_string(),
6990×
                    op.hlc.wall_ms / 1000,
7000×
                    format!("oxy: {}\n\n", op.command),
7010×
                ),
7023×
                many => {
7033×
                    let mut m = format!("oxy: {} ({} changes)\n\n", op.command, many.len());
70422×
                    for h in many {
70522×
                        let c = self.store.get_change(h)?;
70622×
                        m.push_str(&format!(
70722×
                            "- {} {}\n",
70822×
                            c.change_id.short(8),
70922×
                            c.message.lines().next().unwrap_or("")
71022×
                        ));
711
                    }
7123×
                    m.push_str(&format!(
713
                        "\nOxy-Changes: {}\n",
7143×
                        many.iter()
71522×
                            .map(|h| h.to_hex())
7163×
                            .collect::<Vec<_>>()
7173×
                            .join(" ")
718
                    ));
7193×
                    ("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, m)
720
                }
721
            };
7229×
            message.push_str(&format!("Oxy-State: {}\n", state_ref.to_hex()));
7239×
            if snapshot.is_conflicted() {
7240×
                message.push_str("Oxy-Conflicted: true\n");
7259×
            }
7269×
            let mark = next_mark;
7279×
            next_mark += 1;
7289×
            let parents: Vec<String> = last_parent.clone().into_iter().collect();
7299×
            self.write_commit(
7309×
                &mut stream,
7319×
                "refs/oxy/export",
7329×
                mark,
7339×
                &author,
7349×
                secs,
7359×
                &message,
7369×
                &parents,
7379×
                &snapshot,
7380×
            )?;
7399×
            marks.insert(state_ref, mark);
7409×
            last_parent = Some(format!(":{mark}"));
7419×
            prev = Some((state_ref, state.heads.clone()));
7429×
            report.exported += 1;
743
        }
74410×
        if let Some(p) = &last_parent {
74510×
            stream.extend_from_slice(format!("reset refs/heads/{branch}\nfrom {p}\n\n").as_bytes());
74610×
        }
747
        // Tags: a tag whose state was exported as a bookmark state points at
748
        // that commit; otherwise its state gets a commit of its own.
74910×
        for t in self.tags()? {
7500×
            let target = match (git_states.get(&t.state), marks.get(&t.state)) {
7510×
                (Some(sha), _) => sha.clone(),
7520×
                (None, Some(m)) => format!(":{m}"),
753
                (None, None) => {
7540×
                    let loaded = self.load_state(&t.heads)?;
7550×
                    let snapshot = loaded.materialize();
7560×
                    let mark = next_mark;
7570×
                    next_mark += 1;
7580×
                    let message = format!(
759
                        "oxy: tag {}\n\n{}Oxy-State: {}\n",
760
                        t.name,
7610×
                        if t.message.is_empty() {
7620×
                            String::new()
763
                        } else {
7640×
                            format!("{}\n\n", t.message)
765
                        },
7660×
                        t.state.to_hex()
767
                    );
7680×
                    self.write_commit(
7690×
                        &mut stream,
7700×
                        "refs/oxy/export",
7710×
                        mark,
7720×
                        "oxy <oxy@localhost>",
7730×
                        Hlc::now().wall_ms / 1000,
7740×
                        &message,
7750×
                        &[],
7760×
                        &snapshot,
7770×
                    )?;
7780×
                    marks.insert(t.state, mark);
7790×
                    report.exported += 1;
7800×
                    format!(":{mark}")
781
                }
782
            };
7830×
            stream.extend_from_slice(
7840×
                format!("reset refs/tags/{}\nfrom {target}\n\n", t.name).as_bytes(),
785
            );
786
        }
78710×
        stream.extend_from_slice(b"done\n");
788
78910×
        let mut cmd = Command::new("git");
790
        // `--force`: the scratch ref may jump between unrelated states.
79110×
        cmd.arg("-C")
79210×
            .arg(git_dir)
79310×
            .args(["fast-import", "--quiet", "--done", "--force"])
79410×
            .arg(format!("--export-marks={}", git_marks.display()));
79510×
        if git_marks.exists() {
7965×
            cmd.arg(format!("--import-marks={}", git_marks.display()));
7975×
        }
79810×
        let mut child = cmd
79910×
            .stdin(Stdio::piped())
80010×
            .stdout(Stdio::null())
80110×
            .stderr(Stdio::piped())
80210×
            .spawn()
80310×
            .map_err(|e| RepoError::Msg(format!("running git fast-import: {e}")))?;
80410×
        child.stdin.take().unwrap().write_all(&stream)?;
80510×
        let out = child.wait_with_output()?;
80610×
        if !out.status.success() {
8070×
            return fail(format!(
808
                "git fast-import failed: {}",
8090×
                String::from_utf8_lossy(&out.stderr).trim()
810
            ));
81110×
        }
81210×
        let text: String = marks
81310×
            .iter()
81418×
            .map(|(h, m)| format!("{} {m}\n", h.to_hex()))
81510×
            .collect();
81610×
        write_atomic(&oxy_marks_path, text.as_bytes())?;
81710×
        let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]);
81810×
        report.skipped_subrepos = self.git_export_skipped.replace(0);
81910×
        Ok(report)
82010×
    }
821
822
    /// Import branch `branch` of the Git repository at `git_dir` into this
823
    /// (empty) repository's bookmark (spec §23): one change per commit, its
824
    /// edits the difference between the union of its parents' states and its
825
    /// tree, so merge commits become resolutions. Submodules are skipped.
82611×
    pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result<GitImportReport> {
82711×
        self.import_git_with(git_dir, branch, &mut |_| {})
82811×
    }
829
830
    /// `import_git`, telling `progress` where it is every few hundred
831
    /// commits: the count so far, the merges, and the commit in hand, so a
832
    /// long import is never silent and a run that dies says where.
83311×
    pub fn import_git_with(
83411×
        &self,
83511×
        git_dir: &Path,
83611×
        branch: &str,
83711×
        progress: crate::store::Progress<'_>,
83811×
    ) -> Result<GitImportReport> {
83911×
        if !self.bookmark_heads()?.is_empty() {
8400×
            return fail("git import needs an empty bookmark; import into a fresh repository");
84111×
        }
84211×
        self.import_git_commits(
84311×
            git_dir,
84411×
            branch,
84511×
            &format!("oxy git import {} {branch}", git_dir.display()),
84611×
            progress,
847
        )
84811×
    }
849
850
    /// Import the commits of `branch` that the bookmark does not have yet,
851
    /// then move the bookmark to the union of its heads and theirs.
85214×
    fn import_git_commits(
85314×
        &self,
85414×
        git_dir: &Path,
85514×
        branch: &str,
85614×
        command: &str,
85714×
        progress: crate::store::Progress<'_>,
85814×
    ) -> Result<GitImportReport> {
85914×
        let batch = self.store.begin_batch()?;
86014×
        let result = self.import_git_commits_batched(git_dir, branch, command, progress);
86114×
        batch.finish()?;
86214×
        result
86314×
    }
864
86514×
    fn import_git_commits_batched(
86614×
        &self,
86714×
        git_dir: &Path,
86814×
        branch: &str,
86914×
        command: &str,
87014×
        progress: crate::store::Progress<'_>,
87114×
    ) -> Result<GitImportReport> {
87214×
        let commits = list_commits(git_dir, branch)?;
87314×
        let mut cat = CatFile::start(git_dir)?;
87414×
        let mut report = GitImportReport::default();
87514×
        let mut known = self.git_known(git_dir)?;
87614×
        let old_heads = self.bookmark_heads()?;
877
        // The bookmark's heads as changes arrive: a change stops being a
878
        // head when another depends on it, so no full state of every
879
        // imported change is needed at the end.
88014×
        let mut heads_now: BTreeSet<ChangeHash> = old_heads.iter().copied().collect();
881
        // The state and Git tree of imported commits whose first-parent
882
        // children are still to come, so a child starts from its parent's
883
        // state instead of rebuilding it from history. The last child takes
884
        // it; earlier ones clone it. At most `max_kept` are kept (a later
885
        // child of an evicted one rebuilds).
886
        // OXY_IMPORT_KEEP overrides it, for measuring.
88714×
        let max_kept: usize = std::env::var("OXY_IMPORT_KEEP")
88814×
            .ok()
88914×
            .and_then(|v| v.parse().ok())
89014×
            .unwrap_or(8);
89114×
        let mut waiting: HashMap<&str, usize> = HashMap::new();
89285×
        for gc in &commits {
89385×
            if let Some(p1) = gc.parents.first() {
89469×
                *waiting.entry(p1.as_str()).or_default() += 1;
89569×
            }
896
        }
897
        type Kept = (State, BTreeMap<String, (String, String)>);
89814×
        let mut kept: HashMap<String, Kept> = HashMap::new();
899
        // Every commit to import that has a parent, diffed against its first.
90014×
        let pairs: Vec<(String, String)> = commits
90114×
            .iter()
90285×
            .filter(|gc| {
90385×
                !known.imported.contains_key(&gc.sha) && !known.exported.contains_key(&gc.sha)
90485×
            })
90580×
            .filter_map(|gc| Some((gc.sha.clone(), gc.parents.first()?.clone())))
90614×
            .collect();
90714×
        let mut diffs = DiffTree::start(git_dir, pairs)?;
90814×
        let mut rebuilt = 0usize;
90914×
        let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
91014×
        let mut last: Option<String> = None;
91114×
        let mut blob_cache: HashMap<String, Vec<u8>> = HashMap::new();
91214×
        let mut cache_bytes: usize = 0;
913
        // Time spent per phase over all commits (`OXY_TIMINGS=1`).
91414×
        let mut spent = [std::time::Duration::ZERO; 6];
91514×
        let mut clock = std::time::Instant::now();
916
        // One line per commit on stderr with OXY_IMPORT_TRACE=1: where its
917
        // base came from, what a merge added, the graphs its state holds,
918
        // the kept states and the process's resident memory.
91914×
        let trace = std::env::var_os("OXY_IMPORT_TRACE").is_some();
92014×
        let rss_mb = || -> u64 {
9210×
            std::fs::read_to_string("/proc/self/statm")
9220×
                .ok()
9230×
                .and_then(|s| s.split_whitespace().nth(1)?.parse::<u64>().ok())
9240×
                .map(|pages| (pages * 4096) >> 20)
9250×
                .unwrap_or(0)
9260×
        };
92714×
        let mut done = 0usize;
928
        // Atoms a kept state's graphs may hold before they are shed;
929
        // OXY_IMPORT_KEPT_ATOMS overrides it, for measuring.
93014×
        let kept_atoms: usize = std::env::var("OXY_IMPORT_KEPT_ATOMS")
93114×
            .ok()
93214×
            .and_then(|v| v.parse().ok())
93314×
            .unwrap_or(KEPT_ATOMS);
934
        // Graphs turned back into history (see `State::shed_graphs`).
93514×
        let mut shed = 0usize;
936480×
        let mut lap = |i: usize, clock: &mut std::time::Instant| {
937480×
            let now = std::time::Instant::now();
938480×
            spent[i] += now - *clock;
939480×
            *clock = now;
940480×
        };
941
942
        // OXY_IMPORT_CHECKPOINT overrides it, for testing.
94314×
        let checkpoint_every: usize = std::env::var("OXY_IMPORT_CHECKPOINT")
94414×
            .ok()
94514×
            .and_then(|v| v.parse().ok())
94614×
            .filter(|n| *n > 0)
94714×
            .unwrap_or(CHECKPOINT_COMMITS);
948
        // OXY_IMPORT_STOP_AFTER ends the import cleanly after that many
949
        // commits, its timings reported, for measuring at depth.
95014×
        let stop_after: Option<usize> = std::env::var("OXY_IMPORT_STOP_AFTER")
95114×
            .ok()
95214×
            .and_then(|v| v.parse().ok());
953
        // Merges built from the paths they can change (oxygen#80): on VS
954
        // Code at 139k commits the same 400 commits came out identical to
955
        // whole-tree merges, with merges 3.2x faster. OXY_IMPORT_WHOLE_MERGES=1
956
        // builds them from the whole tree, for comparing.
95714×
        let partial_merges = std::env::var_os("OXY_IMPORT_WHOLE_MERGES").is_none_or(|v| v == "0");
95814×
        let total = commits.len();
95985×
        for (index, gc) in commits.iter().enumerate() {
96085×
            if known.imported.contains_key(&gc.sha) || known.exported.contains_key(&gc.sha) {
9615×
                continue;
96280×
            }
96380×
            if stop_after.is_some_and(|n| done >= n) {
9640×
                break;
96580×
            }
96680×
            let commit_started = std::time::Instant::now();
96780×
            if index % 500 == 0 {
96812×
                progress(&format!(
96912×
                    "{index} of {total} commits, {} merges; at {} {}",
97012×
                    report.merges,
97112×
                    &gc.sha[..12.min(gc.sha.len())],
97212×
                    gc.message
97312×
                        .lines()
97412×
                        .next()
97512×
                        .unwrap_or("")
97612×
                        .chars()
97712×
                        .take(60)
97812×
                        .collect::<String>()
97912×
                ));
98068×
            }
98180×
            let mut base_how = "root";
982
            // Base: union of every parent's state, starting from the first
983
            // parent's kept state when there is one; its tree tells which
984
            // paths this commit leaves alone.
98580×
            let (mut base, prev_tree) = match gc.parents.first() {
98666×
                Some(p1) => {
98766×
                    let left = waiting.get_mut(p1.as_str()).map(|n| {
98866×
                        *n -= 1;
98966×
                        *n
99066×
                    });
99166×
                    let found = if left == Some(0) {
99265×
                        kept_order.retain(|k| k != p1);
99347×
                        kept.remove(p1.as_str())
994
                    } else {
99519×
                        kept.get(p1.as_str()).cloned()
996
                    };
99766×
                    let how = match (&found, left) {
99846×
                        (Some(_), Some(0)) => "kept",
99919×
                        (Some(_), _) => "cloned",
10001×
                        (None, _) => "rebuilt",
1001
                    };
100266×
                    if trace {
10030×
                        base_how = how;
100466×
                    }
100566×
                    match found {
100665×
                        Some((st, tree)) => (st, Some(tree)),
1007
                        None => {
1008
                            // History only: graphs are built for the files a
1009
                            // commit changes, so memory follows what changes,
1010
                            // not the whole tree.
10111×
                            let mut st = State::new_history_only();
10121×
                            self.add_git_ancestors(&mut st, git_dir, &[p1.as_str()], &known)?;
1013
                            // The parent's tree, so this commit still
1014
                            // renders only what it changes.
10151×
                            rebuilt += 1;
10161×
                            (st, Some(list_tree(git_dir, p1)?))
1017
                        }
1018
                    }
1019
                }
102014×
                None => (State::new_history_only(), None),
1021
            };
1022
            // Nodes the other parents' changes touch: the first parent's
1023
            // tree cannot vouch for them.
102480×
            let mut touched: HashSet<NodeId> = HashSet::new();
102580×
            let mut merged = 0usize;
1026
            // Where the first parent has each node, read before the merged
1027
            // changes move anything: a merge's partial build names the paths
1028
            // they touch in the first parent's tree as well as in the merged
1029
            // state (oxygen#80).
103080×
            let mut first_parent_view: Option<crate::tree::TreeView> = None;
103180×
            if gc.parents.len() > 1 {
103223×
                report.merges += 1;
103323×
                if prev_tree.is_some() && partial_merges {
103423×
                    first_parent_view = Some(base.tree().view());
103523×
                }
103623×
                let mut args: Vec<String> = gc.parents[1..].to_vec();
103723×
                args.push(format!("^{}", gc.parents[0]));
103823×
                let refs: Vec<&str> = args.iter().map(String::as_str).collect();
1039
                // A merge of a long-lived branch brings thousands of
1040
                // changes: added to built graphs, which kept states share,
1041
                // they were copied and applied whole (vscode's import passed
1042
                // 14 GB on such merges). Shed the graphs first, so the
1043
                // changes go in as skeletons and only what this commit
1044
                // touches is built again.
104523×
                if base.built_count() > 0 {
104620×
                    let mut count_args = vec!["rev-list", "--count"];
104720×
                    count_args.extend(refs.iter().copied());
104820×
                    let others: usize = git(git_dir, &count_args)?.trim().parse().unwrap_or(0);
104920×
                    let shed_at = std::env::var("OXY_MERGE_SHED")
105020×
                        .ok()
105120×
                        .and_then(|v| v.parse().ok())
105220×
                        .unwrap_or(MERGE_SHED_COMMITS);
105320×
                    if others > shed_at {
10540×
                        shed += base.shed_graphs(|b| self.store_blob(b))?;
105520×
                    }
10563×
                }
105723×
                let before = base.len();
105823×
                self.add_git_ancestors_touching(&mut base, git_dir, &refs, &known, &mut touched)?;
105923×
                merged = base.len() - before;
106057×
            }
106180×
            lap(0, &mut clock);
1062
            // The commit's tree: its first parent's with the diff applied,
1063
            // or listed whole when that tree is not kept.
106480×
            let diff = match gc.parents.first() {
106566×
                Some(_) => Some(diffs.diff(&gc.sha)?),
106614×
                None => None,
1067
            };
106880×
            let (tree, changed) = match (prev_tree, diff) {
106966×
                (Some(mut t), Some(diff)) => {
107066×
                    let mut changed: HashSet<String> = HashSet::new();
107182×
                    for (path, entry) in diff {
107282×
                        match entry {
107368×
                            Some(v) => {
107468×
                                t.insert(path.clone(), v);
107568×
                            }
107614×
                            None => {
107714×
                                t.remove(&path);
107814×
                            }
1079
                        }
108082×
                        changed.insert(path);
1081
                    }
108266×
                    (t, Some(changed))
1083
                }
108414×
                _ => (list_tree(git_dir, &gc.sha)?, None),
1085
            };
108680×
            lap(1, &mut clock);
1087
            // Text files with the same blob and mode as in the first
1088
            // parent's tree equal the base's rendering: they are neither
1089
            // rendered nor read nor compared (spec §12.3, as for a clean
1090
            // working copy), so a commit costs what it changes.
109180×
            let mut same: HashMap<&str, NodeId> = HashMap::new();
1092
            // Opaque files and symlinks likewise, when the base has one
1093
            // blob for them: they are neither read nor stored again.
109480×
            let mut same_opaque: HashSet<&str> = HashSet::new();
1095
            // An ordinary commit in a tree without submodules is built from
1096
            // the paths it changes alone (oxygen#25): with the `.oxyattributes`
1097
            // files that decide how they are stored, and the directories its
1098
            // removals may leave empty. A merge, or a commit whose parent's
1099
            // tree is not at hand, is built from its whole tree.
1100
            // A merge can differ from its first parent only at the paths
1101
            // Git's diff names and where the merged changes touched: those
1102
            // paths in the first parent's tree and in the merged state. With
1103
            // them added it is built like an ordinary commit, instead of
1104
            // from its whole tree (18 s for any merge at 118k commits of
1105
            // VS Code, oxygen#80). OXY_IMPORT_WHOLE_MERGES=1 takes the
1106
            // whole-tree path.
110780×
            let mut changed = changed;
1108
            // A tree conflict in the merged state (both sides placed a file
1109
            // at one path) is resolved by the whole-tree path.
111080×
            let mut merge_how = if gc.parents.len() > 1 {
111123×
                if first_parent_view.is_some() {
111223×
                    "partial"
1113
                } else {
11140×
                    "whole"
1115
                }
1116
            } else {
111757×
                "-"
1118
            };
1119
            // Only a conflict among what the merged changes touched; one
1120
            // elsewhere stays as it was in the first parent either way.
112180×
            if first_parent_view.is_some()
112223×
                && base.tree().view().conflicts.iter().any(|c| {
1123
                    use crate::tree::TreeConflict::*;
11247×
                    match c {
11257×
                        Name { nodes, .. } => nodes.iter().any(|n| touched.contains(n)),
11261×
                        MultiPlacement { node, .. } | Cycle { node } | Mode { node, .. } => {
11271×
                            touched.contains(node)
1128
                        }
11291×
                        Orphan { node, parent } => {
11301×
                            touched.contains(node) || touched.contains(parent)
1131
                        }
1132
                    }
11337×
                })
11347×
            {
11357×
                first_parent_view = None;
11367×
                merge_how = "conflicted";
113773×
            }
113880×
            if let (Some(ch), Some(fp)) = (changed.as_mut(), &first_parent_view) {
113916×
                let union = base.tree().view();
114032×
                for v in [fp, &union] {
1141133×
                    for (path, e) in v.entries.iter() {
1142133×
                        if touched.contains(&e.node) {
114337×
                            ch.insert(path.clone());
114496×
                        }
1145
                    }
1146
                }
114764×
            }
114880×
            let partial = changed.is_some()
114966×
                && (gc.parents.len() == 1 || first_parent_view.is_some())
1150161×
                && !tree.values().any(|(mode, _)| mode == "160000");
115180×
            let mut rel: BTreeSet<&str> = BTreeSet::new();
1152
            // Text files whose graphs this commit builds, for the trace.
1153
            let need_n: usize;
115480×
            let base_snapshot = if let (true, Some(changed)) = (partial, &changed) {
115558×
                let view = base.tree().view();
115658×
                rel.extend(changed.iter().map(String::as_str));
115758×
                rel.extend(
115858×
                    tree.keys()
115958×
                        .map(String::as_str)
1160158×
                        .filter(|p| *p == ".oxyattributes" || p.ends_with("/.oxyattributes")),
1161
                );
116258×
                let mut dirs: BTreeSet<&str> = BTreeSet::new();
116380×
                for p in changed.iter().filter(|p| !tree.contains_key(*p)) {
116414×
                    let mut end = p.len();
116524×
                    while let Some(i) = p[..end].rfind('/') {
116610×
                        if !dirs.insert(&p[..i]) {
11670×
                            break;
116810×
                        }
116910×
                        end = i;
1170
                    }
1171
                }
117258×
                let need: Vec<NodeId> = rel
117358×
                    .iter()
117480×
                    .filter_map(|p| view.entries.get(*p))
117560×
                    .filter(|e| e.kind == NodeKind::File && !e.mode.opaque)
117658×
                    .map(|e| e.node)
117758×
                    .collect();
117858×
                crate::timing::add("render: same and need", clock.elapsed());
117958×
                need_n = need.len();
118058×
                let t = std::time::Instant::now();
118158×
                self.load_graphs_of(&mut base, &need)?;
118258×
                crate::timing::add("render: load graphs", t.elapsed());
118358×
                let t = std::time::Instant::now();
118458×
                let snap = base.materialize_paths(view, rel.iter().chain(dirs.iter()).copied());
118558×
                crate::timing::add("render: materialize", t.elapsed());
118658×
                snap
1187
            } else {
118822×
                if let Some(changed) = &changed {
11898×
                    let view = base.tree().view();
119019×
                    for path in tree.keys() {
1191
                        // Submodule handling reads these two whatever changed.
119219×
                        if changed.contains(path) || path == ".gitmodules" || path == ".oxymodules"
1193
                        {
119410×
                            continue;
11959×
                        }
11969×
                        if let Some(e) = view.entries.get(path) {
11975×
                            if touched.contains(&e.node) {
11980×
                                continue;
11995×
                            }
12005×
                            let opaque = e.kind == NodeKind::Symlink
12015×
                                || (e.kind == NodeKind::File && e.mode.opaque);
12025×
                            if e.kind == NodeKind::File && !e.mode.opaque {
12035×
                                same.insert(path.as_str(), e.node);
12045×
                            } else if opaque && base.live_blobs(&e.node).len() == 1 {
12050×
                                same_opaque.insert(path.as_str());
12060×
                            }
12074×
                        }
1208
                    }
120914×
                }
1210
                // Build the graphs of every text file the commit may change.
121122×
                let skip: HashSet<NodeId> = same.values().copied().collect();
121222×
                let need: Vec<NodeId> = base
121322×
                    .tree()
121422×
                    .view()
121522×
                    .entries
121622×
                    .values()
121722×
                    .filter(|e| {
121821×
                        e.kind == NodeKind::File && !e.mode.opaque && !skip.contains(&e.node)
121921×
                    })
122022×
                    .map(|e| e.node)
122122×
                    .collect();
122222×
                crate::timing::add("render: same and need", clock.elapsed());
122322×
                need_n = need.len();
122422×
                let t = std::time::Instant::now();
122522×
                let snap = if need.len() > merge_batch() {
1226
                    // A merge onto a base with no built graphs: thousands
1227
                    // of files, whose graphs would not fit at once.
12285×
                    self.materialize_batched(&mut base, &need, &skip)?
1229
                } else {
123017×
                    let g = std::time::Instant::now();
123117×
                    self.load_graphs_of(&mut base, &need)?;
123217×
                    crate::timing::add("render: whole, load graphs", g.elapsed());
123317×
                    let m = std::time::Instant::now();
123417×
                    let snap = base.materialize_except(&skip);
123517×
                    crate::timing::add("render: whole, materialize", m.elapsed());
123617×
                    snap
1237
                };
123822×
                crate::timing::add("render: whole tree", t.elapsed());
123922×
                snap
1240
            };
1241
124280×
            lap(2, &mut clock);
1243
            // The commit's tree as in-memory entries.
124480×
            let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();
124580×
            let mut gitlinks: Vec<(String, String)> = Vec::new();
124680×
            let listed: Box<dyn Iterator<Item = (&String, &(String, String))>> = if partial {
124780×
                Box::new(rel.iter().filter_map(|p| tree.get_key_value(*p)))
1248
            } else {
124922×
                Box::new(tree.iter())
1250
            };
1251110×
            for (path, (mode, sha)) in listed {
1252110×
                let (path, sha) = (path.clone(), sha.clone());
1253110×
                let (kind, exec) = match mode.as_str() {
1254110×
                    "100644" | "100664" => (NodeKind::File, false),
125513×
                    "100755" => (NodeKind::File, true),
12561×
                    "120000" => (NodeKind::Symlink, false),
12571×
                    "160000" => {
12581×
                        gitlinks.push((path, sha));
12591×
                        continue;
1260
                    }
12610×
                    _ => continue,
1262
                };
1263109×
                if same.contains_key(path.as_str()) || same_opaque.contains(path.as_str()) {
12645×
                    let mut e = FsEntry::new(kind, exec, Vec::new());
12655×
                    e.unchanged = true;
12665×
                    entries.insert(path, e);
12675×
                    continue;
1268104×
                }
1269104×
                let content = match blob_cache.get(&sha) {
127032×
                    Some(c) => c.clone(),
1271
                    None => {
127272×
                        let c = cat.read(&sha)?;
127372×
                        if cache_bytes + c.len() > 64 * 1024 * 1024 {
12740×
                            blob_cache.clear();
12750×
                            cache_bytes = 0;
127672×
                        }
127772×
                        cache_bytes += c.len();
127872×
                        blob_cache.insert(sha.clone(), c.clone());
127972×
                        c
1280
                    }
1281
                };
1282104×
                entries.insert(path, FsEntry::new(kind, exec, content));
1283
            }
1284
1285
            // Submodules (spec §23): gitlinks become subrepo nodes with Git
1286
            // pins, and `.gitmodules` also yields `.oxymodules`.
128780×
            if !gitlinks.is_empty() {
12881×
                report.submodules += gitlinks.len();
12891×
                let gm = entries
12901×
                    .get(".gitmodules")
12911×
                    .map(|e| {
12921×
                        crate::subrepo::Modules::from_gitmodules(&String::from_utf8_lossy(
12931×
                            &e.content,
12941×
                        ))
12951×
                    })
12961×
                    .unwrap_or_default();
12971×
                let mut modules = entries
12981×
                    .get(".oxymodules")
12991×
                    .map(|e| crate::subrepo::Modules::parse(&String::from_utf8_lossy(&e.content)))
13001×
                    .unwrap_or_default();
13011×
                for (path, sha) in &gitlinks {
13021×
                    let m = gm.entries.get(path);
13031×
                    let branch = m
13041×
                        .and_then(|m| m.bookmark.clone())
13051×
                        .unwrap_or_else(|| "HEAD".into());
13061×
                    entries.insert(
13071×
                        path.clone(),
13081×
                        FsEntry::new(
13091×
                            NodeKind::Subrepo,
1310
                            false,
13111×
                            crate::subrepo::Pin::Git {
13121×
                                sha: sha.clone(),
13131×
                                branch,
13141×
                            }
13151×
                            .encode(),
1316
                        ),
1317
                    );
13181×
                    if let Some(m) = m {
13191×
                        modules.entries.insert(path.clone(), m.clone());
13201×
                    }
1321
                }
13221×
                if !modules.entries.is_empty() {
13231×
                    entries.insert(
13241×
                        ".oxymodules".into(),
13251×
                        FsEntry::new(NodeKind::File, false, modules.to_text().into_bytes()),
13261×
                    );
13271×
                }
132879×
            }
132980×
            lap(3, &mut clock);
1330
            // New nodes' identifiers derive from the commit, so importing the
1331
            // same Git history twice gives the same changes (spec §23).
133280×
            let seed = *blake3::hash(format!("oxy git import\0{}", gc.sha).as_bytes()).as_bytes();
133380×
            let mut built = if partial {
1334
                // Whether the new tree has anything under `dir`.
133558×
                let dir_needed = |dir: &str| {
133610×
                    let prefix = format!("{dir}/");
133710×
                    tree.range(prefix.clone()..)
133810×
                        .next()
133910×
                        .is_some_and(|(k, _)| k.starts_with(&prefix))
134010×
                };
134158×
                self.build_from_entries_seeded(
134258×
                    &base,
134358×
                    &base_snapshot,
134458×
                    &entries,
134558×
                    seed,
134658×
                    Some(&dir_needed),
13470×
                )?
1348
            } else {
134922×
                self.build_from_entries_seeded(&base, &base_snapshot, &entries, seed, None)?
1350
            };
135180×
            lap(4, &mut clock);
1352
            // A file or submodule with several live blobs (branches that
1353
            // replaced it independently) is resolved to the commit's blob:
1354
            // Git's merge is the authority here, unlike a working copy,
1355
            // where a candidate on disk is not yet a decision (spec §23).
135691×
            for (path, f) in &base_snapshot.files {
135791×
                let (FileContent::Opaque { blobs }, Some(e)) = (&f.content, entries.get(path))
1358
                else {
135987×
                    continue;
1360
                };
13614×
                if blobs.len() < 2 || f.kind != e.kind {
13623×
                    continue;
13631×
                }
13641×
                let chosen = self.store_blob(&e.content)?;
13652×
                for b in blobs {
13662×
                    built.edits.push(Edit::Replace {
13672×
                        node: f.node,
13682×
                        old: Some(*b),
13692×
                        new: Some(chosen),
13702×
                    });
13712×
                }
13721×
                built.resolves = true;
13731×
                built
13741×
                    .contributors
13751×
                    .extend(base.live_replaces(&f.node).into_iter().map(|(c, _)| c));
1376
            }
1377
            // It shares the base's tree view, which applying the change
1378
            // then updates in place rather than copying.
137980×
            drop(base_snapshot);
138080×
            drop(entries);
138180×
            let mut deps: BTreeSet<ChangeHash> = built.contributors.iter().copied().collect();
138280×
            let mut message = gc.message.clone();
138380×
            if !message.ends_with('\n') {
138480×
                message.push('\n');
138580×
            }
138680×
            message.push_str(&format!(
138780×
                "\nGit-Commit: {}\nGit-Author: {}\n",
138880×
                gc.sha, gc.author
138980×
            ));
139080×
            let mut change = Change {
1391
                version: crate::objects::CHANGE_VERSION,
139280×
                change_id: change_id_for(&gc.sha, &gc.message),
139380×
                deps: vec![],
1394
                author: ObjectRef::ZERO,
139580×
                hlc: Hlc {
139680×
                    wall_ms: gc.author_secs * 1000,
139780×
                    logical: 0,
139880×
                },
139980×
                message,
140080×
                tree_ops: built.tree_ops,
140180×
                edits: built.edits,
140280×
                kind: if built.resolves {
140312×
                    ChangeKind::Resolution
1404
                } else {
140568×
                    ChangeKind::Normal
1406
                },
1407
            };
140880×
            let t = std::time::Instant::now();
1409
            // A deletion depends on whoever inserted the live atoms after
1410
            // it, which only the file's graph knows: build the few the
1411
            // batched rendering left deferred.
141280×
            let need: Vec<NodeId> = change
141380×
                .edits
141480×
                .iter()
141589×
                .filter(|e| matches!(e, Edit::Delete { .. }))
141680×
                .map(|e| e.node())
141780×
                .filter(|n| !base.is_built(n))
141880×
                .collect();
141980×
            if !need.is_empty() {
14200×
                self.load_graphs_of(&mut base, &need)?;
142180×
            }
142280×
            deps.extend(referenced_changes(&change, &base));
142380×
            crate::timing::add("store: referenced", t.elapsed());
142480×
            change.deps = deps.into_iter().collect();
142580×
            change.normalize();
142680×
            let hash = self.store.put_change(&change)?;
142788×
            for d in &change.deps {
142888×
                heads_now.remove(d);
142988×
            }
143080×
            heads_now.insert(hash);
1431
            // Built graphs take its edits whole; files not built keep only
1432
            // where its text is, read back from the store.
143380×
            let t = std::time::Instant::now();
143480×
            let (skeleton, texts) = self.store.get_change_lazy(&hash)?;
143580×
            base.add_known_mixed(hash, change, skeleton, texts);
143680×
            crate::timing::add("store: add to state", t.elapsed());
143780×
            known.imported.insert(gc.sha.clone(), hash);
143880×
            if waiting.get(gc.sha.as_str()).is_some_and(|n| *n > 0) {
1439
                // A kept state holds its graphs until its children come:
1440
                // past a few thousand, they go back into its history.
1441
                // Past a few thousand graphs, or past a budget of atoms: a
1442
                // long-lived file's graph keeps every line it ever had, so a
1443
                // handful of them can hold gigabytes (#25).
144446×
                if base.built_count() > KEPT_GRAPHS || base.built_atoms() > kept_atoms {
14450×
                    shed += base.shed_graphs(|b| self.store_blob(b))?;
144646×
                }
144746×
                kept.insert(gc.sha.clone(), (base, tree));
144846×
                kept_order.push_back(gc.sha.clone());
144946×
                if kept_order.len() > max_kept {
14500×
                    if let Some(old) = kept_order.pop_front() {
14510×
                        kept.remove(&old);
14520×
                    }
145346×
                }
145434×
            }
145580×
            last = Some(gc.sha.clone());
145680×
            report.commits += 1;
145780×
            lap(5, &mut clock);
145880×
            done += 1;
145980×
            if done.is_multiple_of(checkpoint_every) {
14600×
                self.checkpoint_import(&heads_now, done)?;
146180×
            }
1462
            // A merge renders thousands of files on many threads and frees
1463
            // it all; the allocator keeps those pages unless asked to give
1464
            // them back, and the next merge piles on top (#25).
146580×
            let before_trim = if trace { rss_mb() } else { 0 };
146680×
            if gc.parents.len() > 1 {
146723×
                crate::alloc_stats::release_freed_memory();
146857×
            }
146980×
            if trace {
14700×
                let built = kept
14710×
                    .get(gc.sha.as_str())
14720×
                    .map(|(st, _)| st.built_count())
14730×
                    .unwrap_or(0);
14740×
                let (edits, most) = kept
14750×
                    .get(gc.sha.as_str())
14760×
                    .map(|(st, _)| st.deferred_edit_count())
14770×
                    .unwrap_or((0, 0));
14780×
                let atoms = kept
14790×
                    .get(gc.sha.as_str())
14800×
                    .map(|(st, _)| st.built_atoms())
14810×
                    .unwrap_or(0);
14820×
                eprintln!(
1483
                    "trace {done} {} parents={} base={base_how} merged={merged} built={built} atoms={atoms} kept={} shed={shed} rss_mb={} before_trim_mb={before_trim} deferred_edits={edits} most={most} need={need_n} merge={merge_how} ms={}",
14840×
                    &gc.sha[..12.min(gc.sha.len())],
14850×
                    gc.parents.len(),
14860×
                    kept.len(),
14870×
                    rss_mb(),
14880×
                    commit_started.elapsed().as_millis()
1489
                );
149080×
            }
1491
        }
149284×
        for (name, d) in [
149314×
            "git import: bases",
149414×
            "git import: tree",
149514×
            "git import: render base",
149614×
            "git import: read blobs",
149714×
            "git import: build change",
149814×
            "git import: store change",
149914×
        ]
150014×
        .iter()
150114×
        .zip(spent)
150284×
        {
150384×
            crate::timing::record(name, d);
150484×
        }
150514×
        crate::timing::report_totals();
150614×
        if crate::timing::enabled() {
15070×
            eprintln!(
15080×
                "timing git import: {rebuilt} bases rebuilt from history, {shed} graphs shed"
15090×
            );
151014×
        }
1511
1512
        // The bookmark: heads of the old state plus every new change.
151314×
        report.heads = old_heads.len();
151414×
        if last.is_none() {
15151×
            return Ok(report);
151613×
        }
151713×
        let heads: Vec<ChangeHash> = heads_now.into_iter().collect();
151813×
        report.heads = heads.len();
151913×
        let (op_ref, op) = self.head_op()?;
152013×
        let mut view = op.view.clone();
152113×
        let old_state_ref = view.bookmarks.get(&self.config.bookmark).copied();
152213×
        let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?;
152313×
        view.bookmarks
152413×
            .insert(self.config.bookmark.clone(), state_ref);
152513×
        self.record_op(&[op_ref], command, view)?;
1526
        // The new state equals the tip's tree when the old one was itself a
1527
        // Git commit's tree (fetched or exported), or there was none.
152813×
        let old_known = old_heads.is_empty()
15291×
            || old_state_ref.is_some_and(|s| known.exported.values().any(|e| *e == s));
153013×
        if let (Some(tip), true) = (commits.last(), old_known) {
153113×
            self.record_git_state(git_dir, &state_ref, &tip.sha)?;
15320×
        }
1533
        // The working copy follows when it was clean on the old heads.
153413×
        if old_heads.is_empty() {
153512×
            self.checkout(&heads, true, &format!("{command} (checkout)"))?;
1536
        } else {
15371×
            let snap = self.snapshot(command)?;
15381×
            let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
15391×
            let mut base = snap.base.maximal(&snap.wc.deps);
15401×
            base.sort();
15411×
            let mut sorted_old = old_heads.clone();
15421×
            sorted_old.sort();
1543
            // An empty base too: an import that stopped after a checkpoint
1544
            // never checked anything out, and resuming it should.
15451×
            if clean && (base == sorted_old || base.is_empty()) {
15461×
                self.checkout(&heads, false, &format!("{command} (checkout)"))?;
15470×
            }
1548
        }
154913×
        Ok(report)
155014×
    }
1551
1552
    /// Add to `st` the changes of every Git commit `rev-list args` lists:
1553
    /// an imported commit's change, or the whole state an exported commit
1554
    /// was made from (which may hold changes Git never saw).
15551×
    fn add_git_ancestors(
15561×
        &self,
15571×
        st: &mut State,
15581×
        git_dir: &Path,
15591×
        args: &[&str],
15601×
        known: &GitKnown,
15611×
    ) -> Result<()> {
15621×
        self.add_git_ancestors_touching(st, git_dir, args, known, &mut HashSet::new())
15631×
    }
1564
1565
    /// `add_git_ancestors`, also collecting in `touched` every node the
1566
    /// changes it adds edit, place or remove.
156724×
    fn add_git_ancestors_touching(
156824×
        &self,
156924×
        st: &mut State,
157024×
        git_dir: &Path,
157124×
        args: &[&str],
157224×
        known: &GitKnown,
157324×
        touched: &mut HashSet<NodeId>,
157424×
    ) -> Result<()> {
157524×
        let mut full = vec!["rev-list"];
157624×
        full.extend(args);
157724×
        let out = git(git_dir, &full)?;
157831×
        let mut add = |st: &mut State, h: &ChangeHash| -> Result<()> {
157931×
            if st.contains(h) {
15802×
                return Ok(());
158129×
            }
158229×
            if !st.is_history_only() {
15830×
                let c = self.store.get_change(h)?;
15840×
                touched.extend(c.edits.iter().map(|e| e.node()));
15850×
                touched.extend(c.tree_ops.iter().map(|o| o.node()));
15860×
                st.add_unchecked(c);
15870×
                return Ok(());
158829×
            }
1589
            // History only: the text of an edit waits unread unless its
1590
            // file's graph is built.
159129×
            let (skeleton, texts) = self.store.get_change_lazy(h)?;
159232×
            touched.extend(skeleton.edits.iter().map(|e| e.node()));
159329×
            touched.extend(skeleton.tree_ops.iter().map(|o| o.node()));
159429×
            let built = skeleton
159529×
                .edits
159629×
                .iter()
159732×
                .any(|e| !matches!(e, Edit::Replace { .. }) && st.is_built(&e.node()));
159829×
            if built {
15994×
                let full = self.store.get_change(h)?;
16004×
                st.add_known_mixed(*h, full, skeleton, texts);
160125×
            } else {
160225×
                st.add_known(*h, skeleton, texts);
160325×
            }
160429×
            Ok(())
160531×
        };
160629×
        for sha in out.lines() {
160729×
            if let Some(h) = known.imported.get(sha) {
160828×
                add(st, h)?;
16091×
            } else if let Some(state_ref) = known.exported.get(sha) {
16101×
                let heads = self.store.get_state(state_ref)?.heads;
16113×
                for h in self.negotiate(&[], &heads)? {
16123×
                    add(st, &h)?;
1613
                }
16140×
            }
1615
        }
161624×
        Ok(())
161724×
    }
1618
1619
    /// Recover an import that stopped before it moved the bookmark: its
1620
    /// unfinished pack indexed (`Store::adopt_unfinished_packs`), then every
1621
    /// change in the store that carries a `Git-Commit:` trailer, its heads
1622
    /// (those no other such change depends on) made the bookmark's heads,
1623
    /// so `oxy git fetch` carries on from there. Reads every object once.
1624
    /// Returns how many imported changes it found and the heads.
16250×
    pub fn recover_git_import(&self, progress: &dyn Fn(&str)) -> Result<(usize, usize)> {
1626
        // A stopped import's objects are in a pack it never indexed.
1627
        // OXY_RECOVER_MIN_AGE (seconds) overrides the five minutes it must
1628
        // have been left alone, for testing.
16290×
        let min_age = std::env::var("OXY_RECOVER_MIN_AGE")
16300×
            .ok()
16310×
            .and_then(|v| v.parse().ok())
16320×
            .unwrap_or(300);
16330×
        self.store
16340×
            .adopt_unfinished_packs(std::time::Duration::from_secs(min_age), progress)?;
16350×
        let all = self
16360×
            .store
16370×
            .list()
16380×
            .map_err(|e| RepoError::Msg(format!("listing the store: {e}")))?;
16390×
        let total = all.len();
16400×
        let mut found: HashMap<ChangeHash, Vec<ChangeHash>> = HashMap::new();
16410×
        for (i, r) in all.iter().enumerate() {
16420×
            if i.is_multiple_of(100_000) {
16430×
                progress(&format!(
16440×
                    "{i} of {total} objects read, {} imported changes",
16450×
                    found.len()
16460×
                ));
16470×
            }
16480×
            let Ok(meta) = self.store.get_change_meta(r) else {
16490×
                continue;
1650
            };
16510×
            if meta.message.lines().any(|l| l.starts_with("Git-Commit: ")) {
16520×
                found.insert(*r, meta.deps.clone());
16530×
            }
1654
        }
16550×
        if found.is_empty() {
16560×
            return fail("no imported Git commit's change in this store");
16570×
        }
16580×
        let depended: HashSet<ChangeHash> = found.values().flatten().copied().collect();
16590×
        let mut heads: Vec<ChangeHash> = found
16600×
            .keys()
16610×
            .filter(|h| !depended.contains(h))
16620×
            .copied()
16630×
            .collect();
1664
        // The bookmark's present heads stay: a recovery only adds.
16650×
        heads.extend(self.bookmark_heads().unwrap_or_default());
16660×
        let st = self.load_state_history(&heads)?;
16670×
        let mut heads = st.maximal(&heads);
16680×
        heads.sort();
16690×
        heads.dedup();
16700×
        let (op_ref, op) = self.head_op()?;
16710×
        let mut view = op.view.clone();
16720×
        let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?;
16730×
        view.bookmarks
16740×
            .insert(self.config.bookmark.clone(), state_ref);
16750×
        self.record_op(&[op_ref], "oxy git recover", view)?;
16760×
        Ok((found.len(), heads.len()))
16770×
    }
1678
1679
    /// Point the bookmark at an import's heads so far (see
1680
    /// `CHECKPOINT_COMMITS`); the working copy is left alone until the end.
16810×
    fn checkpoint_import(&self, heads: &BTreeSet<ChangeHash>, done: usize) -> Result<()> {
16820×
        let heads: Vec<ChangeHash> = heads.iter().copied().collect();
16830×
        let (op_ref, op) = self.head_op()?;
16840×
        let mut view = op.view.clone();
16850×
        let state_ref = self.store.put_state(&StateObject::new(heads))?;
16860×
        view.bookmarks
16870×
            .insert(self.config.bookmark.clone(), state_ref);
16880×
        self.record_op(
16890×
            &[op_ref],
16900×
            &format!("oxy git import (checkpoint after {done} commits)"),
16910×
            view,
16920×
        )?;
16930×
        Ok(())
16940×
    }
1695
1696
    /// Commits this repository can relate to Oxygen history: imported
1697
    /// ones by change, exported ones (through `git_dir`'s marks) by state.
169818×
    fn git_known(&self, git_dir: &Path) -> Result<GitKnown> {
169918×
        let imported = self.imported_commits()?;
170018×
        let marks_dir = self.git_marks_dir(git_dir)?;
170118×
        let oxy_marks = Self::read_oxy_marks(&marks_dir.join("oxy-marks"))?;
170218×
        let mut sha_of_mark: HashMap<u64, String> = HashMap::new();
170318×
        if let Ok(text) = fs::read_to_string(marks_dir.join("git-marks")) {
17044×
            for line in text.lines() {
17054×
                if let Some((m, sha)) = line.split_once(' ') {
17064×
                    if let Ok(m) = m.trim_start_matches(':').parse::<u64>() {
17074×
                        sha_of_mark.insert(m, sha.trim().to_string());
17084×
                    }
17090×
                }
1710
            }
171114×
        }
171218×
        let mut exported = HashMap::new();
171318×
        for (state_ref, m) in oxy_marks {
17144×
            if let Some(sha) = sha_of_mark.get(&m) {
17154×
                exported.insert(sha.clone(), state_ref);
17164×
            }
1717
        }
171818×
        for (state_ref, sha) in Self::read_git_states(&marks_dir.join("git-states"))? {
17197×
            exported.entry(sha).or_insert(state_ref);
17207×
        }
172118×
        Ok(GitKnown { imported, exported })
172218×
    }
1723
17246×
    fn git_author(&self, c: &Change) -> String {
17256×
        match self.store.get_identity(&c.author) {
17266×
            Ok(id) if !id.name.is_empty() => format!(
1727
                "{} <{}>",
17286×
                id.name.replace(['<', '>', '\n'], " "),
17296×
                id.email.replace(['<', '>', '\n', ' '], "")
1730
            ),
17310×
            _ => "oxy <oxy@localhost>".to_string(),
1732
        }
17336×
    }
1734
1735
    #[allow(clippy::too_many_arguments)]
17369×
    fn write_commit(
17379×
        &self,
17389×
        out: &mut Vec<u8>,
17399×
        reference: &str,
17409×
        mark: u64,
17419×
        author: &str,
17429×
        secs: u64,
17439×
        message: &str,
17449×
        parents: &[String],
17459×
        snapshot: &Snapshot,
17469×
    ) -> Result<()> {
17479×
        out.extend_from_slice(format!("commit {reference}\nmark :{mark}\nauthor {author} {secs} +0000\ncommitter {author} {secs} +0000\ndata {}\n{message}\n", message.len()).as_bytes());
17489×
        for (i, p) in parents.iter().enumerate() {
17495×
            out.extend_from_slice(
17505×
                format!("{} {p}\n", if i == 0 { "from" } else { "merge" }).as_bytes(),
1751
            );
1752
        }
17539×
        out.extend_from_slice(b"deleteall\n");
17549×
        let has_gitmodules = snapshot.files.contains_key(".gitmodules");
175538×
        for (path, f) in &snapshot.files {
175638×
            if path == ".oxymodules" {
1757
                // Git reads `.gitmodules`; derive it unless the tree has one.
17581×
                if !has_gitmodules {
17590×
                    if let FileContent::Text { bytes, .. } = &f.content {
17600×
                        let gm = crate::subrepo::Modules::parse(&String::from_utf8_lossy(bytes))
17610×
                            .to_gitmodules();
17620×
                        if !gm.is_empty() {
17630×
                            out.extend_from_slice(
17640×
                                format!("M 100644 inline .gitmodules\ndata {}\n", gm.len())
17650×
                                    .as_bytes(),
17660×
                            );
17670×
                            out.extend_from_slice(gm.as_bytes());
17680×
                            out.push(b'\n');
17690×
                        }
17700×
                    }
17711×
                }
17721×
                continue;
177337×
            }
177437×
            if f.kind == NodeKind::Subrepo {
17751×
                if let FileContent::Opaque { blobs } = &f.content {
17761×
                    if let Some(b) = blobs.first() {
17771×
                        match crate::subrepo::Pin::decode(&self.read_blob(b)?)? {
17781×
                            crate::subrepo::Pin::Git { sha, .. } => {
17791×
                                out.extend_from_slice(format!("M 160000 {sha} {path}\n").as_bytes())
1780
                            }
17810×
                            crate::subrepo::Pin::Oxy { .. } => self
17820×
                                .git_export_skipped
17830×
                                .set(self.git_export_skipped.get() + 1),
1784
                        }
17850×
                    }
17860×
                }
17871×
                continue;
178836×
            }
178936×
            let (mode, bytes): (&str, Vec<u8>) = match (&f.kind, &f.content) {
17909×
                (NodeKind::Dir, _) => continue,
17910×
                (NodeKind::Symlink, FileContent::Opaque { blobs }) => (
17920×
                    "120000",
17930×
                    blobs
17940×
                        .first()
17950×
                        .map(|b| self.read_blob(b))
17960×
                        .transpose()?
17970×
                        .unwrap_or_default(),
1798
                ),
179924×
                (_, FileContent::Text { bytes, .. }) => {
180024×
                    (if f.mode.exec { "100755" } else { "100644" }, bytes.clone())
1801
                }
18023×
                (_, FileContent::Opaque { blobs }) => (
18033×
                    if f.mode.exec { "100755" } else { "100644" },
18043×
                    blobs
18053×
                        .first()
18063×
                        .map(|b| self.read_blob(b))
18073×
                        .transpose()?
18083×
                        .unwrap_or_default(),
1809
                ),
18100×
                (_, FileContent::Dir) => continue,
1811
            };
181227×
            out.extend_from_slice(
181327×
                format!("M {mode} inline {path}\ndata {}\n", bytes.len()).as_bytes(),
1814
            );
181527×
            out.extend_from_slice(&bytes);
181627×
            out.push(b'\n');
1817
        }
18189×
        out.push(b'\n');
18199×
        Ok(())
18209×
    }
1821
}
1822
1823
/// What `verify_git_import` found (spec §23: import must reproduce every tree).
1824
#[derive(Debug, Clone, Default, PartialEq, Eq)]
1825
pub struct GitVerifyReport {
1826
    pub commits: usize,
1827
    /// Commits followed but not rebuilt (all but the last few, when asked).
1828
    pub skipped: usize,
1829
    pub files_compared: usize,
1830
    /// `(commit, what differs)`; empty when every commit's tree is reproduced.
1831
    pub mismatches: Vec<(String, String)>,
1832
    /// Commits of the branch after the last imported one: an import not
1833
    /// finished yet. Not checked.
1834
    pub not_yet: usize,
1835
}
1836
1837
impl Repo {
1838
    /// Check an import against its Git source: for every commit of
1839
    /// `branch`, the state formed by the changes imported for it and all
1840
    /// its ancestors must materialize to exactly that commit's tree, byte
1841
    /// for byte, with the same executable bits, symlink targets and
1842
    /// submodule pins. Reads only; stops listing after 50 differences.
18438×
    pub fn verify_git_import(&self, git_dir: &Path, branch: &str) -> Result<GitVerifyReport> {
18448×
        self.verify_git_import_last(git_dir, branch, None, 50)
18458×
    }
1846
1847
    /// [`Repo::verify_git_import`] of the last `last` commits only (all
1848
    /// when `None`): earlier ones are only followed, not rebuilt, for
1849
    /// looking again at where an import went wrong.
1850
    /// It stops after `max` differences (0: never), so one that repeats
1851
    /// through history does not hide the commits after it.
18528×
    pub fn verify_git_import_last(
18538×
        &self,
18548×
        git_dir: &Path,
18558×
        branch: &str,
18568×
        last: Option<usize>,
18578×
        max: usize,
18588×
    ) -> Result<GitVerifyReport> {
18598×
        let mut rep = GitVerifyReport::default();
18608×
        let mut commits = list_commits(git_dir, branch)?;
18618×
        let imported = self.imported_commits()?;
1862
        // An import still going (or stopped) has reached part of the
1863
        // branch: what follows its last imported commit is not checked,
1864
        // only counted, and --last counts back from there (oxygen#81).
18658×
        if let Some(end) = commits.iter().rposition(|c| imported.contains_key(&c.sha)) {
18668×
            let rest = commits.len() - end - 1;
18678×
            if rest > 0 {
18680×
                commits.truncate(end + 1);
18698×
            }
18708×
            rep.not_yet = rest;
18710×
        }
18728×
        let mut cat = CatFile::start(git_dir)?;
1873
        // Changes of each commit and all its ancestors, built up in topological order.
18748×
        let mut closure: HashMap<String, Arc<BTreeSet<ChangeHash>>> = HashMap::new();
1875
        // Graphs (by key) and blobs already found equal to a Git blob.
18768×
        let mut verified_graphs: HashMap<[u8; 32], String> = HashMap::new();
18778×
        let mut verified_blobs: HashMap<ObjectRef, String> = HashMap::new();
1878
        // The state of a commit whose first-parent children are still to
1879
        // come, so a child adds its own changes instead of loading all its
1880
        // history again (as the import does).
18818×
        let mut waiting: HashMap<&str, usize> = HashMap::new();
1882
        // Children still to come per commit, through any parent: a commit's
1883
        // closure is dropped after its last child, so only the open
1884
        // frontier is held. Kept for every commit, the closures grew with
1885
        // the square of the history and VS Code's verify ran out of memory
1886
        // (oxygen#81).
18878×
        let mut children: HashMap<&str, usize> = HashMap::new();
188855×
        for c in &commits {
188955×
            if let Some(p1) = c.parents.first() {
189045×
                *waiting.entry(p1.as_str()).or_default() += 1;
189145×
            }
189262×
            for p in &c.parents {
189362×
                *children.entry(p.as_str()).or_default() += 1;
189462×
            }
1895
        }
18968×
        let mut kept: HashMap<String, State> = HashMap::new();
18978×
        let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
189855×
        for c in &commits {
189955×
            rep.commits += 1;
1900
            // The first parent's closure, taken over when this is its last
1901
            // child (no copy), then the other parents' added.
190255×
            let mut set: BTreeSet<ChangeHash> = BTreeSet::new();
190362×
            for (i, p) in c.parents.iter().enumerate() {
190462×
                let left = children.get_mut(p.as_str()).map(|n| {
190562×
                    *n -= 1;
190662×
                    *n
190762×
                });
190862×
                let ps = if left == Some(0) {
190947×
                    closure.remove(p)
1910
                } else {
191115×
                    closure.get(p).cloned()
1912
                };
191362×
                if let Some(ps) = ps {
191462×
                    if i == 0 && set.is_empty() {
191545×
                        set = Arc::try_unwrap(ps).unwrap_or_else(|a| (*a).clone());
191617×
                    } else {
191717×
                        set.extend(ps.iter().copied());
191817×
                    }
19190×
                }
1920
            }
192155×
            match imported.get(&c.sha) {
192255×
                Some(h) => {
192355×
                    set.insert(*h);
192455×
                }
19250×
                None => {
19260×
                    rep.mismatches.push((
19270×
                        c.sha.clone(),
19280×
                        "no imported change carries this commit".into(),
19290×
                    ));
19300×
                }
1931
            }
193255×
            let set = Arc::new(set);
193355×
            if children.get(c.sha.as_str()).is_some_and(|n| *n > 0) {
193447×
                closure.insert(c.sha.clone(), set.clone());
193547×
            }
193655×
            if max > 0 && rep.mismatches.len() >= max {
19370×
                break;
193855×
            }
193955×
            if last.is_some_and(|n| rep.commits + n <= commits.len()) {
19400×
                rep.skipped += 1;
19410×
                continue;
194255×
            }
194355×
            let heads: Vec<ChangeHash> = set.iter().copied().collect();
1944
            // History only: a file whose graph (by key) or blob was already
1945
            // found equal to this Git blob is not built, rendered or read
1946
            // again; the rest are.
194755×
            let from_parent = c.parents.first().and_then(|p1| {
194845×
                let left = waiting.get_mut(p1.as_str()).map(|n| {
194945×
                    *n -= 1;
195045×
                    *n
195145×
                });
195245×
                if left == Some(0) {
195347×
                    kept_order.retain(|k| k != p1);
195432×
                    kept.remove(p1.as_str())
1955
                } else {
195613×
                    kept.get(p1.as_str()).cloned()
1957
                }
195845×
            });
195955×
            let state = match from_parent {
196045×
                Some(mut st) => {
1961
                    // The changes the parent's state lacks. The state never
1962
                    // builds a graph (they are built beside it), so they
1963
                    // are read without their text.
1964202×
                    for h in set.iter() {
1965202×
                        if !st.contains(h) {
196665×
                            let (change, texts) = self.store.get_change_lazy(h)?;
196765×
                            st.add_known(*h, change, texts);
1968137×
                        }
1969
                    }
197045×
                    st
1971
                }
197210×
                None => self.load_state_history(&heads)?,
1973
            };
197455×
            let view = state.tree().view();
197555×
            let tree = list_tree(git_dir, &c.sha)?;
197655×
            let short = &c.sha[..12.min(c.sha.len())];
197755×
            let mut differ = |what: String| rep.mismatches.push((c.sha.clone(), what));
197855×
            let mut conflicted: Vec<String> =
197955×
                view.conflicts.iter().map(|c| format!("{c:?}")).collect();
198055×
            let mut need: Vec<NodeId> = Vec::new();
1981131×
            for (path, (mode, sha)) in &tree {
1982131×
                let Some(e) = view.entries.get(path) else {
19830×
                    continue;
1984
                };
1985131×
                if mode != "160000"
1986131×
                    && mode != "120000"
1987131×
                    && e.kind == NodeKind::File
1988131×
                    && !e.mode.opaque
1989
                {
1990120×
                    let known = state
1991120×
                        .graph_key(&e.node)
1992120×
                        .is_some_and(|k| verified_graphs.get(&k) == Some(sha));
1993120×
                    if !known {
199446×
                        need.push(e.node);
199574×
                    }
199611×
                }
1997
            }
1998
            // A graph's key is known only before it is built.
199955×
            let keys: HashMap<NodeId, [u8; 32]> = need
200055×
                .iter()
200155×
                .filter_map(|n| Some((*n, state.graph_key(n)?)))
200255×
                .collect();
200355×
            let need_set: HashSet<NodeId> = need.iter().copied().collect();
200455×
            let graphs = self.build_graphs_detached(&state, &need)?;
2005131×
            for (path, (mode, sha)) in &tree {
2006131×
                let Some(e) = view.entries.get(path) else {
20070×
                    differ(format!("{short}: {path} is missing"));
20080×
                    continue;
2009
                };
2010131×
                rep.files_compared += 1;
2011131×
                match mode.as_str() {
2012131×
                    "160000" => {
20130×
                        let blobs = state.live_blobs(&e.node);
20140×
                        match (e.kind, blobs.as_slice()) {
20150×
                            (NodeKind::Subrepo, [b]) => {
20160×
                                let pin = crate::subrepo::Pin::decode(&self.read_blob(b)?)?;
20170×
                                if !matches!(&pin, crate::subrepo::Pin::Git { sha: s, .. } if s == sha)
20180×
                                {
20190×
                                    differ(format!("{short}: {path} pins another commit"));
20200×
                                }
2021
                            }
20220×
                            (kind, _) => differ(format!(
20230×
                                "{short}: {path} is not one submodule pin: {kind:?} with {} pins",
20240×
                                blobs.len()
20250×
                            )),
2026
                        }
2027
                    }
2028131×
                    "120000" => {
20290×
                        let target = cat.read(sha)?;
20300×
                        let blobs = state.live_blobs(&e.node);
20310×
                        match (e.kind, blobs.as_slice()) {
20320×
                            (NodeKind::Symlink, [b]) if self.read_blob(b)? == target => {}
20330×
                            _ => differ(format!("{short}: symlink {path} differs")),
2034
                        }
2035
                    }
2036
                    _ => {
2037131×
                        if e.mode.exec != (mode == "100755") {
20380×
                            differ(format!("{short}: {path} executable bit differs"));
2039131×
                        }
2040131×
                        if e.kind == NodeKind::File && !e.mode.opaque {
2041120×
                            if !need_set.contains(&e.node) {
2042
                                // Known equal: skipped when the graphs were chosen.
204374×
                                continue;
204446×
                            }
204546×
                            let (have, m) = match graphs.get(&e.node) {
204646×
                                Some(g) => state.render_graph(g),
20470×
                                None => (Vec::new(), Default::default()),
2048
                            };
204946×
                            if m.is_conflicted() {
2050
                                // Who conflicts: a copy of the state with this graph built.
20510×
                                let mut detail = state.clone();
20520×
                                self.load_graphs_of(&mut detail, &[e.node])?;
20530×
                                let who: Vec<String> = detail
20540×
                                    .conflict_regions(&e.node)
20550×
                                    .contributors
20560×
                                    .iter()
20570×
                                    .filter_map(|h| state.get(h))
20580×
                                    .map(|m| {
20590×
                                        let sha = m
20600×
                                            .message
20610×
                                            .lines()
20620×
                                            .rev()
20630×
                                            .find_map(|l| l.strip_prefix("Git-Commit: "))
20640×
                                            .unwrap_or("?");
20650×
                                        format!(
2066
                                            "{} {:?}",
20670×
                                            &sha[..12.min(sha.len())],
20680×
                                            m.message.lines().next().unwrap_or("")
2069
                                        )
20700×
                                    })
20710×
                                    .collect();
20720×
                                conflicted.push(format!("{path} between {}", who.join(" and ")));
207346×
                            }
207446×
                            let want = cat.read(sha)?;
207546×
                            if have != want {
20760×
                                differ(format!(
20770×
                                    "{short}: {path} differs: {}",
20780×
                                    first_difference(&have, &want)
20790×
                                ));
208046×
                            } else if !m.is_conflicted() {
208146×
                                if let Some(k) = keys.get(&e.node) {
208246×
                                    verified_graphs.insert(*k, sha.clone());
208346×
                                }
20840×
                            }
208546×
                            continue;
208611×
                        }
208711×
                        let blobs = state.live_blobs(&e.node);
208811×
                        let [b] = blobs.as_slice() else {
20890×
                            differ(format!("{short}: {path} is not one file"));
20900×
                            continue;
2091
                        };
209211×
                        if verified_blobs.get(b) == Some(sha) {
20936×
                            continue;
20945×
                        }
20955×
                        if self.read_blob(b)? != cat.read(sha)? {
20960×
                            differ(format!("{short}: {path} differs"));
20975×
                        } else {
20985×
                            verified_blobs.insert(*b, sha.clone());
20995×
                        }
2100
                    }
2101
                }
2102
            }
210355×
            if !conflicted.is_empty() {
21040×
                differ(format!(
21050×
                    "{short}: the state is conflicted: {}",
21060×
                    conflicted.join("; ")
21070×
                ));
210855×
            }
2109
            // Import records submodules in `.oxymodules` as well (spec §23),
2110
            // and export leaves it out again.
2111131×
            let has_submodules = tree.values().any(|(m, _)| m == "160000");
2112166×
            for (path, e) in view.entries.iter() {
2113166×
                if e.kind != NodeKind::Dir
2114131×
                    && !tree.contains_key(path)
21150×
                    && !(has_submodules && path == ".oxymodules")
21160×
                {
21170×
                    differ(format!("{short}: {path} is extra"));
2118166×
                }
2119
            }
212055×
            if waiting.get(c.sha.as_str()).is_some_and(|n| *n > 0) {
212132×
                kept.insert(c.sha.clone(), state);
212232×
                kept_order.push_back(c.sha.clone());
212332×
                if kept_order.len() > 8 {
21240×
                    if let Some(old) = kept_order.pop_front() {
21250×
                        kept.remove(&old);
21260×
                    }
212732×
                }
212823×
            }
212955×
            if max > 0 && rep.mismatches.len() >= max {
21300×
                break;
213155×
            }
2132
        }
21338×
        Ok(rep)
21348×
    }
2135
}
2136
2137
/// Where two versions of a file first part, for a verification report.
21380×
fn first_difference(have: &[u8], want: &[u8]) -> String {
21390×
    let (h, w): (Vec<&[u8]>, Vec<&[u8]>) = (
21400×
        have.split(|b| *b == b'\n').collect(),
21410×
        want.split(|b| *b == b'\n').collect(),
2142
    );
21430×
    let i = h
21440×
        .iter()
21450×
        .zip(&w)
21460×
        .position(|(a, b)| a != b)
21470×
        .unwrap_or(h.len().min(w.len()));
21480×
    let show = |l: Option<&&[u8]>| {
21490×
        l.map(|l| String::from_utf8_lossy(&l[..l.len().min(60)]).into_owned())
21500×
            .unwrap_or_else(|| "(end)".into())
21510×
    };
21520×
    format!(
2153
        "line {}: has {:?}, Git has {:?} ({} lines, Git has {})",
21540×
        i + 1,
21550×
        show(h.get(i)),
21560×
        show(w.get(i)),
21570×
        h.len(),
21580×
        w.len()
2159
    )
21600×
}