OXYGENHUB
# 72215f29df4c oxygen / crates / oxy-core / src / repo / git.rs
55.9 KiB · Rust · last changed Verify a Git import from each parent's state 1 month ago source blame history raw
79ab8528 Andy McCall ↶ 1 
//! Git bridge (spec §23), export side: each state the current bookmark
2 
//! passed through becomes a Git commit through `git fast-import`, with the
3 
//! state's full tree and the previous state's commit as parent. A state
4 
//! that added exactly one change carries that change's message, author,
5 
//! and `Change-Id`/`Oxy-Hash` trailers; every commit carries `Oxy-State`.
6 
//!
7 
//! Marks make re-exports incremental: `.oxy/git/<target>/git-marks` is
8 
//! git's own marks file and `oxy-marks` maps state refs to marks.
9
10 
use super::*;
11 
use std::process::{Command, Stdio};
62794ecc Andy McCall ↶ 12 
use std::sync::Arc;
79ab8528 Andy McCall ↶ 13
14 
#[derive(Debug, Clone, Default, PartialEq, Eq)]
c3069537 Andy McCall ↶ 15 
pub struct GitImportReport {
16 
    pub commits: usize,
17 
    pub merges: usize,
4dee277c Andy McCall ↶ 18 
    /// Gitlink entries imported as subrepo nodes with Git pins.
19 
    pub submodules: usize,
c3069537 Andy McCall ↶ 20 
    pub heads: usize,
21 
}
22
23 
/// A running `git cat-file --batch`, for reading many blobs cheaply.
24 
struct CatFile {
25 
    child: std::process::Child,
26 
    stdin: std::process::ChildStdin,
27 
    stdout: std::io::BufReader<std::process::ChildStdout>,
28 
}
29
30 
impl CatFile {
31 
    fn start(git_dir: &Path) -> Result<CatFile> {
32 
        let mut child = Command::new("git")
33 
            .arg("-C")
34 
            .arg(git_dir)
35 
            .args(["cat-file", "--batch"])
36 
            .stdin(Stdio::piped())
37 
            .stdout(Stdio::piped())
38 
            .stderr(Stdio::null())
39 
            .spawn()
40 
            .map_err(|e| RepoError::Msg(format!("running git cat-file: {e}")))?;
41 
        let stdin = child.stdin.take().unwrap();
42 
        let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
75167672 Andy McCall 43 
        Ok(CatFile {
44 
            child,
45 
            stdin,
46 
            stdout,
47 
        })
c3069537 Andy McCall ↶ 48 
    }
49
50 
    fn read(&mut self, sha: &str) -> Result<Vec<u8>> {
51 
        use std::io::{BufRead, Read};
52 
        writeln!(self.stdin, "{sha}")?;
53 
        self.stdin.flush()?;
54 
        let mut header = String::new();
55 
        self.stdout.read_line(&mut header)?;
56 
        let parts: Vec<&str> = header.trim_end().split(' ').collect();
57 
        if parts.len() != 3 {
58 
            return fail(format!("git cat-file: object {sha} not found"));
59 
        }
75167672 Andy McCall 60 
        let size: usize = parts[2]
61 
            .parse()
62 
            .map_err(|_| RepoError::Msg("git cat-file: bad size".into()))?;
c3069537 Andy McCall ↶ 63 
        let mut body = vec![0u8; size];
64 
        self.stdout.read_exact(&mut body)?;
65 
        let mut nl = [0u8; 1];
66 
        self.stdout.read_exact(&mut nl)?;
67 
        Ok(body)
68 
    }
69 
}
70
71 
impl Drop for CatFile {
72 
    fn drop(&mut self) {
73 
        let _ = self.child.kill();
74 
        let _ = self.child.wait();
75 
    }
76 
}
77
78 
/// One commit as `git log` describes it.
79 
struct GitCommit {
80 
    sha: String,
81 
    parents: Vec<String>,
82 
    author_secs: u64,
83 
    author: String,
84 
    message: String,
85 
}
86
87 
fn list_commits(git_dir: &Path, branch: &str) -> Result<Vec<GitCommit>> {
88 
    let out = Command::new("git")
89 
        .arg("-C")
90 
        .arg(git_dir)
75167672 Andy McCall 91 
        .args([
92 
            "log",
93 
            "--reverse",
94 
            "--topo-order",
95 
            "--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e",
96 
            branch,
97 
        ])
c3069537 Andy McCall ↶ 98 
        .output()
99 
        .map_err(|e| RepoError::Msg(format!("running git log: {e}")))?;
100 
    if !out.status.success() {
75167672 Andy McCall 101 
        return fail(format!(
102 
            "git log {branch} failed: {}",
103 
            String::from_utf8_lossy(&out.stderr).trim()
104 
        ));
c3069537 Andy McCall ↶ 105 
    }
106 
    let text = String::from_utf8_lossy(&out.stdout);
107 
    let mut commits = Vec::new();
108 
    for record in text.split('\x1e') {
109 
        let record = record.trim_start_matches('\n');
110 
        if record.trim().is_empty() {
111 
            continue;
112 
        }
113 
        let f: Vec<&str> = record.splitn(5, '\0').collect();
114 
        if f.len() != 5 {
115 
            return fail("unexpected git log record");
116 
        }
117 
        commits.push(GitCommit {
118 
            sha: f[0].to_string(),
119 
            parents: f[1].split_whitespace().map(str::to_string).collect(),
120 
            author_secs: f[2].parse().unwrap_or(0),
121 
            author: f[3].to_string(),
122 
            message: f[4].trim_end().to_string(),
123 
        });
124 
    }
125 
    Ok(commits)
126 
}
127
128 
/// `path → (mode, blob sha)` for a commit's tree.
129 
fn list_tree(git_dir: &Path, sha: &str) -> Result<BTreeMap<String, (String, String)>> {
130 
    let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?;
131 
    let mut out = BTreeMap::new();
132 
    for entry in text.split('\0') {
133 
        if entry.is_empty() {
134 
            continue;
135 
        }
75167672 Andy McCall 136 
        let Some((meta, path)) = entry.split_once('\t') else {
137 
            continue;
138 
        };
c3069537 Andy McCall ↶ 139 
        let m: Vec<&str> = meta.split(' ').collect();
140 
        if m.len() == 3 {
141 
            out.insert(path.to_string(), (m[0].to_string(), m[2].to_string()));
142 
        }
143 
    }
144 
    Ok(out)
145 
}
146
147 
/// The ChangeId for a Git commit: its `Change-Id` trailer if it has a
148 
/// 32-hex one, else BLAKE3-128 of the commit hash (spec §23).
149 
fn change_id_for(sha: &str, message: &str) -> ChangeId {
150 
    for line in message.lines().rev() {
151 
        if let Some(v) = line.strip_prefix("Change-Id:") {
152 
            if let Some(id) = ChangeId::from_hex(v.trim()) {
153 
                return id;
154 
            }
155 
        }
156 
    }
157 
    let h = blake3::hash(sha.as_bytes());
158 
    ChangeId(h.as_bytes()[..16].try_into().unwrap())
159 
}
160
30ea9c1e Andy McCall ↶ 161 
/// Git commits with a known Oxygen counterpart.
162 
struct GitKnown {
163 
    /// Imported commit → its change.
164 
    imported: HashMap<String, ChangeHash>,
165 
    /// Exported commit (or a fetched tip) → the bookmark state it equals.
166 
    exported: HashMap<String, ObjectRef>,
167 
}
168
c3069537 Andy McCall ↶ 169 
#[derive(Debug, Clone, Default, PartialEq, Eq)]
79ab8528 Andy McCall ↶ 170 
pub struct GitExportReport {
171 
    pub exported: usize,
172 
    pub skipped: usize,
4dee277c Andy McCall ↶ 173 
    /// Subrepo nodes with Oxygen pins, which Git cannot represent.
174 
    pub skipped_subrepos: usize,
79ab8528 Andy McCall ↶ 175 
    pub branch: String,
176 
    pub heads: usize,
177 
}
178
179 
fn git(dir: &Path, args: &[&str]) -> Result<String> {
75167672 Andy McCall 180 
    let out = Command::new("git")
181 
        .arg("-C")
182 
        .arg(dir)
183 
        .args(args)
184 
        .output()
185 
        .map_err(|e| RepoError::Msg(format!("running git: {e}")))?;
79ab8528 Andy McCall ↶ 186 
    if !out.status.success() {
75167672 Andy McCall 187 
        return fail(format!(
188 
            "git {} failed: {}",
189 
            args.join(" "),
190 
            String::from_utf8_lossy(&out.stderr).trim()
191 
        ));
79ab8528 Andy McCall ↶ 192 
    }
193 
    Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
194 
}
195
196 
impl Repo {
197 
    fn git_marks_dir(&self, git_dir: &Path) -> Result<PathBuf> {
75167672 Andy McCall 198 
        let canonical = git_dir
199 
            .canonicalize()
200 
            .unwrap_or_else(|_| git_dir.to_path_buf());
79ab8528 Andy McCall ↶ 201 
        let key = hash_object(ObjectType::Chunk, canonical.to_string_lossy().as_bytes()).short(16);
202 
        let dir = self.oxy.join("git").join(key);
203 
        fs::create_dir_all(&dir)?;
204 
        Ok(dir)
205 
    }
206
30ea9c1e Andy McCall ↶ 207 
    /// `state ref → Git commit sha` for bookmark states that equal a Git
208 
    /// commit's tree (recorded by import and fetch).
209 
    fn read_git_states(path: &Path) -> Result<BTreeMap<ObjectRef, String>> {
210 
        let mut out = BTreeMap::new();
211 
        if let Ok(text) = fs::read_to_string(path) {
212 
            for line in text.lines() {
213 
                if let Some((h, sha)) = line.split_once(' ') {
214 
                    if let Some(h) = ObjectRef::from_hex(h) {
215 
                        out.insert(h, sha.trim().to_string());
216 
                    }
217 
                }
218 
            }
219 
        }
220 
        Ok(out)
221 
    }
222
223 
    fn record_git_state(&self, git_dir: &Path, state: &ObjectRef, sha: &str) -> Result<()> {
224 
        let path = self.git_marks_dir(git_dir)?.join("git-states");
225 
        let mut states = Self::read_git_states(&path)?;
226 
        states.insert(*state, sha.to_string());
75167672 Andy McCall 227 
        let text: String = states
228 
            .iter()
229 
            .map(|(h, s)| format!("{} {s}\n", h.to_hex()))
230 
            .collect();
30ea9c1e Andy McCall ↶ 231 
        write_atomic(&path, text.as_bytes())
232 
    }
233
234 
    /// The bare mirror this repository keeps for the Git remote `url`.
235 
    fn git_mirror(&self, url: &str) -> Result<PathBuf> {
236 
        let key = hash_object(ObjectType::Chunk, url.as_bytes()).short(16);
237 
        let dir = self.oxy.join("git").join(key).join("mirror.git");
238 
        if !dir.join("HEAD").exists() {
239 
            fs::create_dir_all(&dir)?;
240 
            git(&dir, &["init", "--bare", "-q"])?;
241 
        }
242 
        Ok(dir)
243 
    }
244
245 
    /// Git commits already imported into the bookmark, by sha.
7891e21c Andy McCall ↶ 246 
    pub(crate) fn imported_commits(&self) -> Result<HashMap<String, ChangeHash>> {
30ea9c1e Andy McCall ↶ 247 
        let mut map = HashMap::new();
248 
        for e in self.log()? {
249 
            if e.is_working_copy {
250 
                continue;
251 
            }
75167672 Andy McCall 252 
            if let Some(sha) = e
253 
                .change
254 
                .message
255 
                .lines()
256 
                .rev()
257 
                .find_map(|l| l.strip_prefix("Git-Commit: "))
258 
            {
30ea9c1e Andy McCall ↶ 259 
                map.insert(sha.trim().to_string(), e.hash);
260 
            }
261 
        }
262 
        Ok(map)
263 
    }
264
265 
    /// Bring the mirror's `branch` up to date with the Git remote `url`.
266 
    /// A remote without that branch yet leaves the mirror as it is.
267 
    fn git_mirror_fetch(&self, mirror: &Path, url: &str, branch: &str) -> Result<()> {
268 
        let spec = format!("+refs/heads/{branch}:refs/heads/{branch}");
269 
        match git(mirror, &["fetch", "-q", url, &spec]) {
270 
            Ok(_) => Ok(()),
271 
            Err(e) => {
272 
                let ls = git(mirror, &["ls-remote", "--heads", url, branch]).unwrap_or_default();
75167672 Andy McCall 273 
                if ls.trim().is_empty() {
274 
                    Ok(())
275 
                } else {
276 
                    Err(e)
277 
                }
30ea9c1e Andy McCall ↶ 278 
            }
279 
        }
280 
    }
281
282 
    /// Fetch `branch` of the Git repository at `url` (a path or a Git URL)
283 
    /// through a mirror and import the commits not yet in the bookmark
284 
    /// (spec §23). Works on a repository that already has history: the
285 
    /// new changes join the bookmark like a pull.
286 
    pub fn git_fetch(&self, url: &str, branch: &str) -> Result<GitImportReport> {
287 
        let mirror = self.git_mirror(url)?;
288 
        self.git_mirror_fetch(&mirror, url, branch)?;
75167672 Andy McCall 289 
        if git(
290 
            &mirror,
291 
            &[
292 
                "rev-parse",
293 
                "--verify",
294 
                "-q",
295 
                &format!("refs/heads/{branch}"),
296 
            ],
297 
        )
298 
        .is_err()
299 
        {
30ea9c1e Andy McCall ↶ 300 
            return fail(format!("{url} has no branch {branch:?}"));
301 
        }
302 
        self.import_git_commits(&mirror, branch, &format!("oxy git fetch {url} {branch}"))
303 
    }
304
305 
    /// Export the bookmark to `branch` of the Git repository at `url`
306 
    /// through the mirror and push it. The remote's tip must already have
307 
    /// been fetched (`git_fetch`), so the push fast-forwards.
308 
    pub fn git_push(&self, url: &str, branch: &str) -> Result<GitExportReport> {
309 
        let mirror = self.git_mirror(url)?;
310 
        self.git_mirror_fetch(&mirror, url, branch)?;
75167672 Andy McCall 311 
        if let Ok(tip) = git(
312 
            &mirror,
313 
            &[
314 
                "rev-parse",
315 
                "--verify",
316 
                "-q",
317 
                &format!("refs/heads/{branch}"),
318 
            ],
319 
        ) {
30ea9c1e Andy McCall ↶ 320 
            let known = self.git_known(&mirror)?;
321 
            if !known.imported.contains_key(&tip) && !known.exported.contains_key(&tip) {
75167672 Andy McCall 322 
                return fail(format!(
323 
                    "{url} {branch} has commits not yet fetched; run `oxy git fetch {url}` first"
324 
                ));
30ea9c1e Andy McCall ↶ 325 
            }
326 
        }
327 
        let report = self.git_export_as(&mirror, branch)?;
328 
        let spec = format!("refs/heads/{branch}:refs/heads/{branch}");
329 
        git(&mirror, &["push", "-q", url, &spec])?;
330 
        Ok(report)
331 
    }
332
79ab8528 Andy McCall ↶ 333 
    fn read_oxy_marks(path: &Path) -> Result<BTreeMap<ObjectRef, u64>> {
334 
        let mut out = BTreeMap::new();
335 
        if let Ok(text) = fs::read_to_string(path) {
336 
            for line in text.lines() {
337 
                if let Some((h, m)) = line.split_once(' ') {
338 
                    if let (Some(h), Ok(m)) = (ObjectRef::from_hex(h), m.parse()) {
339 
                        out.insert(h, m);
340 
                    }
341 
                }
342 
            }
343 
        }
344 
        Ok(out)
345 
    }
346
347 
    /// Export the current bookmark to the Git repository at `git_dir`,
348 
    /// creating a bare one if nothing is there. Each state the bookmark
349 
    /// passed through (per the operation log) becomes one commit whose tree
350 
    /// is that state, so the Git branch is a linear history of full trees.
351 
    pub fn git_export(&self, git_dir: &Path) -> Result<GitExportReport> {
30ea9c1e Andy McCall ↶ 352 
        let bookmark = self.config.bookmark.clone();
353 
        self.git_export_as(git_dir, &bookmark)
354 
    }
355
356 
    /// `git_export` to the Git branch `branch`. A state that came from a
357 
    /// Git commit (`oxy git fetch`) is not re-exported: later states chain
358 
    /// onto that commit, so a push back to the Git remote fast-forwards.
359 
    pub fn git_export_as(&self, git_dir: &Path, branch: &str) -> Result<GitExportReport> {
79ab8528 Andy McCall ↶ 360 
        if !git_dir.join("HEAD").exists() && !git_dir.join(".git").exists() {
361 
            fs::create_dir_all(git_dir)?;
362 
            git(git_dir, &["init", "--bare", "-q"])?;
363 
        }
364 
        let marks_dir = self.git_marks_dir(git_dir)?;
365 
        let git_marks = marks_dir.join("git-marks");
366 
        let oxy_marks_path = marks_dir.join("oxy-marks");
367 
        let mut marks = Self::read_oxy_marks(&oxy_marks_path)?;
30ea9c1e Andy McCall ↶ 368 
        let git_states = Self::read_git_states(&marks_dir.join("git-states"))?;
79ab8528 Andy McCall ↶ 369 
        let mut next_mark = marks.values().copied().max().unwrap_or(0) + 1;
370
371 
        let bookmark = self.config.bookmark.clone();
372 
        let mut stream: Vec<u8> = Vec::new();
75167672 Andy McCall 373 
        let mut report = GitExportReport {
374 
            branch: branch.to_string(),
375 
            heads: self.bookmark_heads()?.len(),
376 
            ..Default::default()
377 
        };
79ab8528 Andy McCall ↶ 378
379 
        // States along the bookmark, oldest first, skipping unchanged and pruned ones.
380 
        let mut ops = self.op_log()?;
381 
        ops.reverse();
382 
        let mut prev: Option<(ObjectRef, Vec<ChangeHash>)> = None;
30ea9c1e Andy McCall ↶ 383 
        // The fast-import parent of the next commit: `:mark` or a commit sha.
384 
        let mut last_parent: Option<String> = None;
79ab8528 Andy McCall ↶ 385 
        for (_, op) in &ops {
75167672 Andy McCall 386 
            let Some(state_ref) = op.view.bookmarks.get(&bookmark).copied() else {
387 
                continue;
388 
            };
79ab8528 Andy McCall ↶ 389 
            if prev.as_ref().is_some_and(|(p, _)| *p == state_ref) {
390 
                continue;
391 
            }
75167672 Andy McCall 392 
            let Ok(state) = self.store.get_state(&state_ref) else {
393 
                continue;
394 
            };
79ab8528 Andy McCall ↶ 395 
            if state.heads.is_empty() {
396 
                prev = Some((state_ref, vec![]));
397 
                continue;
398 
            }
30ea9c1e Andy McCall ↶ 399 
            if let Some(sha) = git_states.get(&state_ref) {
400 
                report.skipped += 1;
401 
                last_parent = Some(sha.clone());
402 
                prev = Some((state_ref, state.heads.clone()));
403 
                continue;
404 
            }
79ab8528 Andy McCall ↶ 405 
            if let Some(m) = marks.get(&state_ref) {
406 
                report.skipped += 1;
30ea9c1e Andy McCall ↶ 407 
                last_parent = Some(format!(":{m}"));
79ab8528 Andy McCall ↶ 408 
                prev = Some((state_ref, state.heads.clone()));
409 
                continue;
410 
            }
411 
            let prev_heads = prev.as_ref().map(|(_, h)| h.clone()).unwrap_or_default();
75167672 Andy McCall 412 
            let added = self
413 
                .negotiate(&prev_heads, &state.heads)
414 
                .unwrap_or_default();
79ab8528 Andy McCall ↶ 415 
            let loaded = self.load_state(&state.heads)?;
416 
            let snapshot = loaded.materialize();
417 
            let (author, secs, mut message) = match added.as_slice() {
418 
                [h] => {
419 
                    let c = self.store.get_change(h)?;
420 
                    let mut m = c.message.clone();
421 
                    if !m.ends_with('\n') {
422 
                        m.push('\n');
423 
                    }
75167672 Andy McCall 424 
                    m.push_str(&format!(
425 
                        "\nChange-Id: {}\nOxy-Hash: {}\n",
426 
                        c.change_id.to_hex(),
427 
                        h.to_hex()
428 
                    ));
79ab8528 Andy McCall ↶ 429 
                    (self.git_author(&c), c.hlc.wall_ms / 1000, m)
430 
                }
75167672 Andy McCall 431 
                [] => (
432 
                    "oxy <oxy@localhost>".to_string(),
433 
                    op.hlc.wall_ms / 1000,
434 
                    format!("oxy: {}\n\n", op.command),
435 
                ),
79ab8528 Andy McCall ↶ 436 
                many => {
437 
                    let mut m = format!("oxy: {} ({} changes)\n\n", op.command, many.len());
438 
                    for h in many {
439 
                        let c = self.store.get_change(h)?;
75167672 Andy McCall 440 
                        m.push_str(&format!(
441 
                            "- {} {}\n",
442 
                            c.change_id.short(8),
443 
                            c.message.lines().next().unwrap_or("")
444 
                        ));
79ab8528 Andy McCall ↶ 445 
                    }
75167672 Andy McCall 446 
                    m.push_str(&format!(
447 
                        "\nOxy-Changes: {}\n",
448 
                        many.iter()
449 
                            .map(|h| h.to_hex())
450 
                            .collect::<Vec<_>>()
451 
                            .join(" ")
452 
                    ));
79ab8528 Andy McCall ↶ 453 
                    ("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, m)
454 
                }
455 
            };
456 
            message.push_str(&format!("Oxy-State: {}\n", state_ref.to_hex()));
457 
            if snapshot.is_conflicted() {
458 
                message.push_str("Oxy-Conflicted: true\n");
459 
            }
460 
            let mark = next_mark;
461 
            next_mark += 1;
30ea9c1e Andy McCall ↶ 462 
            let parents: Vec<String> = last_parent.clone().into_iter().collect();
75167672 Andy McCall 463 
            self.write_commit(
464 
                &mut stream,
465 
                "refs/oxy/export",
466 
                mark,
467 
                &author,
468 
                secs,
469 
                &message,
470 
                &parents,
471 
                &snapshot,
472 
            )?;
79ab8528 Andy McCall ↶ 473 
            marks.insert(state_ref, mark);
30ea9c1e Andy McCall ↶ 474 
            last_parent = Some(format!(":{mark}"));
79ab8528 Andy McCall ↶ 475 
            prev = Some((state_ref, state.heads.clone()));
476 
            report.exported += 1;
477 
        }
30ea9c1e Andy McCall ↶ 478 
        if let Some(p) = &last_parent {
479 
            stream.extend_from_slice(format!("reset refs/heads/{branch}\nfrom {p}\n\n").as_bytes());
79ab8528 Andy McCall ↶ 480 
        }
132b884b Andy McCall ↶ 481 
        // Tags: a tag whose state was exported as a bookmark state points at
482 
        // that commit; otherwise its state gets a commit of its own.
483 
        for t in self.tags()? {
30ea9c1e Andy McCall ↶ 484 
            let target = match (git_states.get(&t.state), marks.get(&t.state)) {
485 
                (Some(sha), _) => sha.clone(),
486 
                (None, Some(m)) => format!(":{m}"),
487 
                (None, None) => {
132b884b Andy McCall ↶ 488 
                    let loaded = self.load_state(&t.heads)?;
489 
                    let snapshot = loaded.materialize();
490 
                    let mark = next_mark;
491 
                    next_mark += 1;
75167672 Andy McCall 492 
                    let message = format!(
493 
                        "oxy: tag {}\n\n{}Oxy-State: {}\n",
494 
                        t.name,
495 
                        if t.message.is_empty() {
496 
                            String::new()
497 
                        } else {
498 
                            format!("{}\n\n", t.message)
499 
                        },
500 
                        t.state.to_hex()
501 
                    );
502 
                    self.write_commit(
503 
                        &mut stream,
504 
                        "refs/oxy/export",
505 
                        mark,
506 
                        "oxy <oxy@localhost>",
507 
                        Hlc::now().wall_ms / 1000,
508 
                        &message,
509 
                        &[],
510 
                        &snapshot,
511 
                    )?;
132b884b Andy McCall ↶ 512 
                    marks.insert(t.state, mark);
513 
                    report.exported += 1;
30ea9c1e Andy McCall ↶ 514 
                    format!(":{mark}")
132b884b Andy McCall ↶ 515 
                }
516 
            };
75167672 Andy McCall 517 
            stream.extend_from_slice(
518 
                format!("reset refs/tags/{}\nfrom {target}\n\n", t.name).as_bytes(),
519 
            );
132b884b Andy McCall ↶ 520 
        }
521 
        stream.extend_from_slice(b"done\n");
79ab8528 Andy McCall ↶ 522
523 
        let mut cmd = Command::new("git");
524 
        // `--force`: the scratch ref may jump between unrelated states.
75167672 Andy McCall 525 
        cmd.arg("-C")
526 
            .arg(git_dir)
527 
            .args(["fast-import", "--quiet", "--done", "--force"])
528 
            .arg(format!("--export-marks={}", git_marks.display()));
79ab8528 Andy McCall ↶ 529 
        if git_marks.exists() {
530 
            cmd.arg(format!("--import-marks={}", git_marks.display()));
531 
        }
75167672 Andy McCall 532 
        let mut child = cmd
533 
            .stdin(Stdio::piped())
534 
            .stdout(Stdio::null())
535 
            .stderr(Stdio::piped())
536 
            .spawn()
537 
            .map_err(|e| RepoError::Msg(format!("running git fast-import: {e}")))?;
79ab8528 Andy McCall ↶ 538 
        child.stdin.take().unwrap().write_all(&stream)?;
539 
        let out = child.wait_with_output()?;
540 
        if !out.status.success() {
75167672 Andy McCall 541 
            return fail(format!(
542 
                "git fast-import failed: {}",
543 
                String::from_utf8_lossy(&out.stderr).trim()
544 
            ));
79ab8528 Andy McCall ↶ 545 
        }
75167672 Andy McCall 546 
        let text: String = marks
547 
            .iter()
548 
            .map(|(h, m)| format!("{} {m}\n", h.to_hex()))
549 
            .collect();
79ab8528 Andy McCall ↶ 550 
        write_atomic(&oxy_marks_path, text.as_bytes())?;
551 
        let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]);
4dee277c Andy McCall ↶ 552 
        report.skipped_subrepos = self.git_export_skipped.replace(0);
79ab8528 Andy McCall ↶ 553 
        Ok(report)
554 
    }
555
c3069537 Andy McCall ↶ 556 
    /// Import branch `branch` of the Git repository at `git_dir` into this
557 
    /// (empty) repository's bookmark (spec §23): one change per commit, its
558 
    /// edits the difference between the union of its parents' states and its
559 
    /// tree, so merge commits become resolutions. Submodules are skipped.
560 
    pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result<GitImportReport> {
561 
        if !self.bookmark_heads()?.is_empty() {
562 
            return fail("git import needs an empty bookmark; import into a fresh repository");
563 
        }
75167672 Andy McCall 564 
        self.import_git_commits(
565 
            git_dir,
566 
            branch,
567 
            &format!("oxy git import {} {branch}", git_dir.display()),
568 
        )
30ea9c1e Andy McCall ↶ 569 
    }
570
571 
    /// Import the commits of `branch` that the bookmark does not have yet,
572 
    /// then move the bookmark to the union of its heads and theirs.
75167672 Andy McCall 573 
    fn import_git_commits(
574 
        &self,
575 
        git_dir: &Path,
576 
        branch: &str,
577 
        command: &str,
578 
    ) -> Result<GitImportReport> {
8f10a07a Andy McCall ↶ 579 
        let batch = self.store.begin_batch()?;
580 
        let result = self.import_git_commits_batched(git_dir, branch, command);
581 
        batch.finish()?;
582 
        result
583 
    }
584
75167672 Andy McCall 585 
    fn import_git_commits_batched(
586 
        &self,
587 
        git_dir: &Path,
588 
        branch: &str,
589 
        command: &str,
590 
    ) -> Result<GitImportReport> {
c3069537 Andy McCall ↶ 591 
        let commits = list_commits(git_dir, branch)?;
592 
        let mut cat = CatFile::start(git_dir)?;
593 
        let mut report = GitImportReport::default();
30ea9c1e Andy McCall ↶ 594 
        let mut known = self.git_known(git_dir)?;
595 
        let old_heads = self.bookmark_heads()?;
dfb23a3a Andy McCall ↶ 596 
        // The bookmark's heads as changes arrive: a change stops being a
597 
        // head when another depends on it, so no full state of every
598 
        // imported change is needed at the end.
599 
        let mut heads_now: BTreeSet<ChangeHash> = old_heads.iter().copied().collect();
958fb27d Andy McCall ↶ 600 
        // The state and Git tree of imported commits whose first-parent
601 
        // children are still to come, so a child starts from its parent's
602 
        // state instead of rebuilding it from history. The last child takes
dfb23a3a Andy McCall ↶ 603 
        // it; earlier ones clone it. At most `max_kept` are kept (a later
958fb27d Andy McCall ↶ 604 
        // child of an evicted one rebuilds).
dfb23a3a Andy McCall ↶ 605 
        // OXY_IMPORT_KEEP overrides it, for measuring.
606 
        let max_kept: usize = std::env::var("OXY_IMPORT_KEEP")
607 
            .ok()
608 
            .and_then(|v| v.parse().ok())
609 
            .unwrap_or(8);
958fb27d Andy McCall ↶ 610 
        let mut waiting: HashMap<&str, usize> = HashMap::new();
611 
        for gc in &commits {
612 
            if let Some(p1) = gc.parents.first() {
613 
                *waiting.entry(p1.as_str()).or_default() += 1;
614 
            }
615 
        }
616 
        type Kept = (State, BTreeMap<String, (String, String)>);
617 
        let mut kept: HashMap<String, Kept> = HashMap::new();
618 
        let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
619 
        let mut last: Option<String> = None;
c3069537 Andy McCall ↶ 620 
        let mut blob_cache: HashMap<String, Vec<u8>> = HashMap::new();
621 
        let mut cache_bytes: usize = 0;
958fb27d Andy McCall ↶ 622 
        // Time spent per phase over all commits (`OXY_TIMINGS=1`).
623 
        let mut spent = [std::time::Duration::ZERO; 6];
624 
        let mut clock = std::time::Instant::now();
625 
        let mut lap = |i: usize, clock: &mut std::time::Instant| {
626 
            let now = std::time::Instant::now();
627 
            spent[i] += now - *clock;
628 
            *clock = now;
629 
        };
c3069537 Andy McCall ↶ 630
631 
        for gc in &commits {
30ea9c1e Andy McCall ↶ 632 
            if known.imported.contains_key(&gc.sha) || known.exported.contains_key(&gc.sha) {
633 
                continue;
634 
            }
958fb27d Andy McCall ↶ 635 
            // Base: union of every parent's state, starting from the first
636 
            // parent's kept state when there is one; its tree tells which
637 
            // paths this commit leaves alone.
638 
            let (mut base, prev_tree) = match gc.parents.first() {
639 
                Some(p1) => {
640 
                    let left = waiting.get_mut(p1.as_str()).map(|n| {
641 
                        *n -= 1;
642 
                        *n
643 
                    });
644 
                    let found = if left == Some(0) {
645 
                        kept_order.retain(|k| k != p1);
646 
                        kept.remove(p1.as_str())
647 
                    } else {
648 
                        kept.get(p1.as_str()).cloned()
649 
                    };
650 
                    match found {
651 
                        Some((st, tree)) => (st, Some(tree)),
652 
                        None => {
b493d993 Andy McCall ↶ 653 
                            // History only: graphs are built for the files a
654 
                            // commit changes, so memory follows what changes,
655 
                            // not the whole tree.
656 
                            let mut st = State::new_history_only();
958fb27d Andy McCall ↶ 657 
                            self.add_git_ancestors(&mut st, git_dir, &[p1.as_str()], &known)?;
658 
                            (st, None)
659 
                        }
660 
                    }
30ea9c1e Andy McCall ↶ 661 
                }
b493d993 Andy McCall ↶ 662 
                None => (State::new_history_only(), None),
c3069537 Andy McCall ↶ 663 
            };
958fb27d Andy McCall ↶ 664 
            // Nodes the other parents' changes touch: the first parent's
665 
            // tree cannot vouch for them.
666 
            let mut touched: HashSet<NodeId> = HashSet::new();
c3069537 Andy McCall ↶ 667 
            if gc.parents.len() > 1 {
668 
                report.merges += 1;
669 
                let mut args: Vec<String> = gc.parents[1..].to_vec();
670 
                args.push(format!("^{}", gc.parents[0]));
30ea9c1e Andy McCall ↶ 671 
                let refs: Vec<&str> = args.iter().map(String::as_str).collect();
958fb27d Andy McCall ↶ 672 
                self.add_git_ancestors_touching(&mut base, git_dir, &refs, &known, &mut touched)?;
03571027 Andy McCall ↶ 673 
            }
958fb27d Andy McCall ↶ 674 
            lap(0, &mut clock);
03571027 Andy McCall ↶ 675 
            let tree = list_tree(git_dir, &gc.sha)?;
958fb27d Andy McCall ↶ 676 
            lap(1, &mut clock);
03571027 Andy McCall ↶ 677 
            // Text files with the same blob and mode as in the first
678 
            // parent's tree equal the base's rendering: they are neither
679 
            // rendered nor read nor compared (spec §12.3, as for a clean
680 
            // working copy), so a commit costs what it changes.
681 
            let mut same: HashMap<&str, NodeId> = HashMap::new();
682 
            if let Some(prev) = &prev_tree {
683 
                let view = base.tree().view();
684 
                for (path, v) in &tree {
685 
                    // Submodule handling reads these two whatever changed.
686 
                    if prev.get(path) != Some(v) || path == ".gitmodules" || path == ".oxymodules" {
687 
                        continue;
688 
                    }
689 
                    if let Some(e) = view.entries.get(path) {
958fb27d Andy McCall ↶ 690 
                        if e.kind == NodeKind::File && !e.mode.opaque && !touched.contains(&e.node)
691 
                        {
03571027 Andy McCall ↶ 692 
                            same.insert(path.as_str(), e.node);
693 
                        }
694 
                    }
695 
                }
c3069537 Andy McCall ↶ 696 
            }
b493d993 Andy McCall ↶ 697 
            // Build the graphs of every text file the commit may change.
698 
            let skip: HashSet<NodeId> = same.values().copied().collect();
699 
            let need: Vec<NodeId> = base
700 
                .tree()
701 
                .view()
702 
                .entries
703 
                .values()
704 
                .filter(|e| e.kind == NodeKind::File && !e.mode.opaque && !skip.contains(&e.node))
705 
                .map(|e| e.node)
706 
                .collect();
707 
            self.load_graphs_of(&mut base, &need)?;
708 
            let base_snapshot = base.materialize_except(&skip);
c3069537 Andy McCall ↶ 709
958fb27d Andy McCall ↶ 710 
            lap(2, &mut clock);
c3069537 Andy McCall ↶ 711 
            // The commit's tree as in-memory entries.
712 
            let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();
4dee277c Andy McCall ↶ 713 
            let mut gitlinks: Vec<(String, String)> = Vec::new();
03571027 Andy McCall ↶ 714 
            for (path, (mode, sha)) in &tree {
715 
                let (path, sha) = (path.clone(), sha.clone());
c3069537 Andy McCall ↶ 716 
                let (kind, exec) = match mode.as_str() {
717 
                    "100644" | "100664" => (NodeKind::File, false),
718 
                    "100755" => (NodeKind::File, true),
719 
                    "120000" => (NodeKind::Symlink, false),
720 
                    "160000" => {
4dee277c Andy McCall ↶ 721 
                        gitlinks.push((path, sha));
c3069537 Andy McCall ↶ 722 
                        continue;
723 
                    }
724 
                    _ => continue,
725 
                };
03571027 Andy McCall ↶ 726 
                if same.contains_key(path.as_str()) {
727 
                    let mut e = FsEntry::new(kind, exec, Vec::new());
728 
                    e.unchanged = true;
729 
                    entries.insert(path, e);
730 
                    continue;
731 
                }
c3069537 Andy McCall ↶ 732 
                let content = match blob_cache.get(&sha) {
733 
                    Some(c) => c.clone(),
734 
                    None => {
735 
                        let c = cat.read(&sha)?;
dfb23a3a Andy McCall ↶ 736 
                        if cache_bytes + c.len() > 64 * 1024 * 1024 {
c3069537 Andy McCall ↶ 737 
                            blob_cache.clear();
738 
                            cache_bytes = 0;
739 
                        }
740 
                        cache_bytes += c.len();
741 
                        blob_cache.insert(sha.clone(), c.clone());
742 
                        c
743 
                    }
744 
                };
b7f102fc Andy McCall ↶ 745 
                entries.insert(path, FsEntry::new(kind, exec, content));
c3069537 Andy McCall ↶ 746 
            }
747
4dee277c Andy McCall ↶ 748 
            // Submodules (spec §23): gitlinks become subrepo nodes with Git
749 
            // pins, and `.gitmodules` also yields `.oxymodules`.
750 
            if !gitlinks.is_empty() {
751 
                report.submodules += gitlinks.len();
75167672 Andy McCall 752 
                let gm = entries
753 
                    .get(".gitmodules")
754 
                    .map(|e| {
755 
                        crate::subrepo::Modules::from_gitmodules(&String::from_utf8_lossy(
756 
                            &e.content,
757 
                        ))
758 
                    })
759 
                    .unwrap_or_default();
760 
                let mut modules = entries
761 
                    .get(".oxymodules")
762 
                    .map(|e| crate::subrepo::Modules::parse(&String::from_utf8_lossy(&e.content)))
763 
                    .unwrap_or_default();
4dee277c Andy McCall ↶ 764 
                for (path, sha) in &gitlinks {
765 
                    let m = gm.entries.get(path);
75167672 Andy McCall 766 
                    let branch = m
767 
                        .and_then(|m| m.bookmark.clone())
768 
                        .unwrap_or_else(|| "HEAD".into());
769 
                    entries.insert(
770 
                        path.clone(),
771 
                        FsEntry::new(
772 
                            NodeKind::Subrepo,
773 
                            false,
774 
                            crate::subrepo::Pin::Git {
775 
                                sha: sha.clone(),
776 
                                branch,
777 
                            }
778 
                            .encode(),
779 
                        ),
780 
                    );
4dee277c Andy McCall ↶ 781 
                    if let Some(m) = m {
782 
                        modules.entries.insert(path.clone(), m.clone());
783 
                    }
784 
                }
785 
                if !modules.entries.is_empty() {
75167672 Andy McCall 786 
                    entries.insert(
787 
                        ".oxymodules".into(),
788 
                        FsEntry::new(NodeKind::File, false, modules.to_text().into_bytes()),
789 
                    );
4dee277c Andy McCall ↶ 790 
                }
791 
            }
958fb27d Andy McCall ↶ 792 
            lap(3, &mut clock);
dfb23a3a Andy McCall ↶ 793 
            // New nodes' identifiers derive from the commit, so importing the
794 
            // same Git history twice gives the same changes (spec §23).
795 
            let seed = *blake3::hash(format!("oxy git import\0{}", gc.sha).as_bytes()).as_bytes();
62794ecc Andy McCall ↶ 796 
            let mut built =
dfb23a3a Andy McCall ↶ 797 
                self.build_from_entries_seeded(&base, &base_snapshot, &entries, seed)?;
958fb27d Andy McCall ↶ 798 
            lap(4, &mut clock);
62794ecc Andy McCall ↶ 799 
            // A file or submodule with several live blobs (branches that
800 
            // replaced it independently) is resolved to the commit's blob:
801 
            // Git's merge is the authority here, unlike a working copy,
802 
            // where a candidate on disk is not yet a decision (spec §23).
803 
            for (path, f) in &base_snapshot.files {
804 
                let (FileContent::Opaque { blobs }, Some(e)) = (&f.content, entries.get(path))
805 
                else {
806 
                    continue;
807 
                };
808 
                if blobs.len() < 2 || f.kind != e.kind {
809 
                    continue;
810 
                }
811 
                let chosen = self.store_blob(&e.content)?;
812 
                for b in blobs {
813 
                    built.edits.push(Edit::Replace {
814 
                        node: f.node,
815 
                        old: Some(*b),
816 
                        new: Some(chosen),
817 
                    });
818 
                }
819 
                built.resolves = true;
820 
                built
821 
                    .contributors
822 
                    .extend(base.live_replaces(&f.node).into_iter().map(|(c, _)| c));
823 
            }
c3069537 Andy McCall ↶ 824 
            let mut deps: BTreeSet<ChangeHash> = built.contributors.iter().copied().collect();
825 
            let mut message = gc.message.clone();
826 
            if !message.ends_with('\n') {
827 
                message.push('\n');
828 
            }
75167672 Andy McCall 829 
            message.push_str(&format!(
830 
                "\nGit-Commit: {}\nGit-Author: {}\n",
831 
                gc.sha, gc.author
832 
            ));
c3069537 Andy McCall ↶ 833 
            let mut change = Change {
6f8aeb99 Andy McCall ↶ 834 
                version: crate::objects::CHANGE_VERSION,
c3069537 Andy McCall ↶ 835 
                change_id: change_id_for(&gc.sha, &gc.message),
836 
                deps: vec![],
837 
                author: ObjectRef::ZERO,
75167672 Andy McCall 838 
                hlc: Hlc {
839 
                    wall_ms: gc.author_secs * 1000,
840 
                    logical: 0,
841 
                },
c3069537 Andy McCall ↶ 842 
                message,
843 
                tree_ops: built.tree_ops,
844 
                edits: built.edits,
75167672 Andy McCall 845 
                kind: if built.resolves {
846 
                    ChangeKind::Resolution
847 
                } else {
848 
                    ChangeKind::Normal
849 
                },
c3069537 Andy McCall ↶ 850 
            };
851 
            deps.extend(referenced_changes(&change, &base));
852 
            change.deps = deps.into_iter().collect();
853 
            change.normalize();
854 
            let hash = self.store.put_change(&change)?;
dfb23a3a Andy McCall ↶ 855 
            for d in &change.deps {
856 
                heads_now.remove(d);
857 
            }
858 
            heads_now.insert(hash);
b493d993 Andy McCall ↶ 859 
            base.add_known(hash, change, Vec::new());
30ea9c1e Andy McCall ↶ 860 
            known.imported.insert(gc.sha.clone(), hash);
958fb27d Andy McCall ↶ 861 
            if waiting.get(gc.sha.as_str()).is_some_and(|n| *n > 0) {
862 
                kept.insert(gc.sha.clone(), (base, tree));
863 
                kept_order.push_back(gc.sha.clone());
dfb23a3a Andy McCall ↶ 864 
                if kept_order.len() > max_kept {
958fb27d Andy McCall ↶ 865 
                    if let Some(old) = kept_order.pop_front() {
866 
                        kept.remove(&old);
867 
                    }
868 
                }
869 
            }
870 
            last = Some(gc.sha.clone());
c3069537 Andy McCall ↶ 871 
            report.commits += 1;
958fb27d Andy McCall ↶ 872 
            lap(5, &mut clock);
873 
        }
874 
        for (name, d) in [
875 
            "git import: bases",
876 
            "git import: ls-tree",
877 
            "git import: render base",
878 
            "git import: read blobs",
879 
            "git import: build change",
880 
            "git import: store change",
881 
        ]
882 
        .iter()
883 
        .zip(spent)
884 
        {
885 
            crate::timing::record(name, d);
c3069537 Andy McCall ↶ 886 
        }
887
30ea9c1e Andy McCall ↶ 888 
        // The bookmark: heads of the old state plus every new change.
889 
        report.heads = old_heads.len();
890 
        if last.is_none() {
891 
            return Ok(report);
892 
        }
dfb23a3a Andy McCall ↶ 893 
        let heads: Vec<ChangeHash> = heads_now.into_iter().collect();
c3069537 Andy McCall ↶ 894 
        report.heads = heads.len();
895 
        let (op_ref, op) = self.head_op()?;
896 
        let mut view = op.view.clone();
30ea9c1e Andy McCall ↶ 897 
        let old_state_ref = view.bookmarks.get(&self.config.bookmark).copied();
898 
        let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?;
75167672 Andy McCall 899 
        view.bookmarks
900 
            .insert(self.config.bookmark.clone(), state_ref);
30ea9c1e Andy McCall ↶ 901 
        self.record_op(&[op_ref], command, view)?;
902 
        // The new state equals the tip's tree when the old one was itself a
903 
        // Git commit's tree (fetched or exported), or there was none.
75167672 Andy McCall 904 
        let old_known = old_heads.is_empty()
905 
            || old_state_ref.is_some_and(|s| known.exported.values().any(|e| *e == s));
30ea9c1e Andy McCall ↶ 906 
        if let (Some(tip), true) = (commits.last(), old_known) {
907 
            self.record_git_state(git_dir, &state_ref, &tip.sha)?;
908 
        }
909 
        // The working copy follows when it was clean on the old heads.
910 
        if old_heads.is_empty() {
911 
            self.checkout(&heads, true, &format!("{command} (checkout)"))?;
912 
        } else {
913 
            let snap = self.snapshot(command)?;
914 
            let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
915 
            let mut base = snap.base.maximal(&snap.wc.deps);
916 
            base.sort();
917 
            let mut sorted_old = old_heads.clone();
918 
            sorted_old.sort();
919 
            if clean && base == sorted_old {
920 
                self.checkout(&heads, false, &format!("{command} (checkout)"))?;
921 
            }
922 
        }
c3069537 Andy McCall ↶ 923 
        Ok(report)
924 
    }
925
30ea9c1e Andy McCall ↶ 926 
    /// Add to `st` the changes of every Git commit `rev-list args` lists:
927 
    /// an imported commit's change, or the whole state an exported commit
928 
    /// was made from (which may hold changes Git never saw).
75167672 Andy McCall 929 
    fn add_git_ancestors(
958fb27d Andy McCall ↶ 930 
        &self,
931 
        st: &mut State,
932 
        git_dir: &Path,
933 
        args: &[&str],
934 
        known: &GitKnown,
935 
    ) -> Result<()> {
936 
        self.add_git_ancestors_touching(st, git_dir, args, known, &mut HashSet::new())
937 
    }
938
939 
    /// `add_git_ancestors`, also collecting in `touched` every node the
940 
    /// changes it adds edit, place or remove.
941 
    fn add_git_ancestors_touching(
75167672 Andy McCall 942 
        &self,
943 
        st: &mut State,
944 
        git_dir: &Path,
945 
        args: &[&str],
946 
        known: &GitKnown,
958fb27d Andy McCall ↶ 947 
        touched: &mut HashSet<NodeId>,
75167672 Andy McCall 948 
    ) -> Result<()> {
30ea9c1e Andy McCall ↶ 949 
        let mut full = vec!["rev-list"];
950 
        full.extend(args);
951 
        let out = git(git_dir, &full)?;
958fb27d Andy McCall ↶ 952 
        let mut add = |st: &mut State, h: &ChangeHash| -> Result<()> {
953 
            if st.contains(h) {
954 
                return Ok(());
955 
            }
956 
            let c = self.store.get_change(h)?;
957 
            touched.extend(c.edits.iter().map(|e| e.node()));
958 
            touched.extend(c.tree_ops.iter().map(|o| o.node()));
959 
            st.add_unchecked(c);
960 
            Ok(())
961 
        };
c3069537 Andy McCall ↶ 962 
        for sha in out.lines() {
30ea9c1e Andy McCall ↶ 963 
            if let Some(h) = known.imported.get(sha) {
958fb27d Andy McCall ↶ 964 
                add(st, h)?;
30ea9c1e Andy McCall ↶ 965 
            } else if let Some(state_ref) = known.exported.get(sha) {
966 
                let heads = self.store.get_state(state_ref)?.heads;
967 
                for h in self.negotiate(&[], &heads)? {
958fb27d Andy McCall ↶ 968 
                    add(st, &h)?;
30ea9c1e Andy McCall ↶ 969 
                }
970 
            }
971 
        }
972 
        Ok(())
973 
    }
974
975 
    /// Commits this repository can relate to Oxygen history: imported
976 
    /// ones by change, exported ones (through `git_dir`'s marks) by state.
977 
    fn git_known(&self, git_dir: &Path) -> Result<GitKnown> {
978 
        let imported = self.imported_commits()?;
979 
        let marks_dir = self.git_marks_dir(git_dir)?;
980 
        let oxy_marks = Self::read_oxy_marks(&marks_dir.join("oxy-marks"))?;
981 
        let mut sha_of_mark: HashMap<u64, String> = HashMap::new();
982 
        if let Ok(text) = fs::read_to_string(marks_dir.join("git-marks")) {
983 
            for line in text.lines() {
984 
                if let Some((m, sha)) = line.split_once(' ') {
985 
                    if let Ok(m) = m.trim_start_matches(':').parse::<u64>() {
986 
                        sha_of_mark.insert(m, sha.trim().to_string());
987 
                    }
988 
                }
989 
            }
990 
        }
991 
        let mut exported = HashMap::new();
992 
        for (state_ref, m) in oxy_marks {
993 
            if let Some(sha) = sha_of_mark.get(&m) {
994 
                exported.insert(sha.clone(), state_ref);
c3069537 Andy McCall ↶ 995 
            }
996 
        }
30ea9c1e Andy McCall ↶ 997 
        for (state_ref, sha) in Self::read_git_states(&marks_dir.join("git-states"))? {
998 
            exported.entry(sha).or_insert(state_ref);
999 
        }
1000 
        Ok(GitKnown { imported, exported })
c3069537 Andy McCall ↶ 1001 
    }
1002
79ab8528 Andy McCall ↶ 1003 
    fn git_author(&self, c: &Change) -> String {
1004 
        match self.store.get_identity(&c.author) {
75167672 Andy McCall 1005 
            Ok(id) if !id.name.is_empty() => format!(
1006 
                "{} <{}>",
1007 
                id.name.replace(['<', '>', '\n'], " "),
1008 
                id.email.replace(['<', '>', '\n', ' '], "")
1009 
            ),
79ab8528 Andy McCall ↶ 1010 
            _ => "oxy <oxy@localhost>".to_string(),
1011 
        }
1012 
    }
1013
1014 
    #[allow(clippy::too_many_arguments)]
75167672 Andy McCall 1015 
    fn write_commit(
1016 
        &self,
1017 
        out: &mut Vec<u8>,
1018 
        reference: &str,
1019 
        mark: u64,
1020 
        author: &str,
1021 
        secs: u64,
1022 
        message: &str,
1023 
        parents: &[String],
1024 
        snapshot: &Snapshot,
1025 
    ) -> Result<()> {
79ab8528 Andy McCall ↶ 1026 
        out.extend_from_slice(format!("commit {reference}\nmark :{mark}\nauthor {author} {secs} +0000\ncommitter {author} {secs} +0000\ndata {}\n{message}\n", message.len()).as_bytes());
1027 
        for (i, p) in parents.iter().enumerate() {
75167672 Andy McCall 1028 
            out.extend_from_slice(
1029 
                format!("{} {p}\n", if i == 0 { "from" } else { "merge" }).as_bytes(),
1030 
            );
79ab8528 Andy McCall ↶ 1031 
        }
1032 
        out.extend_from_slice(b"deleteall\n");
4dee277c Andy McCall ↶ 1033 
        let has_gitmodules = snapshot.files.contains_key(".gitmodules");
79ab8528 Andy McCall ↶ 1034 
        for (path, f) in &snapshot.files {
4dee277c Andy McCall ↶ 1035 
            if path == ".oxymodules" {
1036 
                // Git reads `.gitmodules`; derive it unless the tree has one.
1037 
                if !has_gitmodules {
1038 
                    if let FileContent::Text { bytes, .. } = &f.content {
75167672 Andy McCall 1039 
                        let gm = crate::subrepo::Modules::parse(&String::from_utf8_lossy(bytes))
1040 
                            .to_gitmodules();
4dee277c Andy McCall ↶ 1041 
                        if !gm.is_empty() {
75167672 Andy McCall 1042 
                            out.extend_from_slice(
1043 
                                format!("M 100644 inline .gitmodules\ndata {}\n", gm.len())
1044 
                                    .as_bytes(),
1045 
                            );
4dee277c Andy McCall ↶ 1046 
                            out.extend_from_slice(gm.as_bytes());
1047 
                            out.push(b'\n');
1048 
                        }
1049 
                    }
1050 
                }
1051 
                continue;
1052 
            }
1053 
            if f.kind == NodeKind::Subrepo {
1054 
                if let FileContent::Opaque { blobs } = &f.content {
1055 
                    if let Some(b) = blobs.first() {
1056 
                        match crate::subrepo::Pin::decode(&self.read_blob(b)?)? {
75167672 Andy McCall 1057 
                            crate::subrepo::Pin::Git { sha, .. } => {
1058 
                                out.extend_from_slice(format!("M 160000 {sha} {path}\n").as_bytes())
1059 
                            }
1060 
                            crate::subrepo::Pin::Oxy { .. } => self
1061 
                                .git_export_skipped
1062 
                                .set(self.git_export_skipped.get() + 1),
4dee277c Andy McCall ↶ 1063 
                        }
1064 
                    }
1065 
                }
1066 
                continue;
1067 
            }
79ab8528 Andy McCall ↶ 1068 
            let (mode, bytes): (&str, Vec<u8>) = match (&f.kind, &f.content) {
1069 
                (NodeKind::Dir, _) => continue,
75167672 Andy McCall 1070 
                (NodeKind::Symlink, FileContent::Opaque { blobs }) => (
1071 
                    "120000",
1072 
                    blobs
1073 
                        .first()
1074 
                        .map(|b| self.read_blob(b))
1075 
                        .transpose()?
1076 
                        .unwrap_or_default(),
1077 
                ),
1078 
                (_, FileContent::Text { bytes, .. }) => {
1079 
                    (if f.mode.exec { "100755" } else { "100644" }, bytes.clone())
1080 
                }
1081 
                (_, FileContent::Opaque { blobs }) => (
1082 
                    if f.mode.exec { "100755" } else { "100644" },
1083 
                    blobs
1084 
                        .first()
1085 
                        .map(|b| self.read_blob(b))
1086 
                        .transpose()?
1087 
                        .unwrap_or_default(),
1088 
                ),
79ab8528 Andy McCall ↶ 1089 
                (_, FileContent::Dir) => continue,
1090 
            };
75167672 Andy McCall 1091 
            out.extend_from_slice(
1092 
                format!("M {mode} inline {path}\ndata {}\n", bytes.len()).as_bytes(),
1093 
            );
79ab8528 Andy McCall ↶ 1094 
            out.extend_from_slice(&bytes);
1095 
            out.push(b'\n');
1096 
        }
1097 
        out.push(b'\n');
1098 
        Ok(())
1099 
    }
1100 
}
62794ecc Andy McCall ↶ 1101
1102 
/// What `verify_git_import` found (spec §23: import must reproduce every tree).
1103 
#[derive(Debug, Clone, Default, PartialEq, Eq)]
1104 
pub struct GitVerifyReport {
1105 
    pub commits: usize,
1106 
    pub files_compared: usize,
1107 
    /// `(commit, what differs)`; empty when every commit's tree is reproduced.
1108 
    pub mismatches: Vec<(String, String)>,
1109 
}
1110
1111 
impl Repo {
1112 
    /// Check an import against its Git source: for every commit of
1113 
    /// `branch`, the state formed by the changes imported for it and all
1114 
    /// its ancestors must materialize to exactly that commit's tree, byte
1115 
    /// for byte, with the same executable bits, symlink targets and
1116 
    /// submodule pins. Reads only; stops listing after 50 differences.
1117 
    pub fn verify_git_import(&self, git_dir: &Path, branch: &str) -> Result<GitVerifyReport> {
1118 
        let commits = list_commits(git_dir, branch)?;
1119 
        let imported = self.imported_commits()?;
1120 
        let mut cat = CatFile::start(git_dir)?;
1121 
        let mut rep = GitVerifyReport::default();
1122 
        // Changes of each commit and all its ancestors, built up in topological order.
1123 
        let mut closure: HashMap<String, Arc<BTreeSet<ChangeHash>>> = HashMap::new();
a87c7b9d Andy McCall ↶ 1124 
        // Graphs (by key) and blobs already found equal to a Git blob.
1125 
        let mut verified_graphs: HashMap<[u8; 32], String> = HashMap::new();
1126 
        let mut verified_blobs: HashMap<ObjectRef, String> = HashMap::new();
57d8dc05 Andy McCall ↶ 1127 
        // The state of a commit whose first-parent children are still to
1128 
        // come, so a child adds its own changes instead of loading all its
1129 
        // history again (as the import does).
1130 
        let mut waiting: HashMap<&str, usize> = HashMap::new();
62794ecc Andy McCall ↶ 1131 
        for c in &commits {
57d8dc05 Andy McCall ↶ 1132 
            if let Some(p1) = c.parents.first() {
1133 
                *waiting.entry(p1.as_str()).or_default() += 1;
1134 
            }
1135 
        }
1136 
        let mut kept: HashMap<String, State> = HashMap::new();
1137 
        let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
1138 
        for c in &commits {
62794ecc Andy McCall ↶ 1139 
            rep.commits += 1;
1140 
            let mut set: BTreeSet<ChangeHash> = BTreeSet::new();
1141 
            for p in &c.parents {
1142 
                if let Some(ps) = closure.get(p) {
1143 
                    set.extend(ps.iter().copied());
1144 
                }
1145 
            }
1146 
            match imported.get(&c.sha) {
1147 
                Some(h) => {
1148 
                    set.insert(*h);
1149 
                }
1150 
                None => {
1151 
                    rep.mismatches.push((
1152 
                        c.sha.clone(),
1153 
                        "no imported change carries this commit".into(),
1154 
                    ));
1155 
                }
1156 
            }
1157 
            let set = Arc::new(set);
1158 
            closure.insert(c.sha.clone(), set.clone());
1159 
            if rep.mismatches.len() >= 50 {
1160 
                break;
1161 
            }
1162 
            let heads: Vec<ChangeHash> = set.iter().copied().collect();
a87c7b9d Andy McCall ↶ 1163 
            // History only: a file whose graph (by key) or blob was already
1164 
            // found equal to this Git blob is not built, rendered or read
1165 
            // again; the rest are.
57d8dc05 Andy McCall ↶ 1166 
            let from_parent = c.parents.first().and_then(|p1| {
1167 
                let left = waiting.get_mut(p1.as_str()).map(|n| {
1168 
                    *n -= 1;
1169 
                    *n
1170 
                });
1171 
                if left == Some(0) {
1172 
                    kept_order.retain(|k| k != p1);
1173 
                    kept.remove(p1.as_str())
1174 
                } else {
1175 
                    kept.get(p1.as_str()).cloned()
1176 
                }
1177 
            });
1178 
            let state = match from_parent {
1179 
                Some(mut st) => {
1180 
                    // The changes the parent's state lacks. The state never
1181 
                    // builds a graph (they are built beside it), so they
1182 
                    // are read without their text.
1183 
                    for h in set.iter() {
1184 
                        if !st.contains(h) {
1185 
                            let (change, texts) = self.store.get_change_lazy(h)?;
1186 
                            st.add_known(*h, change, texts);
1187 
                        }
1188 
                    }
1189 
                    st
1190 
                }
1191 
                None => self.load_state_history(&heads)?,
1192 
            };
a87c7b9d Andy McCall ↶ 1193 
            let view = state.tree().view();
62794ecc Andy McCall ↶ 1194 
            let tree = list_tree(git_dir, &c.sha)?;
1195 
            let short = &c.sha[..12.min(c.sha.len())];
1196 
            let mut differ = |what: String| rep.mismatches.push((c.sha.clone(), what));
a87c7b9d Andy McCall ↶ 1197 
            let mut conflicted: Vec<String> =
1198 
                view.conflicts.iter().map(|c| format!("{c:?}")).collect();
1199 
            let mut need: Vec<NodeId> = Vec::new();
1200 
            for (path, (mode, sha)) in &tree {
1201 
                let Some(e) = view.entries.get(path) else {
1202 
                    continue;
1203 
                };
1204 
                if mode != "160000"
1205 
                    && mode != "120000"
1206 
                    && e.kind == NodeKind::File
1207 
                    && !e.mode.opaque
1208 
                {
1209 
                    let known = state
1210 
                        .graph_key(&e.node)
1211 
                        .is_some_and(|k| verified_graphs.get(&k) == Some(sha));
1212 
                    if !known {
1213 
                        need.push(e.node);
62794ecc Andy McCall ↶ 1214 
                    }
1215 
                }
1216 
            }
a87c7b9d Andy McCall ↶ 1217 
            // A graph's key is known only before it is built.
1218 
            let keys: HashMap<NodeId, [u8; 32]> = need
1219 
                .iter()
1220 
                .filter_map(|n| Some((*n, state.graph_key(n)?)))
1221 
                .collect();
1222 
            let need_set: HashSet<NodeId> = need.iter().copied().collect();
57d8dc05 Andy McCall ↶ 1223 
            let graphs = self.build_graphs_detached(&state, &need)?;
62794ecc Andy McCall ↶ 1224 
            for (path, (mode, sha)) in &tree {
a87c7b9d Andy McCall ↶ 1225 
                let Some(e) = view.entries.get(path) else {
62794ecc Andy McCall ↶ 1226 
                    differ(format!("{short}: {path} is missing"));
1227 
                    continue;
1228 
                };
1229 
                rep.files_compared += 1;
1230 
                match mode.as_str() {
a87c7b9d Andy McCall ↶ 1231 
                    "160000" => {
1232 
                        let blobs = state.live_blobs(&e.node);
1233 
                        match (e.kind, blobs.as_slice()) {
1234 
                            (NodeKind::Subrepo, [b]) => {
1235 
                                let pin = crate::subrepo::Pin::decode(&self.read_blob(b)?)?;
1236 
                                if !matches!(&pin, crate::subrepo::Pin::Git { sha: s, .. } if s == sha)
1237 
                                {
1238 
                                    differ(format!("{short}: {path} pins another commit"));
1239 
                                }
62794ecc Andy McCall ↶ 1240 
                            }
a87c7b9d Andy McCall ↶ 1241 
                            (kind, _) => differ(format!(
1242 
                                "{short}: {path} is not one submodule pin: {kind:?} with {} pins",
1243 
                                blobs.len()
1244 
                            )),
62794ecc Andy McCall ↶ 1245 
                        }
a87c7b9d Andy McCall ↶ 1246 
                    }
62794ecc Andy McCall ↶ 1247 
                    "120000" => {
1248 
                        let target = cat.read(sha)?;
a87c7b9d Andy McCall ↶ 1249 
                        let blobs = state.live_blobs(&e.node);
1250 
                        match (e.kind, blobs.as_slice()) {
1251 
                            (NodeKind::Symlink, [b]) if self.read_blob(b)? == target => {}
62794ecc Andy McCall ↶ 1252 
                            _ => differ(format!("{short}: symlink {path} differs")),
1253 
                        }
1254 
                    }
1255 
                    _ => {
a87c7b9d Andy McCall ↶ 1256 
                        if e.mode.exec != (mode == "100755") {
1257 
                            differ(format!("{short}: {path} executable bit differs"));
1258 
                        }
1259 
                        if e.kind == NodeKind::File && !e.mode.opaque {
1260 
                            if !need_set.contains(&e.node) {
1261 
                                // Known equal: skipped when the graphs were chosen.
62794ecc Andy McCall ↶ 1262 
                                continue;
1263 
                            }
57d8dc05 Andy McCall ↶ 1264 
                            let (have, m) = match graphs.get(&e.node) {
1265 
                                Some(g) => state.render_graph(g),
1266 
                                None => (Vec::new(), Default::default()),
1267 
                            };
a87c7b9d Andy McCall ↶ 1268 
                            if m.is_conflicted() {
57d8dc05 Andy McCall ↶ 1269 
                                // Who conflicts: a copy of the state with this graph built.
1270 
                                let mut detail = state.clone();
1271 
                                self.load_graphs_of(&mut detail, &[e.node])?;
1272 
                                let who: Vec<String> = detail
a87c7b9d Andy McCall ↶ 1273 
                                    .conflict_regions(&e.node)
1274 
                                    .contributors
1275 
                                    .iter()
1276 
                                    .filter_map(|h| state.get(h))
1277 
                                    .map(|m| {
1278 
                                        let sha = m
1279 
                                            .message
1280 
                                            .lines()
1281 
                                            .rev()
1282 
                                            .find_map(|l| l.strip_prefix("Git-Commit: "))
1283 
                                            .unwrap_or("?");
1284 
                                        format!(
1285 
                                            "{} {:?}",
1286 
                                            &sha[..12.min(sha.len())],
1287 
                                            m.message.lines().next().unwrap_or("")
1288 
                                        )
1289 
                                    })
1290 
                                    .collect();
1291 
                                conflicted.push(format!("{path} between {}", who.join(" and ")));
1292 
                            }
1293 
                            let want = cat.read(sha)?;
1294 
                            if have != want {
1295 
                                differ(format!(
1296 
                                    "{short}: {path} differs: {}",
1297 
                                    first_difference(&have, &want)
1298 
                                ));
1299 
                            } else if !m.is_conflicted() {
1300 
                                if let Some(k) = keys.get(&e.node) {
1301 
                                    verified_graphs.insert(*k, sha.clone());
1302 
                                }
1303 
                            }
1304 
                            continue;
1305 
                        }
1306 
                        let blobs = state.live_blobs(&e.node);
1307 
                        let [b] = blobs.as_slice() else {
1308 
                            differ(format!("{short}: {path} is not one file"));
1309 
                            continue;
62794ecc Andy McCall ↶ 1310 
                        };
a87c7b9d Andy McCall ↶ 1311 
                        if verified_blobs.get(b) == Some(sha) {
1312 
                            continue;
62794ecc Andy McCall ↶ 1313 
                        }
a87c7b9d Andy McCall ↶ 1314 
                        if self.read_blob(b)? != cat.read(sha)? {
1315 
                            differ(format!("{short}: {path} differs"));
1316 
                        } else {
1317 
                            verified_blobs.insert(*b, sha.clone());
62794ecc Andy McCall ↶ 1318 
                        }
1319 
                    }
1320 
                }
1321 
            }
a87c7b9d Andy McCall ↶ 1322 
            if !conflicted.is_empty() {
1323 
                differ(format!(
1324 
                    "{short}: the state is conflicted: {}",
1325 
                    conflicted.join("; ")
1326 
                ));
1327 
            }
62794ecc Andy McCall ↶ 1328 
            // Import records submodules in `.oxymodules` as well (spec §23),
1329 
            // and export leaves it out again.
1330 
            let has_submodules = tree.values().any(|(m, _)| m == "160000");
a87c7b9d Andy McCall ↶ 1331 
            for (path, e) in &view.entries {
1332 
                if e.kind != NodeKind::Dir
62794ecc Andy McCall ↶ 1333 
                    && !tree.contains_key(path)
1334 
                    && !(has_submodules && path == ".oxymodules")
1335 
                {
1336 
                    differ(format!("{short}: {path} is extra"));
57d8dc05 Andy McCall ↶ 1337 
                }
1338 
            }
1339 
            if waiting.get(c.sha.as_str()).is_some_and(|n| *n > 0) {
1340 
                kept.insert(c.sha.clone(), state);
1341 
                kept_order.push_back(c.sha.clone());
1342 
                if kept_order.len() > 8 {
1343 
                    if let Some(old) = kept_order.pop_front() {
1344 
                        kept.remove(&old);
1345 
                    }
62794ecc Andy McCall ↶ 1346 
                }
1347 
            }
1348 
            if rep.mismatches.len() >= 50 {
1349 
                break;
1350 
            }
1351 
        }
1352 
        Ok(rep)
1353 
    }
1354 
}
1355
1356 
/// Where two versions of a file first part, for a verification report.
1357 
fn first_difference(have: &[u8], want: &[u8]) -> String {
1358 
    let (h, w): (Vec<&[u8]>, Vec<&[u8]>) = (
1359 
        have.split(|b| *b == b'\n').collect(),
1360 
        want.split(|b| *b == b'\n').collect(),
1361 
    );
1362 
    let i = h
1363 
        .iter()
1364 
        .zip(&w)
1365 
        .position(|(a, b)| a != b)
1366 
        .unwrap_or(h.len().min(w.len()));
1367 
    let show = |l: Option<&&[u8]>| {
1368 
        l.map(|l| String::from_utf8_lossy(&l[..l.len().min(60)]).into_owned())
1369 
            .unwrap_or_else(|| "(end)".into())
1370 
    };
1371 
    format!(
1372 
        "line {}: has {:?}, Git has {:?} ({} lines, Git has {})",
1373 
        i + 1,
1374 
        show(h.get(i)),
1375 
        show(w.get(i)),
1376 
        h.len(),
1377 
        w.len()
1378 
    )
1379 
}