OXYGENHUB
◆ agents-in-readme oxygen / crates / oxy-core / src / repo / git.rs
76.8 KiB · Rust · last changed oxy git import says where it is 1 month ago source blame history raw
79ab8528 Andy McCall ↶ 1 
//! Git bridge (spec §23), export side: each state the current bookmark
2 
//! passed through becomes a Git commit through `git fast-import`, with the
3 
//! state's full tree and the previous state's commit as parent. A state
4 
//! that added exactly one change carries that change's message, author,
5 
//! and `Change-Id`/`Oxy-Hash` trailers; every commit carries `Oxy-State`.
6 
//!
7 
//! Marks make re-exports incremental: `.oxy/git/<target>/git-marks` is
8 
//! git's own marks file and `oxy-marks` maps state refs to marks.
9
10 
use super::*;
11 
use std::process::{Command, Stdio};
62794ecc Andy McCall ↶ 12 
use std::sync::Arc;
79ab8528 Andy McCall ↶ 13
14 
#[derive(Debug, Clone, Default, PartialEq, Eq)]
c3069537 Andy McCall ↶ 15 
pub struct GitImportReport {
16 
    pub commits: usize,
17 
    pub merges: usize,
4dee277c Andy McCall ↶ 18 
    /// Gitlink entries imported as subrepo nodes with Git pins.
19 
    pub submodules: usize,
c3069537 Andy McCall ↶ 20 
    pub heads: usize,
21 
}
22
23 
/// A running `git cat-file --batch`, for reading many blobs cheaply.
24 
struct CatFile {
25 
    child: std::process::Child,
26 
    stdin: std::process::ChildStdin,
27 
    stdout: std::io::BufReader<std::process::ChildStdout>,
28 
}
29
30 
impl CatFile {
31 
    fn start(git_dir: &Path) -> Result<CatFile> {
32 
        let mut child = Command::new("git")
33 
            .arg("-C")
34 
            .arg(git_dir)
35 
            .args(["cat-file", "--batch"])
36 
            .stdin(Stdio::piped())
37 
            .stdout(Stdio::piped())
38 
            .stderr(Stdio::null())
39 
            .spawn()
40 
            .map_err(|e| RepoError::Msg(format!("running git cat-file: {e}")))?;
41 
        let stdin = child.stdin.take().unwrap();
42 
        let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
75167672 Andy McCall 43 
        Ok(CatFile {
44 
            child,
45 
            stdin,
46 
            stdout,
47 
        })
c3069537 Andy McCall ↶ 48 
    }
49
50 
    fn read(&mut self, sha: &str) -> Result<Vec<u8>> {
51 
        use std::io::{BufRead, Read};
52 
        writeln!(self.stdin, "{sha}")?;
53 
        self.stdin.flush()?;
54 
        let mut header = String::new();
55 
        self.stdout.read_line(&mut header)?;
56 
        let parts: Vec<&str> = header.trim_end().split(' ').collect();
57 
        if parts.len() != 3 {
58 
            return fail(format!("git cat-file: object {sha} not found"));
59 
        }
75167672 Andy McCall 60 
        let size: usize = parts[2]
61 
            .parse()
62 
            .map_err(|_| RepoError::Msg("git cat-file: bad size".into()))?;
c3069537 Andy McCall ↶ 63 
        let mut body = vec![0u8; size];
64 
        self.stdout.read_exact(&mut body)?;
65 
        let mut nl = [0u8; 1];
66 
        self.stdout.read_exact(&mut nl)?;
67 
        Ok(body)
68 
    }
69 
}
70
71 
impl Drop for CatFile {
72 
    fn drop(&mut self) {
73 
        let _ = self.child.kill();
74 
        let _ = self.child.wait();
75 
    }
76 
}
77
2306a4d2 Andy McCall ↶ 78 
/// One path a commit changes against its first parent: its new mode and
79 
/// blob, or `None` when the commit deletes it.
80 
type TreeDiff = Vec<(String, Option<(String, String)>)>;
81
82 
/// A running `git diff-tree --stdin`, fed every `commit parent` pair up
83 
/// front, whose diffs are read in the same order as they are needed: a
84 
/// commit's tree is its first parent's plus its diff, at a cost that
85 
/// follows what it changes rather than the whole tree (oxygen#25).
86 
struct DiffTree {
87 
    child: std::process::Child,
88 
    stdout: std::io::BufReader<std::process::ChildStdout>,
89 
    feeder: Option<std::thread::JoinHandle<()>>,
90 
    /// The header read at the end of the previous diff.
91 
    next: Option<String>,
92 
}
93
94 
impl DiffTree {
95 
    fn start(git_dir: &Path, pairs: Vec<(String, String)>) -> Result<DiffTree> {
96 
        let mut child = Command::new("git")
97 
            .arg("-C")
98 
            .arg(git_dir)
99 
            .args([
100 
                "diff-tree",
101 
                "--stdin",
102 
                "--always",
103 
                "-r",
104 
                "-z",
105 
                "--no-renames",
106 
            ])
107 
            .stdin(Stdio::piped())
108 
            .stdout(Stdio::piped())
109 
            .stderr(Stdio::null())
110 
            .spawn()
111 
            .map_err(|e| RepoError::Msg(format!("running git diff-tree: {e}")))?;
112 
        let mut stdin = child.stdin.take().unwrap();
113 
        let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
114 
        // A thread writes, so git never waits on a full pipe for us to read.
115 
        let feeder = std::thread::spawn(move || {
116 
            let mut w = std::io::BufWriter::new(&mut stdin);
117 
            for (c, p) in pairs {
118 
                if writeln!(w, "{c} {p}").is_err() {
119 
                    return;
120 
                }
121 
            }
122 
            let _ = w.flush();
123 
        });
124 
        Ok(DiffTree {
125 
            child,
126 
            stdout,
127 
            feeder: Some(feeder),
128 
            next: None,
129 
        })
130 
    }
131
132 
    fn field(&mut self) -> Result<Option<String>> {
133 
        use std::io::BufRead;
134 
        let mut buf = Vec::new();
135 
        if self.stdout.read_until(0, &mut buf)? == 0 {
136 
            return Ok(None);
137 
        }
138 
        if buf.last() == Some(&0) {
139 
            buf.pop();
140 
        }
141 
        Ok(Some(String::from_utf8_lossy(&buf).into_owned()))
142 
    }
143
144 
    /// The diff of `sha` against the parent it was fed with.
145 
    fn diff(&mut self, sha: &str) -> Result<TreeDiff> {
146 
        let header = match self.next.take() {
147 
            Some(h) => Some(h),
148 
            None => self.field()?,
149 
        };
150 
        if header.as_deref() != Some(sha) {
151 
            return fail(format!("git diff-tree: expected {sha}, got {header:?}"));
152 
        }
153 
        let mut out = Vec::new();
154 
        while let Some(f) = self.field()? {
155 
            let Some(meta) = f.strip_prefix(':') else {
156 
                self.next = Some(f);
157 
                break;
158 
            };
159 
            let Some(path) = self.field()? else {
160 
                return fail("git diff-tree: record without a path");
161 
            };
162 
            // `oldmode newmode oldsha newsha status`
163 
            let m: Vec<&str> = meta.split(' ').collect();
164 
            if m.len() != 5 {
165 
                return fail(format!("git diff-tree: bad record {f:?}"));
166 
            }
167 
            let entry = if m[4].starts_with('D') {
168 
                None
169 
            } else {
170 
                Some((m[1].to_string(), m[3].to_string()))
171 
            };
172 
            out.push((path, entry));
173 
        }
174 
        Ok(out)
175 
    }
176 
}
177
178 
impl Drop for DiffTree {
179 
    fn drop(&mut self) {
180 
        let _ = self.child.kill();
181 
        let _ = self.child.wait();
182 
        if let Some(f) = self.feeder.take() {
183 
            let _ = f.join();
184 
        }
185 
    }
186 
}
187
c3069537 Andy McCall ↶ 188 
/// One commit as `git log` describes it.
189 
struct GitCommit {
190 
    sha: String,
191 
    parents: Vec<String>,
192 
    author_secs: u64,
193 
    author: String,
194 
    message: String,
195 
}
196
197 
fn list_commits(git_dir: &Path, branch: &str) -> Result<Vec<GitCommit>> {
198 
    let out = Command::new("git")
199 
        .arg("-C")
200 
        .arg(git_dir)
75167672 Andy McCall 201 
        .args([
202 
            "log",
203 
            "--reverse",
204 
            "--topo-order",
205 
            "--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e",
206 
            branch,
207 
        ])
c3069537 Andy McCall ↶ 208 
        .output()
209 
        .map_err(|e| RepoError::Msg(format!("running git log: {e}")))?;
210 
    if !out.status.success() {
75167672 Andy McCall 211 
        return fail(format!(
212 
            "git log {branch} failed: {}",
213 
            String::from_utf8_lossy(&out.stderr).trim()
214 
        ));
c3069537 Andy McCall ↶ 215 
    }
216 
    let text = String::from_utf8_lossy(&out.stdout);
217 
    let mut commits = Vec::new();
218 
    for record in text.split('\x1e') {
219 
        let record = record.trim_start_matches('\n');
220 
        if record.trim().is_empty() {
221 
            continue;
222 
        }
223 
        let f: Vec<&str> = record.splitn(5, '\0').collect();
224 
        if f.len() != 5 {
225 
            return fail("unexpected git log record");
226 
        }
227 
        commits.push(GitCommit {
228 
            sha: f[0].to_string(),
229 
            parents: f[1].split_whitespace().map(str::to_string).collect(),
230 
            author_secs: f[2].parse().unwrap_or(0),
231 
            author: f[3].to_string(),
232 
            message: f[4].trim_end().to_string(),
233 
        });
234 
    }
235 
    Ok(commits)
236 
}
237
238 
/// `path → (mode, blob sha)` for a commit's tree.
239 
fn list_tree(git_dir: &Path, sha: &str) -> Result<BTreeMap<String, (String, String)>> {
240 
    let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?;
241 
    let mut out = BTreeMap::new();
242 
    for entry in text.split('\0') {
243 
        if entry.is_empty() {
244 
            continue;
245 
        }
75167672 Andy McCall 246 
        let Some((meta, path)) = entry.split_once('\t') else {
247 
            continue;
248 
        };
c3069537 Andy McCall ↶ 249 
        let m: Vec<&str> = meta.split(' ').collect();
250 
        if m.len() == 3 {
251 
            out.insert(path.to_string(), (m[0].to_string(), m[2].to_string()));
252 
        }
253 
    }
254 
    Ok(out)
255 
}
256
257 
/// The ChangeId for a Git commit: its `Change-Id` trailer if it has a
258 
/// 32-hex one, else BLAKE3-128 of the commit hash (spec §23).
259 
fn change_id_for(sha: &str, message: &str) -> ChangeId {
260 
    for line in message.lines().rev() {
261 
        if let Some(v) = line.strip_prefix("Change-Id:") {
262 
            if let Some(id) = ChangeId::from_hex(v.trim()) {
263 
                return id;
264 
            }
265 
        }
266 
    }
267 
    let h = blake3::hash(sha.as_bytes());
268 
    ChangeId(h.as_bytes()[..16].try_into().unwrap())
269 
}
270
30ea9c1e Andy McCall ↶ 271 
/// Git commits with a known Oxygen counterpart.
272 
struct GitKnown {
273 
    /// Imported commit → its change.
274 
    imported: HashMap<String, ChangeHash>,
275 
    /// Exported commit (or a fetched tip) → the bookmark state it equals.
276 
    exported: HashMap<String, ObjectRef>,
277 
}
278
c3069537 Andy McCall ↶ 279 
#[derive(Debug, Clone, Default, PartialEq, Eq)]
79ab8528 Andy McCall ↶ 280 
pub struct GitExportReport {
281 
    pub exported: usize,
282 
    pub skipped: usize,
4dee277c Andy McCall ↶ 283 
    /// Subrepo nodes with Oxygen pins, which Git cannot represent.
284 
    pub skipped_subrepos: usize,
79ab8528 Andy McCall ↶ 285 
    pub branch: String,
286 
    pub heads: usize,
287 
}
288
fd280228 Andy McCall ↶ 289 
/// A merge bringing more commits than this sheds its base's graphs first.
290 
const MERGE_SHED_COMMITS: usize = 100;
291 
/// A kept state with more built graphs than this sheds them.
292 
const KEPT_GRAPHS: usize = 2000;
293
c3d2ef61 Andy McCall ↶ 294 
/// Text files of a merge rendered this many at a time (oxygen#25): a
295 
/// batch's graphs are built beside the state and dropped once rendered,
296 
/// so what is held for the whole tree is the renderings, not the graphs
297 
/// with every deleted line of their history.
298 
const MERGE_BATCH: usize = 500;
299
300 
thread_local! {
301 
    /// A smaller batch for this thread's imports: tests set 1, so a small
302 
    /// merge takes the batched path.
303 
    static MERGE_BATCH_OVERRIDE: std::cell::Cell<Option<usize>> = const { std::cell::Cell::new(None) };
304 
}
305
306 
#[cfg(test)]
307 
pub(crate) fn set_merge_batch_for_tests(n: Option<usize>) {
308 
    MERGE_BATCH_OVERRIDE.with(|c| c.set(n));
309 
}
310
311 
fn merge_batch() -> usize {
312 
    MERGE_BATCH_OVERRIDE
313 
        .with(|c| c.get())
314 
        .filter(|n| *n > 0)
315 
        .unwrap_or(MERGE_BATCH)
316 
}
317
318 
impl Repo {
319 
    /// As `State::materialize_except`, for a tree whose graphs would not
320 
    /// fit in memory at once. The few files the change builder reads
321 
    /// graphs for (conflicted ones, and renderings whose lines are not
322 
    /// one per live atom) are built in the state afterwards.
323 
    fn materialize_batched(
324 
        &self,
325 
        base: &mut State,
326 
        need: &[crate::ids::NodeId],
327 
        skip: &HashSet<crate::ids::NodeId>,
328 
    ) -> Result<crate::state::Snapshot> {
329 
        use crate::graph::Materialized;
330 
        use crate::state::{FileContent, FileState};
331 
        use rayon::prelude::*;
332 
        let view = base.tree().view();
333 
        let mut rendered: HashMap<crate::ids::NodeId, (Vec<u8>, Materialized)> =
334 
            HashMap::with_capacity(need.len());
335 
        let mut keep_built: Vec<crate::ids::NodeId> = Vec::new();
336 
        for batch in need.chunks(merge_batch()) {
337 
            let t = std::time::Instant::now();
338 
            let graphs = self.build_graphs_detached(base, batch)?;
339 
            crate::timing::add("render: build graphs, batch", t.elapsed());
340 
            let t = std::time::Instant::now();
341 
            let batch_rendered: Vec<(crate::ids::NodeId, Vec<u8>, Materialized)> = graphs
342 
                .par_iter()
343 
                .map(|(node, g)| {
344 
                    let (bytes, m) = base.render_graph(g);
345 
                    (*node, bytes, m)
346 
                })
347 
                .collect();
348 
            drop(graphs);
349 
            crate::timing::add("render: render, batch", t.elapsed());
350 
            for (node, bytes, m) in batch_rendered {
351 
                if m.is_conflicted()
352 
                    || m.live_atoms().len() != crate::diff::split_lines(&bytes).len()
353 
                {
354 
                    keep_built.push(node);
355 
                }
356 
                rendered.insert(node, (bytes, m));
357 
            }
358 
        }
359 
        if !keep_built.is_empty() {
360 
            self.load_graphs_of(base, &keep_built)?;
361 
        }
362 
        let mut files = BTreeMap::new();
363 
        let mut unrendered = BTreeSet::new();
364 
        for (path, e) in view.entries.iter() {
365 
            let f = if e.kind == NodeKind::File && !e.mode.opaque {
366 
                match rendered.remove(&e.node) {
367 
                    Some((bytes, materialized)) => FileState {
368 
                        node: e.node,
369 
                        kind: e.kind,
370 
                        mode: e.mode,
371 
                        content: FileContent::Text {
372 
                            bytes,
373 
                            materialized,
374 
                        },
375 
                    },
376 
                    None if skip.contains(&e.node) => {
377 
                        unrendered.insert(path.clone());
378 
                        FileState {
379 
                            node: e.node,
380 
                            kind: e.kind,
381 
                            mode: e.mode,
382 
                            content: FileContent::Text {
383 
                                bytes: Vec::new(),
384 
                                materialized: Materialized::default(),
385 
                            },
386 
                        }
387 
                    }
388 
                    None => {
389 
                        // Neither asked for nor known unchanged: render it
390 
                        // from a built graph, as the whole-tree path does.
391 
                        self.load_graphs_of(base, &[e.node])?;
392 
                        base.entry_state(e)
393 
                    }
394 
                }
395 
            } else {
396 
                base.entry_state(e)
397 
            };
398 
            files.insert(path.clone(), f);
399 
        }
400 
        Ok(crate::state::Snapshot {
401 
            tree: view,
402 
            files,
403 
            unrendered,
404 
        })
405 
    }
406 
}
407
79ab8528 Andy McCall ↶ 408 
fn git(dir: &Path, args: &[&str]) -> Result<String> {
75167672 Andy McCall 409 
    let out = Command::new("git")
410 
        .arg("-C")
411 
        .arg(dir)
412 
        .args(args)
413 
        .output()
414 
        .map_err(|e| RepoError::Msg(format!("running git: {e}")))?;
79ab8528 Andy McCall ↶ 415 
    if !out.status.success() {
75167672 Andy McCall 416 
        return fail(format!(
417 
            "git {} failed: {}",
418 
            args.join(" "),
419 
            String::from_utf8_lossy(&out.stderr).trim()
420 
        ));
79ab8528 Andy McCall ↶ 421 
    }
422 
    Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
423 
}
424
425 
impl Repo {
426 
    fn git_marks_dir(&self, git_dir: &Path) -> Result<PathBuf> {
75167672 Andy McCall 427 
        let canonical = git_dir
428 
            .canonicalize()
429 
            .unwrap_or_else(|_| git_dir.to_path_buf());
79ab8528 Andy McCall ↶ 430 
        let key = hash_object(ObjectType::Chunk, canonical.to_string_lossy().as_bytes()).short(16);
431 
        let dir = self.oxy.join("git").join(key);
432 
        fs::create_dir_all(&dir)?;
433 
        Ok(dir)
434 
    }
435
30ea9c1e Andy McCall ↶ 436 
    /// `state ref → Git commit sha` for bookmark states that equal a Git
437 
    /// commit's tree (recorded by import and fetch).
438 
    fn read_git_states(path: &Path) -> Result<BTreeMap<ObjectRef, String>> {
439 
        let mut out = BTreeMap::new();
440 
        if let Ok(text) = fs::read_to_string(path) {
441 
            for line in text.lines() {
442 
                if let Some((h, sha)) = line.split_once(' ') {
443 
                    if let Some(h) = ObjectRef::from_hex(h) {
444 
                        out.insert(h, sha.trim().to_string());
445 
                    }
446 
                }
447 
            }
448 
        }
449 
        Ok(out)
450 
    }
451
452 
    fn record_git_state(&self, git_dir: &Path, state: &ObjectRef, sha: &str) -> Result<()> {
453 
        let path = self.git_marks_dir(git_dir)?.join("git-states");
454 
        let mut states = Self::read_git_states(&path)?;
455 
        states.insert(*state, sha.to_string());
75167672 Andy McCall 456 
        let text: String = states
457 
            .iter()
458 
            .map(|(h, s)| format!("{} {s}\n", h.to_hex()))
459 
            .collect();
30ea9c1e Andy McCall ↶ 460 
        write_atomic(&path, text.as_bytes())
461 
    }
462
463 
    /// The bare mirror this repository keeps for the Git remote `url`.
464 
    fn git_mirror(&self, url: &str) -> Result<PathBuf> {
465 
        let key = hash_object(ObjectType::Chunk, url.as_bytes()).short(16);
466 
        let dir = self.oxy.join("git").join(key).join("mirror.git");
467 
        if !dir.join("HEAD").exists() {
468 
            fs::create_dir_all(&dir)?;
469 
            git(&dir, &["init", "--bare", "-q"])?;
470 
        }
471 
        Ok(dir)
472 
    }
473
474 
    /// Git commits already imported into the bookmark, by sha.
7891e21c Andy McCall ↶ 475 
    pub(crate) fn imported_commits(&self) -> Result<HashMap<String, ChangeHash>> {
30ea9c1e Andy McCall ↶ 476 
        let mut map = HashMap::new();
477 
        for e in self.log()? {
478 
            if e.is_working_copy {
479 
                continue;
480 
            }
75167672 Andy McCall 481 
            if let Some(sha) = e
482 
                .change
483 
                .message
484 
                .lines()
485 
                .rev()
486 
                .find_map(|l| l.strip_prefix("Git-Commit: "))
487 
            {
30ea9c1e Andy McCall ↶ 488 
                map.insert(sha.trim().to_string(), e.hash);
489 
            }
490 
        }
491 
        Ok(map)
492 
    }
493
494 
    /// Bring the mirror's `branch` up to date with the Git remote `url`.
495 
    /// A remote without that branch yet leaves the mirror as it is.
496 
    fn git_mirror_fetch(&self, mirror: &Path, url: &str, branch: &str) -> Result<()> {
497 
        let spec = format!("+refs/heads/{branch}:refs/heads/{branch}");
498 
        match git(mirror, &["fetch", "-q", url, &spec]) {
499 
            Ok(_) => Ok(()),
500 
            Err(e) => {
501 
                let ls = git(mirror, &["ls-remote", "--heads", url, branch]).unwrap_or_default();
75167672 Andy McCall 502 
                if ls.trim().is_empty() {
503 
                    Ok(())
504 
                } else {
505 
                    Err(e)
506 
                }
30ea9c1e Andy McCall ↶ 507 
            }
508 
        }
509 
    }
510
511 
    /// Fetch `branch` of the Git repository at `url` (a path or a Git URL)
512 
    /// through a mirror and import the commits not yet in the bookmark
513 
    /// (spec §23). Works on a repository that already has history: the
514 
    /// new changes join the bookmark like a pull.
515 
    pub fn git_fetch(&self, url: &str, branch: &str) -> Result<GitImportReport> {
516 
        let mirror = self.git_mirror(url)?;
517 
        self.git_mirror_fetch(&mirror, url, branch)?;
75167672 Andy McCall 518 
        if git(
519 
            &mirror,
520 
            &[
521 
                "rev-parse",
522 
                "--verify",
523 
                "-q",
524 
                &format!("refs/heads/{branch}"),
525 
            ],
526 
        )
527 
        .is_err()
528 
        {
30ea9c1e Andy McCall ↶ 529 
            return fail(format!("{url} has no branch {branch:?}"));
530 
        }
58dfe0e1 Andy McCall ↶ 531 
        self.import_git_commits(
532 
            &mirror,
533 
            branch,
534 
            &format!("oxy git fetch {url} {branch}"),
535 
            &mut |_| {},
536 
        )
30ea9c1e Andy McCall ↶ 537 
    }
538
539 
    /// Export the bookmark to `branch` of the Git repository at `url`
540 
    /// through the mirror and push it. The remote's tip must already have
541 
    /// been fetched (`git_fetch`), so the push fast-forwards.
542 
    pub fn git_push(&self, url: &str, branch: &str) -> Result<GitExportReport> {
543 
        let mirror = self.git_mirror(url)?;
544 
        self.git_mirror_fetch(&mirror, url, branch)?;
75167672 Andy McCall 545 
        if let Ok(tip) = git(
546 
            &mirror,
547 
            &[
548 
                "rev-parse",
549 
                "--verify",
550 
                "-q",
551 
                &format!("refs/heads/{branch}"),
552 
            ],
553 
        ) {
30ea9c1e Andy McCall ↶ 554 
            let known = self.git_known(&mirror)?;
555 
            if !known.imported.contains_key(&tip) && !known.exported.contains_key(&tip) {
75167672 Andy McCall 556 
                return fail(format!(
557 
                    "{url} {branch} has commits not yet fetched; run `oxy git fetch {url}` first"
558 
                ));
30ea9c1e Andy McCall ↶ 559 
            }
560 
        }
561 
        let report = self.git_export_as(&mirror, branch)?;
562 
        let spec = format!("refs/heads/{branch}:refs/heads/{branch}");
563 
        git(&mirror, &["push", "-q", url, &spec])?;
564 
        Ok(report)
565 
    }
566
79ab8528 Andy McCall ↶ 567 
    fn read_oxy_marks(path: &Path) -> Result<BTreeMap<ObjectRef, u64>> {
568 
        let mut out = BTreeMap::new();
569 
        if let Ok(text) = fs::read_to_string(path) {
570 
            for line in text.lines() {
571 
                if let Some((h, m)) = line.split_once(' ') {
572 
                    if let (Some(h), Ok(m)) = (ObjectRef::from_hex(h), m.parse()) {
573 
                        out.insert(h, m);
574 
                    }
575 
                }
576 
            }
577 
        }
578 
        Ok(out)
579 
    }
580
581 
    /// Export the current bookmark to the Git repository at `git_dir`,
582 
    /// creating a bare one if nothing is there. Each state the bookmark
583 
    /// passed through (per the operation log) becomes one commit whose tree
584 
    /// is that state, so the Git branch is a linear history of full trees.
585 
    pub fn git_export(&self, git_dir: &Path) -> Result<GitExportReport> {
30ea9c1e Andy McCall ↶ 586 
        let bookmark = self.config.bookmark.clone();
587 
        self.git_export_as(git_dir, &bookmark)
588 
    }
589
590 
    /// `git_export` to the Git branch `branch`. A state that came from a
591 
    /// Git commit (`oxy git fetch`) is not re-exported: later states chain
592 
    /// onto that commit, so a push back to the Git remote fast-forwards.
593 
    pub fn git_export_as(&self, git_dir: &Path, branch: &str) -> Result<GitExportReport> {
79ab8528 Andy McCall ↶ 594 
        if !git_dir.join("HEAD").exists() && !git_dir.join(".git").exists() {
595 
            fs::create_dir_all(git_dir)?;
596 
            git(git_dir, &["init", "--bare", "-q"])?;
597 
        }
598 
        let marks_dir = self.git_marks_dir(git_dir)?;
599 
        let git_marks = marks_dir.join("git-marks");
600 
        let oxy_marks_path = marks_dir.join("oxy-marks");
601 
        let mut marks = Self::read_oxy_marks(&oxy_marks_path)?;
30ea9c1e Andy McCall ↶ 602 
        let git_states = Self::read_git_states(&marks_dir.join("git-states"))?;
79ab8528 Andy McCall ↶ 603 
        let mut next_mark = marks.values().copied().max().unwrap_or(0) + 1;
604
605 
        let bookmark = self.config.bookmark.clone();
606 
        let mut stream: Vec<u8> = Vec::new();
75167672 Andy McCall 607 
        let mut report = GitExportReport {
608 
            branch: branch.to_string(),
609 
            heads: self.bookmark_heads()?.len(),
610 
            ..Default::default()
611 
        };
79ab8528 Andy McCall ↶ 612
613 
        // States along the bookmark, oldest first, skipping unchanged and pruned ones.
614 
        let mut ops = self.op_log()?;
615 
        ops.reverse();
616 
        let mut prev: Option<(ObjectRef, Vec<ChangeHash>)> = None;
30ea9c1e Andy McCall ↶ 617 
        // The fast-import parent of the next commit: `:mark` or a commit sha.
618 
        let mut last_parent: Option<String> = None;
79ab8528 Andy McCall ↶ 619 
        for (_, op) in &ops {
75167672 Andy McCall 620 
            let Some(state_ref) = op.view.bookmarks.get(&bookmark).copied() else {
621 
                continue;
622 
            };
79ab8528 Andy McCall ↶ 623 
            if prev.as_ref().is_some_and(|(p, _)| *p == state_ref) {
624 
                continue;
625 
            }
75167672 Andy McCall 626 
            let Ok(state) = self.store.get_state(&state_ref) else {
627 
                continue;
628 
            };
79ab8528 Andy McCall ↶ 629 
            if state.heads.is_empty() {
630 
                prev = Some((state_ref, vec![]));
631 
                continue;
632 
            }
30ea9c1e Andy McCall ↶ 633 
            if let Some(sha) = git_states.get(&state_ref) {
634 
                report.skipped += 1;
635 
                last_parent = Some(sha.clone());
636 
                prev = Some((state_ref, state.heads.clone()));
637 
                continue;
638 
            }
79ab8528 Andy McCall ↶ 639 
            if let Some(m) = marks.get(&state_ref) {
640 
                report.skipped += 1;
30ea9c1e Andy McCall ↶ 641 
                last_parent = Some(format!(":{m}"));
79ab8528 Andy McCall ↶ 642 
                prev = Some((state_ref, state.heads.clone()));
643 
                continue;
644 
            }
645 
            let prev_heads = prev.as_ref().map(|(_, h)| h.clone()).unwrap_or_default();
75167672 Andy McCall 646 
            let added = self
647 
                .negotiate(&prev_heads, &state.heads)
648 
                .unwrap_or_default();
79ab8528 Andy McCall ↶ 649 
            let loaded = self.load_state(&state.heads)?;
650 
            let snapshot = loaded.materialize();
651 
            let (author, secs, mut message) = match added.as_slice() {
652 
                [h] => {
653 
                    let c = self.store.get_change(h)?;
654 
                    let mut m = c.message.clone();
655 
                    if !m.ends_with('\n') {
656 
                        m.push('\n');
657 
                    }
75167672 Andy McCall 658 
                    m.push_str(&format!(
659 
                        "\nChange-Id: {}\nOxy-Hash: {}\n",
660 
                        c.change_id.to_hex(),
661 
                        h.to_hex()
662 
                    ));
79ab8528 Andy McCall ↶ 663 
                    (self.git_author(&c), c.hlc.wall_ms / 1000, m)
664 
                }
75167672 Andy McCall 665 
                [] => (
666 
                    "oxy <oxy@localhost>".to_string(),
667 
                    op.hlc.wall_ms / 1000,
668 
                    format!("oxy: {}\n\n", op.command),
669 
                ),
79ab8528 Andy McCall ↶ 670 
                many => {
671 
                    let mut m = format!("oxy: {} ({} changes)\n\n", op.command, many.len());
672 
                    for h in many {
673 
                        let c = self.store.get_change(h)?;
75167672 Andy McCall 674 
                        m.push_str(&format!(
675 
                            "- {} {}\n",
676 
                            c.change_id.short(8),
677 
                            c.message.lines().next().unwrap_or("")
678 
                        ));
79ab8528 Andy McCall ↶ 679 
                    }
75167672 Andy McCall 680 
                    m.push_str(&format!(
681 
                        "\nOxy-Changes: {}\n",
682 
                        many.iter()
683 
                            .map(|h| h.to_hex())
684 
                            .collect::<Vec<_>>()
685 
                            .join(" ")
686 
                    ));
79ab8528 Andy McCall ↶ 687 
                    ("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, m)
688 
                }
689 
            };
690 
            message.push_str(&format!("Oxy-State: {}\n", state_ref.to_hex()));
691 
            if snapshot.is_conflicted() {
692 
                message.push_str("Oxy-Conflicted: true\n");
693 
            }
694 
            let mark = next_mark;
695 
            next_mark += 1;
30ea9c1e Andy McCall ↶ 696 
            let parents: Vec<String> = last_parent.clone().into_iter().collect();
75167672 Andy McCall 697 
            self.write_commit(
698 
                &mut stream,
699 
                "refs/oxy/export",
700 
                mark,
701 
                &author,
702 
                secs,
703 
                &message,
704 
                &parents,
705 
                &snapshot,
706 
            )?;
79ab8528 Andy McCall ↶ 707 
            marks.insert(state_ref, mark);
30ea9c1e Andy McCall ↶ 708 
            last_parent = Some(format!(":{mark}"));
79ab8528 Andy McCall ↶ 709 
            prev = Some((state_ref, state.heads.clone()));
710 
            report.exported += 1;
711 
        }
30ea9c1e Andy McCall ↶ 712 
        if let Some(p) = &last_parent {
713 
            stream.extend_from_slice(format!("reset refs/heads/{branch}\nfrom {p}\n\n").as_bytes());
79ab8528 Andy McCall ↶ 714 
        }
132b884b Andy McCall ↶ 715 
        // Tags: a tag whose state was exported as a bookmark state points at
716 
        // that commit; otherwise its state gets a commit of its own.
717 
        for t in self.tags()? {
30ea9c1e Andy McCall ↶ 718 
            let target = match (git_states.get(&t.state), marks.get(&t.state)) {
719 
                (Some(sha), _) => sha.clone(),
720 
                (None, Some(m)) => format!(":{m}"),
721 
                (None, None) => {
132b884b Andy McCall ↶ 722 
                    let loaded = self.load_state(&t.heads)?;
723 
                    let snapshot = loaded.materialize();
724 
                    let mark = next_mark;
725 
                    next_mark += 1;
75167672 Andy McCall 726 
                    let message = format!(
727 
                        "oxy: tag {}\n\n{}Oxy-State: {}\n",
728 
                        t.name,
729 
                        if t.message.is_empty() {
730 
                            String::new()
731 
                        } else {
732 
                            format!("{}\n\n", t.message)
733 
                        },
734 
                        t.state.to_hex()
735 
                    );
736 
                    self.write_commit(
737 
                        &mut stream,
738 
                        "refs/oxy/export",
739 
                        mark,
740 
                        "oxy <oxy@localhost>",
741 
                        Hlc::now().wall_ms / 1000,
742 
                        &message,
743 
                        &[],
744 
                        &snapshot,
745 
                    )?;
132b884b Andy McCall ↶ 746 
                    marks.insert(t.state, mark);
747 
                    report.exported += 1;
30ea9c1e Andy McCall ↶ 748 
                    format!(":{mark}")
132b884b Andy McCall ↶ 749 
                }
750 
            };
75167672 Andy McCall 751 
            stream.extend_from_slice(
752 
                format!("reset refs/tags/{}\nfrom {target}\n\n", t.name).as_bytes(),
753 
            );
132b884b Andy McCall ↶ 754 
        }
755 
        stream.extend_from_slice(b"done\n");
79ab8528 Andy McCall ↶ 756
757 
        let mut cmd = Command::new("git");
758 
        // `--force`: the scratch ref may jump between unrelated states.
75167672 Andy McCall 759 
        cmd.arg("-C")
760 
            .arg(git_dir)
761 
            .args(["fast-import", "--quiet", "--done", "--force"])
762 
            .arg(format!("--export-marks={}", git_marks.display()));
79ab8528 Andy McCall ↶ 763 
        if git_marks.exists() {
764 
            cmd.arg(format!("--import-marks={}", git_marks.display()));
765 
        }
75167672 Andy McCall 766 
        let mut child = cmd
767 
            .stdin(Stdio::piped())
768 
            .stdout(Stdio::null())
769 
            .stderr(Stdio::piped())
770 
            .spawn()
771 
            .map_err(|e| RepoError::Msg(format!("running git fast-import: {e}")))?;
79ab8528 Andy McCall ↶ 772 
        child.stdin.take().unwrap().write_all(&stream)?;
773 
        let out = child.wait_with_output()?;
774 
        if !out.status.success() {
75167672 Andy McCall 775 
            return fail(format!(
776 
                "git fast-import failed: {}",
777 
                String::from_utf8_lossy(&out.stderr).trim()
778 
            ));
79ab8528 Andy McCall ↶ 779 
        }
75167672 Andy McCall 780 
        let text: String = marks
781 
            .iter()
782 
            .map(|(h, m)| format!("{} {m}\n", h.to_hex()))
783 
            .collect();
79ab8528 Andy McCall ↶ 784 
        write_atomic(&oxy_marks_path, text.as_bytes())?;
785 
        let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]);
4dee277c Andy McCall ↶ 786 
        report.skipped_subrepos = self.git_export_skipped.replace(0);
79ab8528 Andy McCall ↶ 787 
        Ok(report)
788 
    }
789
c3069537 Andy McCall ↶ 790 
    /// Import branch `branch` of the Git repository at `git_dir` into this
791 
    /// (empty) repository's bookmark (spec §23): one change per commit, its
792 
    /// edits the difference between the union of its parents' states and its
793 
    /// tree, so merge commits become resolutions. Submodules are skipped.
794 
    pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result<GitImportReport> {
58dfe0e1 Andy McCall ↶ 795 
        self.import_git_with(git_dir, branch, &mut |_| {})
796 
    }
797
798 
    /// `import_git`, telling `progress` where it is every few hundred
799 
    /// commits: the count so far, the merges, and the commit in hand, so a
800 
    /// long import is never silent and a run that dies says where.
801 
    pub fn import_git_with(
802 
        &self,
803 
        git_dir: &Path,
804 
        branch: &str,
805 
        progress: crate::store::Progress<'_>,
806 
    ) -> Result<GitImportReport> {
c3069537 Andy McCall ↶ 807 
        if !self.bookmark_heads()?.is_empty() {
808 
            return fail("git import needs an empty bookmark; import into a fresh repository");
809 
        }
75167672 Andy McCall 810 
        self.import_git_commits(
811 
            git_dir,
812 
            branch,
813 
            &format!("oxy git import {} {branch}", git_dir.display()),
58dfe0e1 Andy McCall ↶ 814 
            progress,
75167672 Andy McCall 815 
        )
30ea9c1e Andy McCall ↶ 816 
    }
817
818 
    /// Import the commits of `branch` that the bookmark does not have yet,
819 
    /// then move the bookmark to the union of its heads and theirs.
75167672 Andy McCall 820 
    fn import_git_commits(
821 
        &self,
822 
        git_dir: &Path,
823 
        branch: &str,
824 
        command: &str,
58dfe0e1 Andy McCall ↶ 825 
        progress: crate::store::Progress<'_>,
75167672 Andy McCall 826 
    ) -> Result<GitImportReport> {
8f10a07a Andy McCall ↶ 827 
        let batch = self.store.begin_batch()?;
58dfe0e1 Andy McCall ↶ 828 
        let result = self.import_git_commits_batched(git_dir, branch, command, progress);
8f10a07a Andy McCall ↶ 829 
        batch.finish()?;
830 
        result
831 
    }
832
75167672 Andy McCall 833 
    fn import_git_commits_batched(
834 
        &self,
835 
        git_dir: &Path,
836 
        branch: &str,
837 
        command: &str,
58dfe0e1 Andy McCall ↶ 838 
        progress: crate::store::Progress<'_>,
75167672 Andy McCall 839 
    ) -> Result<GitImportReport> {
c3069537 Andy McCall ↶ 840 
        let commits = list_commits(git_dir, branch)?;
841 
        let mut cat = CatFile::start(git_dir)?;
842 
        let mut report = GitImportReport::default();
30ea9c1e Andy McCall ↶ 843 
        let mut known = self.git_known(git_dir)?;
844 
        let old_heads = self.bookmark_heads()?;
dfb23a3a Andy McCall ↶ 845 
        // The bookmark's heads as changes arrive: a change stops being a
846 
        // head when another depends on it, so no full state of every
847 
        // imported change is needed at the end.
848 
        let mut heads_now: BTreeSet<ChangeHash> = old_heads.iter().copied().collect();
958fb27d Andy McCall ↶ 849 
        // The state and Git tree of imported commits whose first-parent
850 
        // children are still to come, so a child starts from its parent's
851 
        // state instead of rebuilding it from history. The last child takes
dfb23a3a Andy McCall ↶ 852 
        // it; earlier ones clone it. At most `max_kept` are kept (a later
958fb27d Andy McCall ↶ 853 
        // child of an evicted one rebuilds).
dfb23a3a Andy McCall ↶ 854 
        // OXY_IMPORT_KEEP overrides it, for measuring.
855 
        let max_kept: usize = std::env::var("OXY_IMPORT_KEEP")
856 
            .ok()
857 
            .and_then(|v| v.parse().ok())
858 
            .unwrap_or(8);
958fb27d Andy McCall ↶ 859 
        let mut waiting: HashMap<&str, usize> = HashMap::new();
860 
        for gc in &commits {
861 
            if let Some(p1) = gc.parents.first() {
862 
                *waiting.entry(p1.as_str()).or_default() += 1;
863 
            }
864 
        }
865 
        type Kept = (State, BTreeMap<String, (String, String)>);
866 
        let mut kept: HashMap<String, Kept> = HashMap::new();
2306a4d2 Andy McCall ↶ 867 
        // Every commit to import that has a parent, diffed against its first.
868 
        let pairs: Vec<(String, String)> = commits
869 
            .iter()
870 
            .filter(|gc| {
871 
                !known.imported.contains_key(&gc.sha) && !known.exported.contains_key(&gc.sha)
872 
            })
873 
            .filter_map(|gc| Some((gc.sha.clone(), gc.parents.first()?.clone())))
874 
            .collect();
875 
        let mut diffs = DiffTree::start(git_dir, pairs)?;
876 
        let mut rebuilt = 0usize;
958fb27d Andy McCall ↶ 877 
        let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
878 
        let mut last: Option<String> = None;
c3069537 Andy McCall ↶ 879 
        let mut blob_cache: HashMap<String, Vec<u8>> = HashMap::new();
880 
        let mut cache_bytes: usize = 0;
958fb27d Andy McCall ↶ 881 
        // Time spent per phase over all commits (`OXY_TIMINGS=1`).
882 
        let mut spent = [std::time::Duration::ZERO; 6];
883 
        let mut clock = std::time::Instant::now();
fb5b6181 Andy McCall ↶ 884 
        // One line per commit on stderr with OXY_IMPORT_TRACE=1: where its
885 
        // base came from, what a merge added, the graphs its state holds,
886 
        // the kept states and the process's resident memory.
887 
        let trace = std::env::var_os("OXY_IMPORT_TRACE").is_some();
888 
        let rss_mb = || -> u64 {
889 
            std::fs::read_to_string("/proc/self/statm")
890 
                .ok()
891 
                .and_then(|s| s.split_whitespace().nth(1)?.parse::<u64>().ok())
d2e3c39d Andy McCall ↶ 892 
                .map(|pages| (pages * 4096) >> 20)
fb5b6181 Andy McCall ↶ 893 
                .unwrap_or(0)
894 
        };
895 
        let mut done = 0usize;
fd280228 Andy McCall ↶ 896 
        // Graphs turned back into history (see `State::shed_graphs`).
897 
        let mut shed = 0usize;
958fb27d Andy McCall ↶ 898 
        let mut lap = |i: usize, clock: &mut std::time::Instant| {
899 
            let now = std::time::Instant::now();
900 
            spent[i] += now - *clock;
901 
            *clock = now;
902 
        };
c3069537 Andy McCall ↶ 903
58dfe0e1 Andy McCall ↶ 904 
        let total = commits.len();
905 
        for (index, gc) in commits.iter().enumerate() {
30ea9c1e Andy McCall ↶ 906 
            if known.imported.contains_key(&gc.sha) || known.exported.contains_key(&gc.sha) {
907 
                continue;
58dfe0e1 Andy McCall ↶ 908 
            }
909 
            if index % 500 == 0 {
910 
                progress(&format!(
911 
                    "{index} of {total} commits, {} merges; at {} {}",
912 
                    report.merges,
913 
                    &gc.sha[..12.min(gc.sha.len())],
914 
                    gc.message
915 
                        .lines()
916 
                        .next()
917 
                        .unwrap_or("")
918 
                        .chars()
919 
                        .take(60)
920 
                        .collect::<String>()
921 
                ));
30ea9c1e Andy McCall ↶ 922 
            }
fb5b6181 Andy McCall ↶ 923 
            let mut base_how = "root";
958fb27d Andy McCall ↶ 924 
            // Base: union of every parent's state, starting from the first
925 
            // parent's kept state when there is one; its tree tells which
926 
            // paths this commit leaves alone.
927 
            let (mut base, prev_tree) = match gc.parents.first() {
928 
                Some(p1) => {
929 
                    let left = waiting.get_mut(p1.as_str()).map(|n| {
930 
                        *n -= 1;
931 
                        *n
932 
                    });
933 
                    let found = if left == Some(0) {
934 
                        kept_order.retain(|k| k != p1);
935 
                        kept.remove(p1.as_str())
936 
                    } else {
937 
                        kept.get(p1.as_str()).cloned()
fb5b6181 Andy McCall ↶ 938 
                    };
939 
                    let how = match (&found, left) {
940 
                        (Some(_), Some(0)) => "kept",
941 
                        (Some(_), _) => "cloned",
942 
                        (None, _) => "rebuilt",
958fb27d Andy McCall ↶ 943 
                    };
fb5b6181 Andy McCall ↶ 944 
                    if trace {
945 
                        base_how = how;
946 
                    }
958fb27d Andy McCall ↶ 947 
                    match found {
948 
                        Some((st, tree)) => (st, Some(tree)),
949 
                        None => {
b493d993 Andy McCall ↶ 950 
                            // History only: graphs are built for the files a
951 
                            // commit changes, so memory follows what changes,
952 
                            // not the whole tree.
953 
                            let mut st = State::new_history_only();
958fb27d Andy McCall ↶ 954 
                            self.add_git_ancestors(&mut st, git_dir, &[p1.as_str()], &known)?;
2306a4d2 Andy McCall ↶ 955 
                            // The parent's tree, so this commit still
956 
                            // renders only what it changes.
957 
                            rebuilt += 1;
958 
                            (st, Some(list_tree(git_dir, p1)?))
958fb27d Andy McCall ↶ 959 
                        }
960 
                    }
30ea9c1e Andy McCall ↶ 961 
                }
b493d993 Andy McCall ↶ 962 
                None => (State::new_history_only(), None),
c3069537 Andy McCall ↶ 963 
            };
958fb27d Andy McCall ↶ 964 
            // Nodes the other parents' changes touch: the first parent's
965 
            // tree cannot vouch for them.
966 
            let mut touched: HashSet<NodeId> = HashSet::new();
fb5b6181 Andy McCall ↶ 967 
            let mut merged = 0usize;
c3069537 Andy McCall ↶ 968 
            if gc.parents.len() > 1 {
969 
                report.merges += 1;
970 
                let mut args: Vec<String> = gc.parents[1..].to_vec();
971 
                args.push(format!("^{}", gc.parents[0]));
30ea9c1e Andy McCall ↶ 972 
                let refs: Vec<&str> = args.iter().map(String::as_str).collect();
fd280228 Andy McCall ↶ 973 
                // A merge of a long-lived branch brings thousands of
974 
                // changes: added to built graphs, which kept states share,
975 
                // they were copied and applied whole (vscode's import passed
976 
                // 14 GB on such merges). Shed the graphs first, so the
977 
                // changes go in as skeletons and only what this commit
978 
                // touches is built again.
979 
                if base.built_count() > 0 {
980 
                    let mut count_args = vec!["rev-list", "--count"];
981 
                    count_args.extend(refs.iter().copied());
982 
                    let others: usize = git(git_dir, &count_args)?.trim().parse().unwrap_or(0);
983 
                    if others > MERGE_SHED_COMMITS {
984 
                        shed += base.shed_graphs(|b| self.store_blob(b))?;
985 
                    }
986 
                }
fb5b6181 Andy McCall ↶ 987 
                let before = base.len();
958fb27d Andy McCall ↶ 988 
                self.add_git_ancestors_touching(&mut base, git_dir, &refs, &known, &mut touched)?;
fb5b6181 Andy McCall ↶ 989 
                merged = base.len() - before;
03571027 Andy McCall ↶ 990 
            }
958fb27d Andy McCall ↶ 991 
            lap(0, &mut clock);
2306a4d2 Andy McCall ↶ 992 
            // The commit's tree: its first parent's with the diff applied,
993 
            // or listed whole when that tree is not kept.
994 
            let diff = match gc.parents.first() {
995 
                Some(_) => Some(diffs.diff(&gc.sha)?),
996 
                None => None,
997 
            };
998 
            let (tree, changed) = match (prev_tree, diff) {
999 
                (Some(mut t), Some(diff)) => {
1000 
                    let mut changed: HashSet<String> = HashSet::new();
1001 
                    for (path, entry) in diff {
1002 
                        match entry {
1003 
                            Some(v) => {
1004 
                                t.insert(path.clone(), v);
1005 
                            }
1006 
                            None => {
1007 
                                t.remove(&path);
1008 
                            }
1009 
                        }
1010 
                        changed.insert(path);
1011 
                    }
1012 
                    (t, Some(changed))
1013 
                }
1014 
                _ => (list_tree(git_dir, &gc.sha)?, None),
1015 
            };
958fb27d Andy McCall ↶ 1016 
            lap(1, &mut clock);
03571027 Andy McCall ↶ 1017 
            // Text files with the same blob and mode as in the first
1018 
            // parent's tree equal the base's rendering: they are neither
1019 
            // rendered nor read nor compared (spec §12.3, as for a clean
1020 
            // working copy), so a commit costs what it changes.
1021 
            let mut same: HashMap<&str, NodeId> = HashMap::new();
2306a4d2 Andy McCall ↶ 1022 
            // Opaque files and symlinks likewise, when the base has one
1023 
            // blob for them: they are neither read nor stored again.
1024 
            let mut same_opaque: HashSet<&str> = HashSet::new();
1025 
            // An ordinary commit in a tree without submodules is built from
1026 
            // the paths it changes alone (oxygen#25): with the `.oxyattributes`
1027 
            // files that decide how they are stored, and the directories its
1028 
            // removals may leave empty. A merge, or a commit whose parent's
1029 
            // tree is not at hand, is built from its whole tree.
1030 
            let partial = changed.is_some()
1031 
                && gc.parents.len() == 1
1032 
                && !tree.values().any(|(mode, _)| mode == "160000");
1033 
            let mut rel: BTreeSet<&str> = BTreeSet::new();
1034 
            let base_snapshot = if let (true, Some(changed)) = (partial, &changed) {
03571027 Andy McCall ↶ 1035 
                let view = base.tree().view();
2306a4d2 Andy McCall ↶ 1036 
                rel.extend(changed.iter().map(String::as_str));
1037 
                rel.extend(
1038 
                    tree.keys()
1039 
                        .map(String::as_str)
1040 
                        .filter(|p| *p == ".oxyattributes" || p.ends_with("/.oxyattributes")),
1041 
                );
1042 
                let mut dirs: BTreeSet<&str> = BTreeSet::new();
1043 
                for p in changed.iter().filter(|p| !tree.contains_key(*p)) {
1044 
                    let mut end = p.len();
1045 
                    while let Some(i) = p[..end].rfind('/') {
1046 
                        if !dirs.insert(&p[..i]) {
1047 
                            break;
1048 
                        }
1049 
                        end = i;
03571027 Andy McCall ↶ 1050 
                    }
2306a4d2 Andy McCall ↶ 1051 
                }
1052 
                let need: Vec<NodeId> = rel
1053 
                    .iter()
1054 
                    .filter_map(|p| view.entries.get(*p))
1055 
                    .filter(|e| e.kind == NodeKind::File && !e.mode.opaque)
1056 
                    .map(|e| e.node)
1057 
                    .collect();
1058 
                crate::timing::add("render: same and need", clock.elapsed());
1059 
                let t = std::time::Instant::now();
1060 
                self.load_graphs_of(&mut base, &need)?;
1061 
                crate::timing::add("render: load graphs", t.elapsed());
1062 
                let t = std::time::Instant::now();
1063 
                let snap = base.materialize_paths(view, rel.iter().chain(dirs.iter()).copied());
1064 
                crate::timing::add("render: materialize", t.elapsed());
1065 
                snap
1066 
            } else {
1067 
                if let Some(changed) = &changed {
1068 
                    let view = base.tree().view();
1069 
                    for path in tree.keys() {
1070 
                        // Submodule handling reads these two whatever changed.
1071 
                        if changed.contains(path) || path == ".gitmodules" || path == ".oxymodules"
958fb27d Andy McCall ↶ 1072 
                        {
2306a4d2 Andy McCall ↶ 1073 
                            continue;
1074 
                        }
1075 
                        if let Some(e) = view.entries.get(path) {
1076 
                            if touched.contains(&e.node) {
1077 
                                continue;
1078 
                            }
1079 
                            let opaque = e.kind == NodeKind::Symlink
1080 
                                || (e.kind == NodeKind::File && e.mode.opaque);
1081 
                            if e.kind == NodeKind::File && !e.mode.opaque {
1082 
                                same.insert(path.as_str(), e.node);
1083 
                            } else if opaque && base.live_blobs(&e.node).len() == 1 {
1084 
                                same_opaque.insert(path.as_str());
1085 
                            }
03571027 Andy McCall ↶ 1086 
                        }
1087 
                    }
1088 
                }
2306a4d2 Andy McCall ↶ 1089 
                // Build the graphs of every text file the commit may change.
1090 
                let skip: HashSet<NodeId> = same.values().copied().collect();
1091 
                let need: Vec<NodeId> = base
1092 
                    .tree()
1093 
                    .view()
1094 
                    .entries
1095 
                    .values()
1096 
                    .filter(|e| {
1097 
                        e.kind == NodeKind::File && !e.mode.opaque && !skip.contains(&e.node)
1098 
                    })
1099 
                    .map(|e| e.node)
1100 
                    .collect();
1101 
                crate::timing::add("render: same and need", clock.elapsed());
1102 
                let t = std::time::Instant::now();
c3d2ef61 Andy McCall ↶ 1103 
                let snap = if need.len() > merge_batch() {
1104 
                    // A merge onto a base with no built graphs: thousands
1105 
                    // of files, whose graphs would not fit at once.
1106 
                    self.materialize_batched(&mut base, &need, &skip)?
1107 
                } else {
1108 
                    self.load_graphs_of(&mut base, &need)?;
1109 
                    base.materialize_except(&skip)
1110 
                };
1111 
                crate::timing::add("render: whole tree", t.elapsed());
2306a4d2 Andy McCall ↶ 1112 
                snap
1113 
            };
c3069537 Andy McCall ↶ 1114
958fb27d Andy McCall ↶ 1115 
            lap(2, &mut clock);
c3069537 Andy McCall ↶ 1116 
            // The commit's tree as in-memory entries.
1117 
            let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();
4dee277c Andy McCall ↶ 1118 
            let mut gitlinks: Vec<(String, String)> = Vec::new();
2306a4d2 Andy McCall ↶ 1119 
            let listed: Box<dyn Iterator<Item = (&String, &(String, String))>> = if partial {
1120 
                Box::new(rel.iter().filter_map(|p| tree.get_key_value(*p)))
1121 
            } else {
1122 
                Box::new(tree.iter())
1123 
            };
1124 
            for (path, (mode, sha)) in listed {
03571027 Andy McCall ↶ 1125 
                let (path, sha) = (path.clone(), sha.clone());
c3069537 Andy McCall ↶ 1126 
                let (kind, exec) = match mode.as_str() {
1127 
                    "100644" | "100664" => (NodeKind::File, false),
1128 
                    "100755" => (NodeKind::File, true),
1129 
                    "120000" => (NodeKind::Symlink, false),
1130 
                    "160000" => {
4dee277c Andy McCall ↶ 1131 
                        gitlinks.push((path, sha));
c3069537 Andy McCall ↶ 1132 
                        continue;
1133 
                    }
1134 
                    _ => continue,
1135 
                };
2306a4d2 Andy McCall ↶ 1136 
                if same.contains_key(path.as_str()) || same_opaque.contains(path.as_str()) {
03571027 Andy McCall ↶ 1137 
                    let mut e = FsEntry::new(kind, exec, Vec::new());
1138 
                    e.unchanged = true;
1139 
                    entries.insert(path, e);
1140 
                    continue;
1141 
                }
c3069537 Andy McCall ↶ 1142 
                let content = match blob_cache.get(&sha) {
1143 
                    Some(c) => c.clone(),
1144 
                    None => {
1145 
                        let c = cat.read(&sha)?;
dfb23a3a Andy McCall ↶ 1146 
                        if cache_bytes + c.len() > 64 * 1024 * 1024 {
c3069537 Andy McCall ↶ 1147 
                            blob_cache.clear();
1148 
                            cache_bytes = 0;
1149 
                        }
1150 
                        cache_bytes += c.len();
1151 
                        blob_cache.insert(sha.clone(), c.clone());
1152 
                        c
1153 
                    }
1154 
                };
b7f102fc Andy McCall ↶ 1155 
                entries.insert(path, FsEntry::new(kind, exec, content));
c3069537 Andy McCall ↶ 1156 
            }
1157
4dee277c Andy McCall ↶ 1158 
            // Submodules (spec §23): gitlinks become subrepo nodes with Git
1159 
            // pins, and `.gitmodules` also yields `.oxymodules`.
1160 
            if !gitlinks.is_empty() {
1161 
                report.submodules += gitlinks.len();
75167672 Andy McCall 1162 
                let gm = entries
1163 
                    .get(".gitmodules")
1164 
                    .map(|e| {
1165 
                        crate::subrepo::Modules::from_gitmodules(&String::from_utf8_lossy(
1166 
                            &e.content,
1167 
                        ))
1168 
                    })
1169 
                    .unwrap_or_default();
1170 
                let mut modules = entries
1171 
                    .get(".oxymodules")
1172 
                    .map(|e| crate::subrepo::Modules::parse(&String::from_utf8_lossy(&e.content)))
1173 
                    .unwrap_or_default();
4dee277c Andy McCall ↶ 1174 
                for (path, sha) in &gitlinks {
1175 
                    let m = gm.entries.get(path);
75167672 Andy McCall 1176 
                    let branch = m
1177 
                        .and_then(|m| m.bookmark.clone())
1178 
                        .unwrap_or_else(|| "HEAD".into());
1179 
                    entries.insert(
1180 
                        path.clone(),
1181 
                        FsEntry::new(
1182 
                            NodeKind::Subrepo,
1183 
                            false,
1184 
                            crate::subrepo::Pin::Git {
1185 
                                sha: sha.clone(),
1186 
                                branch,
1187 
                            }
1188 
                            .encode(),
1189 
                        ),
1190 
                    );
4dee277c Andy McCall ↶ 1191 
                    if let Some(m) = m {
1192 
                        modules.entries.insert(path.clone(), m.clone());
1193 
                    }
1194 
                }
1195 
                if !modules.entries.is_empty() {
75167672 Andy McCall 1196 
                    entries.insert(
1197 
                        ".oxymodules".into(),
1198 
                        FsEntry::new(NodeKind::File, false, modules.to_text().into_bytes()),
1199 
                    );
4dee277c Andy McCall ↶ 1200 
                }
1201 
            }
958fb27d Andy McCall ↶ 1202 
            lap(3, &mut clock);
dfb23a3a Andy McCall ↶ 1203 
            // New nodes' identifiers derive from the commit, so importing the
1204 
            // same Git history twice gives the same changes (spec §23).
1205 
            let seed = *blake3::hash(format!("oxy git import\0{}", gc.sha).as_bytes()).as_bytes();
2306a4d2 Andy McCall ↶ 1206 
            let mut built = if partial {
1207 
                // Whether the new tree has anything under `dir`.
1208 
                let dir_needed = |dir: &str| {
1209 
                    let prefix = format!("{dir}/");
1210 
                    tree.range(prefix.clone()..)
1211 
                        .next()
1212 
                        .is_some_and(|(k, _)| k.starts_with(&prefix))
1213 
                };
1214 
                self.build_from_entries_seeded(
1215 
                    &base,
1216 
                    &base_snapshot,
1217 
                    &entries,
1218 
                    seed,
1219 
                    Some(&dir_needed),
1220 
                )?
1221 
            } else {
1222 
                self.build_from_entries_seeded(&base, &base_snapshot, &entries, seed, None)?
1223 
            };
958fb27d Andy McCall ↶ 1224 
            lap(4, &mut clock);
62794ecc Andy McCall ↶ 1225 
            // A file or submodule with several live blobs (branches that
1226 
            // replaced it independently) is resolved to the commit's blob:
1227 
            // Git's merge is the authority here, unlike a working copy,
1228 
            // where a candidate on disk is not yet a decision (spec §23).
1229 
            for (path, f) in &base_snapshot.files {
1230 
                let (FileContent::Opaque { blobs }, Some(e)) = (&f.content, entries.get(path))
1231 
                else {
1232 
                    continue;
1233 
                };
1234 
                if blobs.len() < 2 || f.kind != e.kind {
1235 
                    continue;
1236 
                }
1237 
                let chosen = self.store_blob(&e.content)?;
1238 
                for b in blobs {
1239 
                    built.edits.push(Edit::Replace {
1240 
                        node: f.node,
1241 
                        old: Some(*b),
1242 
                        new: Some(chosen),
1243 
                    });
1244 
                }
1245 
                built.resolves = true;
1246 
                built
1247 
                    .contributors
1248 
                    .extend(base.live_replaces(&f.node).into_iter().map(|(c, _)| c));
1249 
            }
e83354cd Andy McCall ↶ 1250 
            // It shares the base's tree view, which applying the change
1251 
            // then updates in place rather than copying.
1252 
            drop(base_snapshot);
1253 
            drop(entries);
c3069537 Andy McCall ↶ 1254 
            let mut deps: BTreeSet<ChangeHash> = built.contributors.iter().copied().collect();
1255 
            let mut message = gc.message.clone();
1256 
            if !message.ends_with('\n') {
1257 
                message.push('\n');
1258 
            }
75167672 Andy McCall 1259 
            message.push_str(&format!(
1260 
                "\nGit-Commit: {}\nGit-Author: {}\n",
1261 
                gc.sha, gc.author
1262 
            ));
c3069537 Andy McCall ↶ 1263 
            let mut change = Change {
6f8aeb99 Andy McCall ↶ 1264 
                version: crate::objects::CHANGE_VERSION,
c3069537 Andy McCall ↶ 1265 
                change_id: change_id_for(&gc.sha, &gc.message),
1266 
                deps: vec![],
1267 
                author: ObjectRef::ZERO,
75167672 Andy McCall 1268 
                hlc: Hlc {
1269 
                    wall_ms: gc.author_secs * 1000,
1270 
                    logical: 0,
1271 
                },
c3069537 Andy McCall ↶ 1272 
                message,
1273 
                tree_ops: built.tree_ops,
1274 
                edits: built.edits,
75167672 Andy McCall 1275 
                kind: if built.resolves {
1276 
                    ChangeKind::Resolution
1277 
                } else {
1278 
                    ChangeKind::Normal
1279 
                },
c3069537 Andy McCall ↶ 1280 
            };
2306a4d2 Andy McCall ↶ 1281 
            let t = std::time::Instant::now();
73d592e7 Andy McCall ↶ 1282 
            // A deletion depends on whoever inserted the live atoms after
1283 
            // it, which only the file's graph knows: build the few the
1284 
            // batched rendering left deferred.
1285 
            let need: Vec<NodeId> = change
1286 
                .edits
1287 
                .iter()
1288 
                .filter(|e| matches!(e, Edit::Delete { .. }))
1289 
                .map(|e| e.node())
1290 
                .filter(|n| !base.is_built(n))
1291 
                .collect();
1292 
            if !need.is_empty() {
1293 
                self.load_graphs_of(&mut base, &need)?;
1294 
            }
c3069537 Andy McCall ↶ 1295 
            deps.extend(referenced_changes(&change, &base));
2306a4d2 Andy McCall ↶ 1296 
            crate::timing::add("store: referenced", t.elapsed());
c3069537 Andy McCall ↶ 1297 
            change.deps = deps.into_iter().collect();
1298 
            change.normalize();
1299 
            let hash = self.store.put_change(&change)?;
dfb23a3a Andy McCall ↶ 1300 
            for d in &change.deps {
1301 
                heads_now.remove(d);
1302 
            }
1303 
            heads_now.insert(hash);
ab4d4e31 Andy McCall ↶ 1304 
            // Built graphs take its edits whole; files not built keep only
1305 
            // where its text is, read back from the store.
2306a4d2 Andy McCall ↶ 1306 
            let t = std::time::Instant::now();
ab4d4e31 Andy McCall ↶ 1307 
            let (skeleton, texts) = self.store.get_change_lazy(&hash)?;
1308 
            base.add_known_mixed(hash, change, skeleton, texts);
2306a4d2 Andy McCall ↶ 1309 
            crate::timing::add("store: add to state", t.elapsed());
30ea9c1e Andy McCall ↶ 1310 
            known.imported.insert(gc.sha.clone(), hash);
958fb27d Andy McCall ↶ 1311 
            if waiting.get(gc.sha.as_str()).is_some_and(|n| *n > 0) {
fd280228 Andy McCall ↶ 1312 
                // A kept state holds its graphs until its children come:
1313 
                // past a few thousand, they go back into its history.
1314 
                if base.built_count() > KEPT_GRAPHS {
1315 
                    shed += base.shed_graphs(|b| self.store_blob(b))?;
1316 
                }
958fb27d Andy McCall ↶ 1317 
                kept.insert(gc.sha.clone(), (base, tree));
1318 
                kept_order.push_back(gc.sha.clone());
dfb23a3a Andy McCall ↶ 1319 
                if kept_order.len() > max_kept {
958fb27d Andy McCall ↶ 1320 
                    if let Some(old) = kept_order.pop_front() {
1321 
                        kept.remove(&old);
1322 
                    }
1323 
                }
1324 
            }
1325 
            last = Some(gc.sha.clone());
c3069537 Andy McCall ↶ 1326 
            report.commits += 1;
958fb27d Andy McCall ↶ 1327 
            lap(5, &mut clock);
fb5b6181 Andy McCall ↶ 1328 
            done += 1;
1329 
            if trace {
1330 
                let built = kept
1331 
                    .get(gc.sha.as_str())
1332 
                    .map(|(st, _)| st.built_count())
1333 
                    .unwrap_or(0);
1334 
                eprintln!(
fd280228 Andy McCall ↶ 1335 
                    "trace {done} {} parents={} base={base_how} merged={merged} built={built} kept={} shed={shed} rss_mb={}",
fb5b6181 Andy McCall ↶ 1336 
                    &gc.sha[..12.min(gc.sha.len())],
1337 
                    gc.parents.len(),
1338 
                    kept.len(),
1339 
                    rss_mb()
1340 
                );
1341 
            }
958fb27d Andy McCall ↶ 1342 
        }
1343 
        for (name, d) in [
1344 
            "git import: bases",
2306a4d2 Andy McCall ↶ 1345 
            "git import: tree",
958fb27d Andy McCall ↶ 1346 
            "git import: render base",
1347 
            "git import: read blobs",
1348 
            "git import: build change",
1349 
            "git import: store change",
1350 
        ]
1351 
        .iter()
1352 
        .zip(spent)
1353 
        {
1354 
            crate::timing::record(name, d);
2306a4d2 Andy McCall ↶ 1355 
        }
1356 
        crate::timing::report_totals();
1357 
        if crate::timing::enabled() {
fd280228 Andy McCall ↶ 1358 
            eprintln!(
1359 
                "timing git import: {rebuilt} bases rebuilt from history, {shed} graphs shed"
1360 
            );
c3069537 Andy McCall ↶ 1361 
        }
1362
30ea9c1e Andy McCall ↶ 1363 
        // The bookmark: heads of the old state plus every new change.
1364 
        report.heads = old_heads.len();
1365 
        if last.is_none() {
1366 
            return Ok(report);
1367 
        }
dfb23a3a Andy McCall ↶ 1368 
        let heads: Vec<ChangeHash> = heads_now.into_iter().collect();
c3069537 Andy McCall ↶ 1369 
        report.heads = heads.len();
1370 
        let (op_ref, op) = self.head_op()?;
1371 
        let mut view = op.view.clone();
30ea9c1e Andy McCall ↶ 1372 
        let old_state_ref = view.bookmarks.get(&self.config.bookmark).copied();
1373 
        let state_ref = self.store.put_state(&StateObject::new(heads.clone()))?;
75167672 Andy McCall 1374 
        view.bookmarks
1375 
            .insert(self.config.bookmark.clone(), state_ref);
30ea9c1e Andy McCall ↶ 1376 
        self.record_op(&[op_ref], command, view)?;
1377 
        // The new state equals the tip's tree when the old one was itself a
1378 
        // Git commit's tree (fetched or exported), or there was none.
75167672 Andy McCall 1379 
        let old_known = old_heads.is_empty()
1380 
            || old_state_ref.is_some_and(|s| known.exported.values().any(|e| *e == s));
30ea9c1e Andy McCall ↶ 1381 
        if let (Some(tip), true) = (commits.last(), old_known) {
1382 
            self.record_git_state(git_dir, &state_ref, &tip.sha)?;
1383 
        }
1384 
        // The working copy follows when it was clean on the old heads.
1385 
        if old_heads.is_empty() {
1386 
            self.checkout(&heads, true, &format!("{command} (checkout)"))?;
1387 
        } else {
1388 
            let snap = self.snapshot(command)?;
1389 
            let clean = snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty();
1390 
            let mut base = snap.base.maximal(&snap.wc.deps);
1391 
            base.sort();
1392 
            let mut sorted_old = old_heads.clone();
1393 
            sorted_old.sort();
1394 
            if clean && base == sorted_old {
1395 
                self.checkout(&heads, false, &format!("{command} (checkout)"))?;
1396 
            }
1397 
        }
c3069537 Andy McCall ↶ 1398 
        Ok(report)
1399 
    }
1400
30ea9c1e Andy McCall ↶ 1401 
    /// Add to `st` the changes of every Git commit `rev-list args` lists:
1402 
    /// an imported commit's change, or the whole state an exported commit
1403 
    /// was made from (which may hold changes Git never saw).
75167672 Andy McCall 1404 
    fn add_git_ancestors(
958fb27d Andy McCall ↶ 1405 
        &self,
1406 
        st: &mut State,
1407 
        git_dir: &Path,
1408 
        args: &[&str],
1409 
        known: &GitKnown,
1410 
    ) -> Result<()> {
1411 
        self.add_git_ancestors_touching(st, git_dir, args, known, &mut HashSet::new())
1412 
    }
1413
1414 
    /// `add_git_ancestors`, also collecting in `touched` every node the
1415 
    /// changes it adds edit, place or remove.
1416 
    fn add_git_ancestors_touching(
75167672 Andy McCall 1417 
        &self,
1418 
        st: &mut State,
1419 
        git_dir: &Path,
1420 
        args: &[&str],
1421 
        known: &GitKnown,
958fb27d Andy McCall ↶ 1422 
        touched: &mut HashSet<NodeId>,
75167672 Andy McCall 1423 
    ) -> Result<()> {
30ea9c1e Andy McCall ↶ 1424 
        let mut full = vec!["rev-list"];
1425 
        full.extend(args);
1426 
        let out = git(git_dir, &full)?;
958fb27d Andy McCall ↶ 1427 
        let mut add = |st: &mut State, h: &ChangeHash| -> Result<()> {
1428 
            if st.contains(h) {
e83354cd Andy McCall ↶ 1429 
                return Ok(());
1430 
            }
1431 
            if !st.is_history_only() {
1432 
                let c = self.store.get_change(h)?;
1433 
                touched.extend(c.edits.iter().map(|e| e.node()));
1434 
                touched.extend(c.tree_ops.iter().map(|o| o.node()));
1435 
                st.add_unchecked(c);
958fb27d Andy McCall ↶ 1436 
                return Ok(());
e83354cd Andy McCall ↶ 1437 
            }
1438 
            // History only: the text of an edit waits unread unless its
1439 
            // file's graph is built.
1440 
            let (skeleton, texts) = self.store.get_change_lazy(h)?;
1441 
            touched.extend(skeleton.edits.iter().map(|e| e.node()));
1442 
            touched.extend(skeleton.tree_ops.iter().map(|o| o.node()));
1443 
            let built = skeleton
1444 
                .edits
1445 
                .iter()
1446 
                .any(|e| !matches!(e, Edit::Replace { .. }) && st.is_built(&e.node()));
1447 
            if built {
1448 
                let full = self.store.get_change(h)?;
1449 
                st.add_known_mixed(*h, full, skeleton, texts);
1450 
            } else {
1451 
                st.add_known(*h, skeleton, texts);
958fb27d Andy McCall ↶ 1452 
            }
1453 
            Ok(())
1454 
        };
c3069537 Andy McCall ↶ 1455 
        for sha in out.lines() {
30ea9c1e Andy McCall ↶ 1456 
            if let Some(h) = known.imported.get(sha) {
958fb27d Andy McCall ↶ 1457 
                add(st, h)?;
30ea9c1e Andy McCall ↶ 1458 
            } else if let Some(state_ref) = known.exported.get(sha) {
1459 
                let heads = self.store.get_state(state_ref)?.heads;
1460 
                for h in self.negotiate(&[], &heads)? {
958fb27d Andy McCall ↶ 1461 
                    add(st, &h)?;
30ea9c1e Andy McCall ↶ 1462 
                }
1463 
            }
1464 
        }
1465 
        Ok(())
1466 
    }
1467
1468 
    /// Commits this repository can relate to Oxygen history: imported
1469 
    /// ones by change, exported ones (through `git_dir`'s marks) by state.
1470 
    fn git_known(&self, git_dir: &Path) -> Result<GitKnown> {
1471 
        let imported = self.imported_commits()?;
1472 
        let marks_dir = self.git_marks_dir(git_dir)?;
1473 
        let oxy_marks = Self::read_oxy_marks(&marks_dir.join("oxy-marks"))?;
1474 
        let mut sha_of_mark: HashMap<u64, String> = HashMap::new();
1475 
        if let Ok(text) = fs::read_to_string(marks_dir.join("git-marks")) {
1476 
            for line in text.lines() {
1477 
                if let Some((m, sha)) = line.split_once(' ') {
1478 
                    if let Ok(m) = m.trim_start_matches(':').parse::<u64>() {
1479 
                        sha_of_mark.insert(m, sha.trim().to_string());
1480 
                    }
1481 
                }
1482 
            }
1483 
        }
1484 
        let mut exported = HashMap::new();
1485 
        for (state_ref, m) in oxy_marks {
1486 
            if let Some(sha) = sha_of_mark.get(&m) {
1487 
                exported.insert(sha.clone(), state_ref);
c3069537 Andy McCall ↶ 1488 
            }
1489 
        }
30ea9c1e Andy McCall ↶ 1490 
        for (state_ref, sha) in Self::read_git_states(&marks_dir.join("git-states"))? {
1491 
            exported.entry(sha).or_insert(state_ref);
1492 
        }
1493 
        Ok(GitKnown { imported, exported })
c3069537 Andy McCall ↶ 1494 
    }
1495
79ab8528 Andy McCall ↶ 1496 
    fn git_author(&self, c: &Change) -> String {
1497 
        match self.store.get_identity(&c.author) {
75167672 Andy McCall 1498 
            Ok(id) if !id.name.is_empty() => format!(
1499 
                "{} <{}>",
1500 
                id.name.replace(['<', '>', '\n'], " "),
1501 
                id.email.replace(['<', '>', '\n', ' '], "")
1502 
            ),
79ab8528 Andy McCall ↶ 1503 
            _ => "oxy <oxy@localhost>".to_string(),
1504 
        }
1505 
    }
1506
1507 
    #[allow(clippy::too_many_arguments)]
75167672 Andy McCall 1508 
    fn write_commit(
1509 
        &self,
1510 
        out: &mut Vec<u8>,
1511 
        reference: &str,
1512 
        mark: u64,
1513 
        author: &str,
1514 
        secs: u64,
1515 
        message: &str,
1516 
        parents: &[String],
1517 
        snapshot: &Snapshot,
1518 
    ) -> Result<()> {
79ab8528 Andy McCall ↶ 1519 
        out.extend_from_slice(format!("commit {reference}\nmark :{mark}\nauthor {author} {secs} +0000\ncommitter {author} {secs} +0000\ndata {}\n{message}\n", message.len()).as_bytes());
1520 
        for (i, p) in parents.iter().enumerate() {
75167672 Andy McCall 1521 
            out.extend_from_slice(
1522 
                format!("{} {p}\n", if i == 0 { "from" } else { "merge" }).as_bytes(),
1523 
            );
79ab8528 Andy McCall ↶ 1524 
        }
1525 
        out.extend_from_slice(b"deleteall\n");
4dee277c Andy McCall ↶ 1526 
        let has_gitmodules = snapshot.files.contains_key(".gitmodules");
79ab8528 Andy McCall ↶ 1527 
        for (path, f) in &snapshot.files {
4dee277c Andy McCall ↶ 1528 
            if path == ".oxymodules" {
1529 
                // Git reads `.gitmodules`; derive it unless the tree has one.
1530 
                if !has_gitmodules {
1531 
                    if let FileContent::Text { bytes, .. } = &f.content {
75167672 Andy McCall 1532 
                        let gm = crate::subrepo::Modules::parse(&String::from_utf8_lossy(bytes))
1533 
                            .to_gitmodules();
4dee277c Andy McCall ↶ 1534 
                        if !gm.is_empty() {
75167672 Andy McCall 1535 
                            out.extend_from_slice(
1536 
                                format!("M 100644 inline .gitmodules\ndata {}\n", gm.len())
1537 
                                    .as_bytes(),
1538 
                            );
4dee277c Andy McCall ↶ 1539 
                            out.extend_from_slice(gm.as_bytes());
1540 
                            out.push(b'\n');
1541 
                        }
1542 
                    }
1543 
                }
1544 
                continue;
1545 
            }
1546 
            if f.kind == NodeKind::Subrepo {
1547 
                if let FileContent::Opaque { blobs } = &f.content {
1548 
                    if let Some(b) = blobs.first() {
1549 
                        match crate::subrepo::Pin::decode(&self.read_blob(b)?)? {
75167672 Andy McCall 1550 
                            crate::subrepo::Pin::Git { sha, .. } => {
1551 
                                out.extend_from_slice(format!("M 160000 {sha} {path}\n").as_bytes())
1552 
                            }
1553 
                            crate::subrepo::Pin::Oxy { .. } => self
1554 
                                .git_export_skipped
1555 
                                .set(self.git_export_skipped.get() + 1),
4dee277c Andy McCall ↶ 1556 
                        }
1557 
                    }
1558 
                }
1559 
                continue;
1560 
            }
79ab8528 Andy McCall ↶ 1561 
            let (mode, bytes): (&str, Vec<u8>) = match (&f.kind, &f.content) {
1562 
                (NodeKind::Dir, _) => continue,
75167672 Andy McCall 1563 
                (NodeKind::Symlink, FileContent::Opaque { blobs }) => (
1564 
                    "120000",
1565 
                    blobs
1566 
                        .first()
1567 
                        .map(|b| self.read_blob(b))
1568 
                        .transpose()?
1569 
                        .unwrap_or_default(),
1570 
                ),
1571 
                (_, FileContent::Text { bytes, .. }) => {
1572 
                    (if f.mode.exec { "100755" } else { "100644" }, bytes.clone())
1573 
                }
1574 
                (_, FileContent::Opaque { blobs }) => (
1575 
                    if f.mode.exec { "100755" } else { "100644" },
1576 
                    blobs
1577 
                        .first()
1578 
                        .map(|b| self.read_blob(b))
1579 
                        .transpose()?
1580 
                        .unwrap_or_default(),
1581 
                ),
79ab8528 Andy McCall ↶ 1582 
                (_, FileContent::Dir) => continue,
1583 
            };
75167672 Andy McCall 1584 
            out.extend_from_slice(
1585 
                format!("M {mode} inline {path}\ndata {}\n", bytes.len()).as_bytes(),
1586 
            );
79ab8528 Andy McCall ↶ 1587 
            out.extend_from_slice(&bytes);
1588 
            out.push(b'\n');
1589 
        }
1590 
        out.push(b'\n');
1591 
        Ok(())
1592 
    }
1593 
}
62794ecc Andy McCall ↶ 1594
1595 
/// What `verify_git_import` found (spec §23: import must reproduce every tree).
1596 
#[derive(Debug, Clone, Default, PartialEq, Eq)]
1597 
pub struct GitVerifyReport {
1598 
    pub commits: usize,
7eac74ab Andy McCall ↶ 1599 
    /// Commits followed but not rebuilt (all but the last few, when asked).
1600 
    pub skipped: usize,
62794ecc Andy McCall ↶ 1601 
    pub files_compared: usize,
1602 
    /// `(commit, what differs)`; empty when every commit's tree is reproduced.
1603 
    pub mismatches: Vec<(String, String)>,
1604 
}
1605
1606 
impl Repo {
1607 
    /// Check an import against its Git source: for every commit of
1608 
    /// `branch`, the state formed by the changes imported for it and all
1609 
    /// its ancestors must materialize to exactly that commit's tree, byte
1610 
    /// for byte, with the same executable bits, symlink targets and
1611 
    /// submodule pins. Reads only; stops listing after 50 differences.
1612 
    pub fn verify_git_import(&self, git_dir: &Path, branch: &str) -> Result<GitVerifyReport> {
7eac74ab Andy McCall ↶ 1613 
        self.verify_git_import_last(git_dir, branch, None)
1614 
    }
1615
1616 
    /// [`Repo::verify_git_import`] of the last `last` commits only (all
1617 
    /// when `None`): earlier ones are only followed, not rebuilt, for
1618 
    /// looking again at where an import went wrong.
1619 
    pub fn verify_git_import_last(
1620 
        &self,
1621 
        git_dir: &Path,
1622 
        branch: &str,
1623 
        last: Option<usize>,
1624 
    ) -> Result<GitVerifyReport> {
62794ecc Andy McCall ↶ 1625 
        let commits = list_commits(git_dir, branch)?;
1626 
        let imported = self.imported_commits()?;
1627 
        let mut cat = CatFile::start(git_dir)?;
1628 
        let mut rep = GitVerifyReport::default();
1629 
        // Changes of each commit and all its ancestors, built up in topological order.
1630 
        let mut closure: HashMap<String, Arc<BTreeSet<ChangeHash>>> = HashMap::new();
a87c7b9d Andy McCall ↶ 1631 
        // Graphs (by key) and blobs already found equal to a Git blob.
1632 
        let mut verified_graphs: HashMap<[u8; 32], String> = HashMap::new();
1633 
        let mut verified_blobs: HashMap<ObjectRef, String> = HashMap::new();
57d8dc05 Andy McCall ↶ 1634 
        // The state of a commit whose first-parent children are still to
1635 
        // come, so a child adds its own changes instead of loading all its
1636 
        // history again (as the import does).
1637 
        let mut waiting: HashMap<&str, usize> = HashMap::new();
62794ecc Andy McCall ↶ 1638 
        for c in &commits {
57d8dc05 Andy McCall ↶ 1639 
            if let Some(p1) = c.parents.first() {
1640 
                *waiting.entry(p1.as_str()).or_default() += 1;
1641 
            }
1642 
        }
1643 
        let mut kept: HashMap<String, State> = HashMap::new();
1644 
        let mut kept_order: std::collections::VecDeque<String> = std::collections::VecDeque::new();
1645 
        for c in &commits {
62794ecc Andy McCall ↶ 1646 
            rep.commits += 1;
1647 
            let mut set: BTreeSet<ChangeHash> = BTreeSet::new();
1648 
            for p in &c.parents {
1649 
                if let Some(ps) = closure.get(p) {
1650 
                    set.extend(ps.iter().copied());
1651 
                }
1652 
            }
1653 
            match imported.get(&c.sha) {
1654 
                Some(h) => {
1655 
                    set.insert(*h);
1656 
                }
1657 
                None => {
1658 
                    rep.mismatches.push((
1659 
                        c.sha.clone(),
1660 
                        "no imported change carries this commit".into(),
1661 
                    ));
1662 
                }
1663 
            }
1664 
            let set = Arc::new(set);
1665 
            closure.insert(c.sha.clone(), set.clone());
1666 
            if rep.mismatches.len() >= 50 {
1667 
                break;
7eac74ab Andy McCall ↶ 1668 
            }
1669 
            if last.is_some_and(|n| rep.commits + n <= commits.len()) {
1670 
                rep.skipped += 1;
1671 
                continue;
62794ecc Andy McCall ↶ 1672 
            }
1673 
            let heads: Vec<ChangeHash> = set.iter().copied().collect();
a87c7b9d Andy McCall ↶ 1674 
            // History only: a file whose graph (by key) or blob was already
1675 
            // found equal to this Git blob is not built, rendered or read
1676 
            // again; the rest are.
57d8dc05 Andy McCall ↶ 1677 
            let from_parent = c.parents.first().and_then(|p1| {
1678 
                let left = waiting.get_mut(p1.as_str()).map(|n| {
1679 
                    *n -= 1;
1680 
                    *n
1681 
                });
1682 
                if left == Some(0) {
1683 
                    kept_order.retain(|k| k != p1);
1684 
                    kept.remove(p1.as_str())
1685 
                } else {
1686 
                    kept.get(p1.as_str()).cloned()
1687 
                }
1688 
            });
1689 
            let state = match from_parent {
1690 
                Some(mut st) => {
1691 
                    // The changes the parent's state lacks. The state never
1692 
                    // builds a graph (they are built beside it), so they
1693 
                    // are read without their text.
1694 
                    for h in set.iter() {
1695 
                        if !st.contains(h) {
1696 
                            let (change, texts) = self.store.get_change_lazy(h)?;
1697 
                            st.add_known(*h, change, texts);
1698 
                        }
1699 
                    }
1700 
                    st
1701 
                }
1702 
                None => self.load_state_history(&heads)?,
1703 
            };
a87c7b9d Andy McCall ↶ 1704 
            let view = state.tree().view();
62794ecc Andy McCall ↶ 1705 
            let tree = list_tree(git_dir, &c.sha)?;
1706 
            let short = &c.sha[..12.min(c.sha.len())];
1707 
            let mut differ = |what: String| rep.mismatches.push((c.sha.clone(), what));
a87c7b9d Andy McCall ↶ 1708 
            let mut conflicted: Vec<String> =
1709 
                view.conflicts.iter().map(|c| format!("{c:?}")).collect();
1710 
            let mut need: Vec<NodeId> = Vec::new();
1711 
            for (path, (mode, sha)) in &tree {
1712 
                let Some(e) = view.entries.get(path) else {
1713 
                    continue;
1714 
                };
1715 
                if mode != "160000"
1716 
                    && mode != "120000"
1717 
                    && e.kind == NodeKind::File
1718 
                    && !e.mode.opaque
1719 
                {
1720 
                    let known = state
1721 
                        .graph_key(&e.node)
1722 
                        .is_some_and(|k| verified_graphs.get(&k) == Some(sha));
1723 
                    if !known {
1724 
                        need.push(e.node);
62794ecc Andy McCall ↶ 1725 
                    }
1726 
                }
1727 
            }
a87c7b9d Andy McCall ↶ 1728 
            // A graph's key is known only before it is built.
1729 
            let keys: HashMap<NodeId, [u8; 32]> = need
1730 
                .iter()
1731 
                .filter_map(|n| Some((*n, state.graph_key(n)?)))
1732 
                .collect();
1733 
            let need_set: HashSet<NodeId> = need.iter().copied().collect();
57d8dc05 Andy McCall ↶ 1734 
            let graphs = self.build_graphs_detached(&state, &need)?;
62794ecc Andy McCall ↶ 1735 
            for (path, (mode, sha)) in &tree {
a87c7b9d Andy McCall ↶ 1736 
                let Some(e) = view.entries.get(path) else {
62794ecc Andy McCall ↶ 1737 
                    differ(format!("{short}: {path} is missing"));
1738 
                    continue;
1739 
                };
1740 
                rep.files_compared += 1;
1741 
                match mode.as_str() {
a87c7b9d Andy McCall ↶ 1742 
                    "160000" => {
1743 
                        let blobs = state.live_blobs(&e.node);
1744 
                        match (e.kind, blobs.as_slice()) {
1745 
                            (NodeKind::Subrepo, [b]) => {
1746 
                                let pin = crate::subrepo::Pin::decode(&self.read_blob(b)?)?;
1747 
                                if !matches!(&pin, crate::subrepo::Pin::Git { sha: s, .. } if s == sha)
1748 
                                {
1749 
                                    differ(format!("{short}: {path} pins another commit"));
1750 
                                }
62794ecc Andy McCall ↶ 1751 
                            }
a87c7b9d Andy McCall ↶ 1752 
                            (kind, _) => differ(format!(
1753 
                                "{short}: {path} is not one submodule pin: {kind:?} with {} pins",
1754 
                                blobs.len()
1755 
                            )),
62794ecc Andy McCall ↶ 1756 
                        }
a87c7b9d Andy McCall ↶ 1757 
                    }
62794ecc Andy McCall ↶ 1758 
                    "120000" => {
1759 
                        let target = cat.read(sha)?;
a87c7b9d Andy McCall ↶ 1760 
                        let blobs = state.live_blobs(&e.node);
1761 
                        match (e.kind, blobs.as_slice()) {
1762 
                            (NodeKind::Symlink, [b]) if self.read_blob(b)? == target => {}
62794ecc Andy McCall ↶ 1763 
                            _ => differ(format!("{short}: symlink {path} differs")),
1764 
                        }
1765 
                    }
1766 
                    _ => {
a87c7b9d Andy McCall ↶ 1767 
                        if e.mode.exec != (mode == "100755") {
1768 
                            differ(format!("{short}: {path} executable bit differs"));
1769 
                        }
1770 
                        if e.kind == NodeKind::File && !e.mode.opaque {
1771 
                            if !need_set.contains(&e.node) {
1772 
                                // Known equal: skipped when the graphs were chosen.
62794ecc Andy McCall ↶ 1773 
                                continue;
1774 
                            }
57d8dc05 Andy McCall ↶ 1775 
                            let (have, m) = match graphs.get(&e.node) {
1776 
                                Some(g) => state.render_graph(g),
1777 
                                None => (Vec::new(), Default::default()),
1778 
                            };
a87c7b9d Andy McCall ↶ 1779 
                            if m.is_conflicted() {
57d8dc05 Andy McCall ↶ 1780 
                                // Who conflicts: a copy of the state with this graph built.
1781 
                                let mut detail = state.clone();
1782 
                                self.load_graphs_of(&mut detail, &[e.node])?;
1783 
                                let who: Vec<String> = detail
a87c7b9d Andy McCall ↶ 1784 
                                    .conflict_regions(&e.node)
1785 
                                    .contributors
1786 
                                    .iter()
1787 
                                    .filter_map(|h| state.get(h))
1788 
                                    .map(|m| {
1789 
                                        let sha = m
1790 
                                            .message
1791 
                                            .lines()
1792 
                                            .rev()
1793 
                                            .find_map(|l| l.strip_prefix("Git-Commit: "))
1794 
                                            .unwrap_or("?");
1795 
                                        format!(
1796 
                                            "{} {:?}",
1797 
                                            &sha[..12.min(sha.len())],
1798 
                                            m.message.lines().next().unwrap_or("")
1799 
                                        )
1800 
                                    })
1801 
                                    .collect();
1802 
                                conflicted.push(format!("{path} between {}", who.join(" and ")));
1803 
                            }
1804 
                            let want = cat.read(sha)?;
1805 
                            if have != want {
1806 
                                differ(format!(
1807 
                                    "{short}: {path} differs: {}",
1808 
                                    first_difference(&have, &want)
1809 
                                ));
1810 
                            } else if !m.is_conflicted() {
1811 
                                if let Some(k) = keys.get(&e.node) {
1812 
                                    verified_graphs.insert(*k, sha.clone());
1813 
                                }
1814 
                            }
1815 
                            continue;
1816 
                        }
1817 
                        let blobs = state.live_blobs(&e.node);
1818 
                        let [b] = blobs.as_slice() else {
1819 
                            differ(format!("{short}: {path} is not one file"));
1820 
                            continue;
62794ecc Andy McCall ↶ 1821 
                        };
a87c7b9d Andy McCall ↶ 1822 
                        if verified_blobs.get(b) == Some(sha) {
1823 
                            continue;
62794ecc Andy McCall ↶ 1824 
                        }
a87c7b9d Andy McCall ↶ 1825 
                        if self.read_blob(b)? != cat.read(sha)? {
1826 
                            differ(format!("{short}: {path} differs"));
1827 
                        } else {
1828 
                            verified_blobs.insert(*b, sha.clone());
62794ecc Andy McCall ↶ 1829 
                        }
1830 
                    }
1831 
                }
1832 
            }
a87c7b9d Andy McCall ↶ 1833 
            if !conflicted.is_empty() {
1834 
                differ(format!(
1835 
                    "{short}: the state is conflicted: {}",
1836 
                    conflicted.join("; ")
1837 
                ));
1838 
            }
62794ecc Andy McCall ↶ 1839 
            // Import records submodules in `.oxymodules` as well (spec §23),
1840 
            // and export leaves it out again.
1841 
            let has_submodules = tree.values().any(|(m, _)| m == "160000");
e83354cd Andy McCall ↶ 1842 
            for (path, e) in view.entries.iter() {
a87c7b9d Andy McCall ↶ 1843 
                if e.kind != NodeKind::Dir
62794ecc Andy McCall ↶ 1844 
                    && !tree.contains_key(path)
1845 
                    && !(has_submodules && path == ".oxymodules")
1846 
                {
1847 
                    differ(format!("{short}: {path} is extra"));
57d8dc05 Andy McCall ↶ 1848 
                }
1849 
            }
1850 
            if waiting.get(c.sha.as_str()).is_some_and(|n| *n > 0) {
1851 
                kept.insert(c.sha.clone(), state);
1852 
                kept_order.push_back(c.sha.clone());
1853 
                if kept_order.len() > 8 {
1854 
                    if let Some(old) = kept_order.pop_front() {
1855 
                        kept.remove(&old);
1856 
                    }
62794ecc Andy McCall ↶ 1857 
                }
1858 
            }
1859 
            if rep.mismatches.len() >= 50 {
1860 
                break;
1861 
            }
1862 
        }
1863 
        Ok(rep)
1864 
    }
1865 
}
1866
1867 
/// Where two versions of a file first part, for a verification report.
1868 
fn first_difference(have: &[u8], want: &[u8]) -> String {
1869 
    let (h, w): (Vec<&[u8]>, Vec<&[u8]>) = (
1870 
        have.split(|b| *b == b'\n').collect(),
1871 
        want.split(|b| *b == b'\n').collect(),
1872 
    );
1873 
    let i = h
1874 
        .iter()
1875 
        .zip(&w)
1876 
        .position(|(a, b)| a != b)
1877 
        .unwrap_or(h.len().min(w.len()));
1878 
    let show = |l: Option<&&[u8]>| {
1879 
        l.map(|l| String::from_utf8_lossy(&l[..l.len().min(60)]).into_owned())
1880 
            .unwrap_or_else(|| "(end)".into())
1881 
    };
1882 
    format!(
1883 
        "line {}: has {:?}, Git has {:?} ({} lines, Git has {})",
1884 
        i + 1,
1885 
        show(h.get(i)),
1886 
        show(w.get(i)),
1887 
        h.len(),
1888 
        w.len()
1889 
    )
1890 
}