OXYGENHUB
# ebef3715554c oxygen / crates / oxy-core / src / repo / git.rs
18.9 KiB · Rust · last changed Git bridge, import side: oxy git import with merge commits as resolutions, Change-Id trailers, batched blob reads 1 month ago source blame history raw
79ab8528 Andy McCall ↶ 1
//! Git bridge (spec §23), export side: each state the current bookmark
2
//! passed through becomes a Git commit through `git fast-import`, with the
3
//! state's full tree and the previous state's commit as parent. A state
4
//! that added exactly one change carries that change's message, author,
5
//! and `Change-Id`/`Oxy-Hash` trailers; every commit carries `Oxy-State`.
6
//!
7
//! Marks make re-exports incremental: `.oxy/git/<target>/git-marks` is
8
//! git's own marks file and `oxy-marks` maps state refs to marks.
9
10
use super::*;
11
use std::process::{Command, Stdio};
12
13
#[derive(Debug, Clone, Default, PartialEq, Eq)]
c3069537 Andy McCall ↶ 14
pub struct GitImportReport {
15
    pub commits: usize,
16
    pub merges: usize,
17
    pub skipped_submodules: usize,
18
    pub heads: usize,
19
}
20
21
/// A running `git cat-file --batch`, for reading many blobs cheaply.
22
struct CatFile {
23
    child: std::process::Child,
24
    stdin: std::process::ChildStdin,
25
    stdout: std::io::BufReader<std::process::ChildStdout>,
26
}
27
28
impl CatFile {
29
    fn start(git_dir: &Path) -> Result<CatFile> {
30
        let mut child = Command::new("git")
31
            .arg("-C")
32
            .arg(git_dir)
33
            .args(["cat-file", "--batch"])
34
            .stdin(Stdio::piped())
35
            .stdout(Stdio::piped())
36
            .stderr(Stdio::null())
37
            .spawn()
38
            .map_err(|e| RepoError::Msg(format!("running git cat-file: {e}")))?;
39
        let stdin = child.stdin.take().unwrap();
40
        let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
41
        Ok(CatFile { child, stdin, stdout })
42
    }
43
44
    fn read(&mut self, sha: &str) -> Result<Vec<u8>> {
45
        use std::io::{BufRead, Read};
46
        writeln!(self.stdin, "{sha}")?;
47
        self.stdin.flush()?;
48
        let mut header = String::new();
49
        self.stdout.read_line(&mut header)?;
50
        let parts: Vec<&str> = header.trim_end().split(' ').collect();
51
        if parts.len() != 3 {
52
            return fail(format!("git cat-file: object {sha} not found"));
53
        }
54
        let size: usize = parts[2].parse().map_err(|_| RepoError::Msg("git cat-file: bad size".into()))?;
55
        let mut body = vec![0u8; size];
56
        self.stdout.read_exact(&mut body)?;
57
        let mut nl = [0u8; 1];
58
        self.stdout.read_exact(&mut nl)?;
59
        Ok(body)
60
    }
61
}
62
63
impl Drop for CatFile {
64
    fn drop(&mut self) {
65
        let _ = self.child.kill();
66
        let _ = self.child.wait();
67
    }
68
}
69
70
/// One commit as `git log` describes it.
71
struct GitCommit {
72
    sha: String,
73
    parents: Vec<String>,
74
    author_secs: u64,
75
    author: String,
76
    message: String,
77
}
78
79
fn list_commits(git_dir: &Path, branch: &str) -> Result<Vec<GitCommit>> {
80
    let out = Command::new("git")
81
        .arg("-C")
82
        .arg(git_dir)
83
        .args(["log", "--reverse", "--topo-order", "--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e", branch])
84
        .output()
85
        .map_err(|e| RepoError::Msg(format!("running git log: {e}")))?;
86
    if !out.status.success() {
87
        return fail(format!("git log {branch} failed: {}", String::from_utf8_lossy(&out.stderr).trim()));
88
    }
89
    let text = String::from_utf8_lossy(&out.stdout);
90
    let mut commits = Vec::new();
91
    for record in text.split('\x1e') {
92
        let record = record.trim_start_matches('\n');
93
        if record.trim().is_empty() {
94
            continue;
95
        }
96
        let f: Vec<&str> = record.splitn(5, '\0').collect();
97
        if f.len() != 5 {
98
            return fail("unexpected git log record");
99
        }
100
        commits.push(GitCommit {
101
            sha: f[0].to_string(),
102
            parents: f[1].split_whitespace().map(str::to_string).collect(),
103
            author_secs: f[2].parse().unwrap_or(0),
104
            author: f[3].to_string(),
105
            message: f[4].trim_end().to_string(),
106
        });
107
    }
108
    Ok(commits)
109
}
110
111
/// `path → (mode, blob sha)` for a commit's tree.
112
fn list_tree(git_dir: &Path, sha: &str) -> Result<BTreeMap<String, (String, String)>> {
113
    let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?;
114
    let mut out = BTreeMap::new();
115
    for entry in text.split('\0') {
116
        if entry.is_empty() {
117
            continue;
118
        }
119
        let Some((meta, path)) = entry.split_once('\t') else { continue };
120
        let m: Vec<&str> = meta.split(' ').collect();
121
        if m.len() == 3 {
122
            out.insert(path.to_string(), (m[0].to_string(), m[2].to_string()));
123
        }
124
    }
125
    Ok(out)
126
}
127
128
/// The ChangeId for a Git commit: its `Change-Id` trailer if it has a
129
/// 32-hex one, else BLAKE3-128 of the commit hash (spec §23).
130
fn change_id_for(sha: &str, message: &str) -> ChangeId {
131
    for line in message.lines().rev() {
132
        if let Some(v) = line.strip_prefix("Change-Id:") {
133
            if let Some(id) = ChangeId::from_hex(v.trim()) {
134
                return id;
135
            }
136
        }
137
    }
138
    let h = blake3::hash(sha.as_bytes());
139
    ChangeId(h.as_bytes()[..16].try_into().unwrap())
140
}
141
142
#[derive(Debug, Clone, Default, PartialEq, Eq)]
79ab8528 Andy McCall ↶ 143
pub struct GitExportReport {
144
    pub exported: usize,
145
    pub skipped: usize,
146
    pub branch: String,
147
    pub heads: usize,
148
}
149
150
fn git(dir: &Path, args: &[&str]) -> Result<String> {
151
    let out = Command::new("git").arg("-C").arg(dir).args(args).output().map_err(|e| RepoError::Msg(format!("running git: {e}")))?;
152
    if !out.status.success() {
153
        return fail(format!("git {} failed: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim()));
154
    }
155
    Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
156
}
157
158
impl Repo {
159
    fn git_marks_dir(&self, git_dir: &Path) -> Result<PathBuf> {
160
        let canonical = git_dir.canonicalize().unwrap_or_else(|_| git_dir.to_path_buf());
161
        let key = hash_object(ObjectType::Chunk, canonical.to_string_lossy().as_bytes()).short(16);
162
        let dir = self.oxy.join("git").join(key);
163
        fs::create_dir_all(&dir)?;
164
        Ok(dir)
165
    }
166
167
    fn read_oxy_marks(path: &Path) -> Result<BTreeMap<ObjectRef, u64>> {
168
        let mut out = BTreeMap::new();
169
        if let Ok(text) = fs::read_to_string(path) {
170
            for line in text.lines() {
171
                if let Some((h, m)) = line.split_once(' ') {
172
                    if let (Some(h), Ok(m)) = (ObjectRef::from_hex(h), m.parse()) {
173
                        out.insert(h, m);
174
                    }
175
                }
176
            }
177
        }
178
        Ok(out)
179
    }
180
181
    /// Export the current bookmark to the Git repository at `git_dir`,
182
    /// creating a bare one if nothing is there. Each state the bookmark
183
    /// passed through (per the operation log) becomes one commit whose tree
184
    /// is that state, so the Git branch is a linear history of full trees.
185
    pub fn git_export(&self, git_dir: &Path) -> Result<GitExportReport> {
186
        if !git_dir.join("HEAD").exists() && !git_dir.join(".git").exists() {
187
            fs::create_dir_all(git_dir)?;
188
            git(git_dir, &["init", "--bare", "-q"])?;
189
        }
190
        let marks_dir = self.git_marks_dir(git_dir)?;
191
        let git_marks = marks_dir.join("git-marks");
192
        let oxy_marks_path = marks_dir.join("oxy-marks");
193
        let mut marks = Self::read_oxy_marks(&oxy_marks_path)?;
194
        let mut next_mark = marks.values().copied().max().unwrap_or(0) + 1;
195
196
        let bookmark = self.config.bookmark.clone();
197
        let mut stream: Vec<u8> = Vec::new();
198
        let mut report = GitExportReport { branch: bookmark.clone(), heads: self.bookmark_heads()?.len(), ..Default::default() };
199
200
        // States along the bookmark, oldest first, skipping unchanged and pruned ones.
201
        let mut ops = self.op_log()?;
202
        ops.reverse();
203
        let mut prev: Option<(ObjectRef, Vec<ChangeHash>)> = None;
204
        let mut last_mark: Option<u64> = None;
205
        for (_, op) in &ops {
206
            let Some(state_ref) = op.view.bookmarks.get(&bookmark).copied() else { continue };
207
            if prev.as_ref().is_some_and(|(p, _)| *p == state_ref) {
208
                continue;
209
            }
210
            let Ok(state) = self.store.get_state(&state_ref) else { continue };
211
            if state.heads.is_empty() {
212
                prev = Some((state_ref, vec![]));
213
                continue;
214
            }
215
            if let Some(m) = marks.get(&state_ref) {
216
                report.skipped += 1;
217
                last_mark = Some(*m);
218
                prev = Some((state_ref, state.heads.clone()));
219
                continue;
220
            }
221
            let prev_heads = prev.as_ref().map(|(_, h)| h.clone()).unwrap_or_default();
222
            let added = self.negotiate(&prev_heads, &state.heads).unwrap_or_default();
223
            let loaded = self.load_state(&state.heads)?;
224
            let snapshot = loaded.materialize();
225
            let (author, secs, mut message) = match added.as_slice() {
226
                [h] => {
227
                    let c = self.store.get_change(h)?;
228
                    let mut m = c.message.clone();
229
                    if !m.ends_with('\n') {
230
                        m.push('\n');
231
                    }
232
                    m.push_str(&format!("\nChange-Id: {}\nOxy-Hash: {}\n", c.change_id.to_hex(), h.to_hex()));
233
                    (self.git_author(&c), c.hlc.wall_ms / 1000, m)
234
                }
235
                [] => ("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, format!("oxy: {}\n\n", op.command)),
236
                many => {
237
                    let mut m = format!("oxy: {} ({} changes)\n\n", op.command, many.len());
238
                    for h in many {
239
                        let c = self.store.get_change(h)?;
240
                        m.push_str(&format!("- {} {}\n", c.change_id.short(8), c.message.lines().next().unwrap_or("")));
241
                    }
242
                    m.push_str(&format!("\nOxy-Changes: {}\n", many.iter().map(|h| h.to_hex()).collect::<Vec<_>>().join(" ")));
243
                    ("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, m)
244
                }
245
            };
246
            message.push_str(&format!("Oxy-State: {}\n", state_ref.to_hex()));
247
            if snapshot.is_conflicted() {
248
                message.push_str("Oxy-Conflicted: true\n");
249
            }
250
            let mark = next_mark;
251
            next_mark += 1;
252
            let parents: Vec<u64> = last_mark.into_iter().collect();
253
            self.write_commit(&mut stream, "refs/oxy/export", mark, &author, secs, &message, &parents, &snapshot)?;
254
            marks.insert(state_ref, mark);
255
            last_mark = Some(mark);
256
            prev = Some((state_ref, state.heads.clone()));
257
            report.exported += 1;
258
        }
259
        match last_mark {
260
            Some(m) => stream.extend_from_slice(format!("reset refs/heads/{bookmark}\nfrom :{m}\n\ndone\n").as_bytes()),
261
            None => stream.extend_from_slice(b"done\n"),
262
        }
263
264
        let mut cmd = Command::new("git");
265
        // `--force`: the scratch ref may jump between unrelated states.
266
        cmd.arg("-C").arg(git_dir).args(["fast-import", "--quiet", "--done", "--force"]).arg(format!("--export-marks={}", git_marks.display()));
267
        if git_marks.exists() {
268
            cmd.arg(format!("--import-marks={}", git_marks.display()));
269
        }
270
        let mut child = cmd.stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::piped()).spawn().map_err(|e| RepoError::Msg(format!("running git fast-import: {e}")))?;
271
        child.stdin.take().unwrap().write_all(&stream)?;
272
        let out = child.wait_with_output()?;
273
        if !out.status.success() {
274
            return fail(format!("git fast-import failed: {}", String::from_utf8_lossy(&out.stderr).trim()));
275
        }
276
        let text: String = marks.iter().map(|(h, m)| format!("{} {m}\n", h.to_hex())).collect();
277
        write_atomic(&oxy_marks_path, text.as_bytes())?;
278
        let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]);
279
        Ok(report)
280
    }
281
c3069537 Andy McCall ↶ 282
    /// Import branch `branch` of the Git repository at `git_dir` into this
283
    /// (empty) repository's bookmark (spec §23): one change per commit, its
284
    /// edits the difference between the union of its parents' states and its
285
    /// tree, so merge commits become resolutions. Submodules are skipped.
286
    pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result<GitImportReport> {
287
        if !self.bookmark_heads()?.is_empty() {
288
            return fail("git import needs an empty bookmark; import into a fresh repository");
289
        }
290
        let commits = list_commits(git_dir, branch)?;
291
        let mut cat = CatFile::start(git_dir)?;
292
        let mut report = GitImportReport::default();
293
        let mut change_of: HashMap<String, ChangeHash> = HashMap::new();
294
        // The state after the most recently imported commit, reused when the
295
        // next commit's first parent is that commit.
296
        let mut last: Option<(String, State)> = None;
297
        let mut blob_cache: HashMap<String, Vec<u8>> = HashMap::new();
298
        let mut cache_bytes: usize = 0;
299
300
        for gc in &commits {
301
            // Base: union of every parent's state.
302
            let mut base = match (&gc.parents.first(), &last) {
303
                (Some(p1), Some((sha, st))) if *p1 == sha => st.clone(),
304
                (Some(p1), _) => self.git_ancestor_state(git_dir, &[p1.as_str()], &change_of)?,
305
                (None, _) => State::new(),
306
            };
307
            if gc.parents.len() > 1 {
308
                report.merges += 1;
309
                let mut args: Vec<String> = gc.parents[1..].to_vec();
310
                args.push(format!("^{}", gc.parents[0]));
311
                let extra = git(git_dir, &["rev-list".to_string()].iter().map(String::as_str).chain(args.iter().map(String::as_str)).collect::<Vec<_>>())?;
312
                for sha in extra.lines() {
313
                    if let Some(h) = change_of.get(sha) {
314
                        base.add_unchecked(self.store.get_change(h)?);
315
                    }
316
                }
317
            }
318
            let base_snapshot = base.materialize();
319
320
            // The commit's tree as in-memory entries.
321
            let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();
322
            for (path, (mode, sha)) in list_tree(git_dir, &gc.sha)? {
323
                let (kind, exec) = match mode.as_str() {
324
                    "100644" | "100664" => (NodeKind::File, false),
325
                    "100755" => (NodeKind::File, true),
326
                    "120000" => (NodeKind::Symlink, false),
327
                    "160000" => {
328
                        report.skipped_submodules += 1;
329
                        continue;
330
                    }
331
                    _ => continue,
332
                };
333
                let content = match blob_cache.get(&sha) {
334
                    Some(c) => c.clone(),
335
                    None => {
336
                        let c = cat.read(&sha)?;
337
                        if cache_bytes + c.len() > 256 * 1024 * 1024 {
338
                            blob_cache.clear();
339
                            cache_bytes = 0;
340
                        }
341
                        cache_bytes += c.len();
342
                        blob_cache.insert(sha.clone(), c.clone());
343
                        c
344
                    }
345
                };
346
                entries.insert(path, FsEntry { kind, exec, content });
347
            }
348
349
            let built = self.build_from_entries(&base, &base_snapshot, HashMap::new(), &entries)?;
350
            let mut deps: BTreeSet<ChangeHash> = built.contributors.iter().copied().collect();
351
            let mut message = gc.message.clone();
352
            if !message.ends_with('\n') {
353
                message.push('\n');
354
            }
355
            message.push_str(&format!("\nGit-Commit: {}\nGit-Author: {}\n", gc.sha, gc.author));
356
            let mut change = Change {
357
                change_id: change_id_for(&gc.sha, &gc.message),
358
                deps: vec![],
359
                author: ObjectRef::ZERO,
360
                hlc: Hlc { wall_ms: gc.author_secs * 1000, logical: 0 },
361
                message,
362
                tree_ops: built.tree_ops,
363
                edits: built.edits,
364
                kind: if built.resolves { ChangeKind::Resolution } else { ChangeKind::Normal },
365
            };
366
            deps.extend(referenced_changes(&change, &base));
367
            change.deps = deps.into_iter().collect();
368
            change.normalize();
369
            let hash = self.store.put_change(&change)?;
370
            base.add_unchecked(change);
371
            change_of.insert(gc.sha.clone(), hash);
372
            last = Some((gc.sha.clone(), base));
373
            report.commits += 1;
374
        }
375
376
        // The bookmark: heads of the tip's state; the working copy follows.
377
        let Some((_, tip_state)) = last else { return Ok(report) };
378
        let heads = tip_state.heads();
379
        report.heads = heads.len();
380
        let (op_ref, op) = self.head_op()?;
381
        let mut view = op.view.clone();
382
        view.bookmarks.insert(self.config.bookmark.clone(), self.store.put_state(&StateObject::new(heads.clone()))?);
383
        self.record_op(&[op_ref], &format!("oxy git import {} {branch}", git_dir.display()), view)?;
384
        self.checkout(&heads, true, "oxy git import (checkout)")?;
385
        Ok(report)
386
    }
387
388
    /// The state holding the changes of every Git ancestor of `shas`.
389
    fn git_ancestor_state(&self, git_dir: &Path, shas: &[&str], change_of: &HashMap<String, ChangeHash>) -> Result<State> {
390
        let mut args = vec!["rev-list"];
391
        args.extend(shas);
392
        let out = git(git_dir, &args)?;
393
        let mut st = State::new();
394
        for sha in out.lines() {
395
            if let Some(h) = change_of.get(sha) {
396
                st.add_unchecked(self.store.get_change(h)?);
397
            }
398
        }
399
        Ok(st)
400
    }
401
79ab8528 Andy McCall ↶ 402
    fn git_author(&self, c: &Change) -> String {
403
        match self.store.get_identity(&c.author) {
404
            Ok(id) if !id.name.is_empty() => format!("{} <{}>", id.name.replace(['<', '>', '\n'], " "), id.email.replace(['<', '>', '\n', ' '], "")),
405
            _ => "oxy <oxy@localhost>".to_string(),
406
        }
407
    }
408
409
    #[allow(clippy::too_many_arguments)]
410
    fn write_commit(&self, out: &mut Vec<u8>, reference: &str, mark: u64, author: &str, secs: u64, message: &str, parents: &[u64], snapshot: &Snapshot) -> Result<()> {
411
        out.extend_from_slice(format!("commit {reference}\nmark :{mark}\nauthor {author} {secs} +0000\ncommitter {author} {secs} +0000\ndata {}\n{message}\n", message.len()).as_bytes());
412
        for (i, p) in parents.iter().enumerate() {
413
            out.extend_from_slice(format!("{} :{p}\n", if i == 0 { "from" } else { "merge" }).as_bytes());
414
        }
415
        out.extend_from_slice(b"deleteall\n");
416
        for (path, f) in &snapshot.files {
417
            let (mode, bytes): (&str, Vec<u8>) = match (&f.kind, &f.content) {
418
                (NodeKind::Dir, _) => continue,
419
                (NodeKind::Symlink, FileContent::Opaque { blobs }) => ("120000", blobs.first().map(|b| self.read_blob(b)).transpose()?.unwrap_or_default()),
420
                (_, FileContent::Text { bytes, .. }) => (if f.mode.exec { "100755" } else { "100644" }, bytes.clone()),
421
                (_, FileContent::Opaque { blobs }) => (if f.mode.exec { "100755" } else { "100644" }, blobs.first().map(|b| self.read_blob(b)).transpose()?.unwrap_or_default()),
422
                (_, FileContent::Dir) => continue,
423
            };
424
            out.extend_from_slice(format!("M {mode} inline {path}\ndata {}\n", bytes.len()).as_bytes());
425
            out.extend_from_slice(&bytes);
426
            out.push(b'\n');
427
        }
428
        out.push(b'\n');
429
        Ok(())
430
    }
431
}