OXYGENHUB
# cdb5beba7f05 oxygen / crates / oxy-core / src / repo / git.rs
20.0 KiB · Rust · last changed Tags: immutable named states with tag attestations, in views and on the wire, policy-governed, exported to Git (spec 0.6h) 1 month ago source blame history raw
79ab8528 Andy McCall ↶ 1
//! Git bridge (spec §23), export side: each state the current bookmark
2
//! passed through becomes a Git commit through `git fast-import`, with the
3
//! state's full tree and the previous state's commit as parent. A state
4
//! that added exactly one change carries that change's message, author,
5
//! and `Change-Id`/`Oxy-Hash` trailers; every commit carries `Oxy-State`.
6
//!
7
//! Marks make re-exports incremental: `.oxy/git/<target>/git-marks` is
8
//! git's own marks file and `oxy-marks` maps state refs to marks.
9
10
use super::*;
11
use std::process::{Command, Stdio};
12
13
#[derive(Debug, Clone, Default, PartialEq, Eq)]
c3069537 Andy McCall ↶ 14
pub struct GitImportReport {
15
    pub commits: usize,
16
    pub merges: usize,
17
    pub skipped_submodules: usize,
18
    pub heads: usize,
19
}
20
21
/// A running `git cat-file --batch`, for reading many blobs cheaply.
22
struct CatFile {
23
    child: std::process::Child,
24
    stdin: std::process::ChildStdin,
25
    stdout: std::io::BufReader<std::process::ChildStdout>,
26
}
27
28
impl CatFile {
29
    fn start(git_dir: &Path) -> Result<CatFile> {
30
        let mut child = Command::new("git")
31
            .arg("-C")
32
            .arg(git_dir)
33
            .args(["cat-file", "--batch"])
34
            .stdin(Stdio::piped())
35
            .stdout(Stdio::piped())
36
            .stderr(Stdio::null())
37
            .spawn()
38
            .map_err(|e| RepoError::Msg(format!("running git cat-file: {e}")))?;
39
        let stdin = child.stdin.take().unwrap();
40
        let stdout = std::io::BufReader::new(child.stdout.take().unwrap());
41
        Ok(CatFile { child, stdin, stdout })
42
    }
43
44
    fn read(&mut self, sha: &str) -> Result<Vec<u8>> {
45
        use std::io::{BufRead, Read};
46
        writeln!(self.stdin, "{sha}")?;
47
        self.stdin.flush()?;
48
        let mut header = String::new();
49
        self.stdout.read_line(&mut header)?;
50
        let parts: Vec<&str> = header.trim_end().split(' ').collect();
51
        if parts.len() != 3 {
52
            return fail(format!("git cat-file: object {sha} not found"));
53
        }
54
        let size: usize = parts[2].parse().map_err(|_| RepoError::Msg("git cat-file: bad size".into()))?;
55
        let mut body = vec![0u8; size];
56
        self.stdout.read_exact(&mut body)?;
57
        let mut nl = [0u8; 1];
58
        self.stdout.read_exact(&mut nl)?;
59
        Ok(body)
60
    }
61
}
62
63
impl Drop for CatFile {
64
    fn drop(&mut self) {
65
        let _ = self.child.kill();
66
        let _ = self.child.wait();
67
    }
68
}
69
70
/// One commit as `git log` describes it.
71
struct GitCommit {
72
    sha: String,
73
    parents: Vec<String>,
74
    author_secs: u64,
75
    author: String,
76
    message: String,
77
}
78
79
fn list_commits(git_dir: &Path, branch: &str) -> Result<Vec<GitCommit>> {
80
    let out = Command::new("git")
81
        .arg("-C")
82
        .arg(git_dir)
83
        .args(["log", "--reverse", "--topo-order", "--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e", branch])
84
        .output()
85
        .map_err(|e| RepoError::Msg(format!("running git log: {e}")))?;
86
    if !out.status.success() {
87
        return fail(format!("git log {branch} failed: {}", String::from_utf8_lossy(&out.stderr).trim()));
88
    }
89
    let text = String::from_utf8_lossy(&out.stdout);
90
    let mut commits = Vec::new();
91
    for record in text.split('\x1e') {
92
        let record = record.trim_start_matches('\n');
93
        if record.trim().is_empty() {
94
            continue;
95
        }
96
        let f: Vec<&str> = record.splitn(5, '\0').collect();
97
        if f.len() != 5 {
98
            return fail("unexpected git log record");
99
        }
100
        commits.push(GitCommit {
101
            sha: f[0].to_string(),
102
            parents: f[1].split_whitespace().map(str::to_string).collect(),
103
            author_secs: f[2].parse().unwrap_or(0),
104
            author: f[3].to_string(),
105
            message: f[4].trim_end().to_string(),
106
        });
107
    }
108
    Ok(commits)
109
}
110
111
/// `path → (mode, blob sha)` for a commit's tree.
112
fn list_tree(git_dir: &Path, sha: &str) -> Result<BTreeMap<String, (String, String)>> {
113
    let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?;
114
    let mut out = BTreeMap::new();
115
    for entry in text.split('\0') {
116
        if entry.is_empty() {
117
            continue;
118
        }
119
        let Some((meta, path)) = entry.split_once('\t') else { continue };
120
        let m: Vec<&str> = meta.split(' ').collect();
121
        if m.len() == 3 {
122
            out.insert(path.to_string(), (m[0].to_string(), m[2].to_string()));
123
        }
124
    }
125
    Ok(out)
126
}
127
128
/// The ChangeId for a Git commit: its `Change-Id` trailer if it has a
129
/// 32-hex one, else BLAKE3-128 of the commit hash (spec §23).
130
fn change_id_for(sha: &str, message: &str) -> ChangeId {
131
    for line in message.lines().rev() {
132
        if let Some(v) = line.strip_prefix("Change-Id:") {
133
            if let Some(id) = ChangeId::from_hex(v.trim()) {
134
                return id;
135
            }
136
        }
137
    }
138
    let h = blake3::hash(sha.as_bytes());
139
    ChangeId(h.as_bytes()[..16].try_into().unwrap())
140
}
141
142
#[derive(Debug, Clone, Default, PartialEq, Eq)]
79ab8528 Andy McCall ↶ 143
pub struct GitExportReport {
144
    pub exported: usize,
145
    pub skipped: usize,
146
    pub branch: String,
147
    pub heads: usize,
148
}
149
150
fn git(dir: &Path, args: &[&str]) -> Result<String> {
151
    let out = Command::new("git").arg("-C").arg(dir).args(args).output().map_err(|e| RepoError::Msg(format!("running git: {e}")))?;
152
    if !out.status.success() {
153
        return fail(format!("git {} failed: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim()));
154
    }
155
    Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
156
}
157
158
impl Repo {
159
    fn git_marks_dir(&self, git_dir: &Path) -> Result<PathBuf> {
160
        let canonical = git_dir.canonicalize().unwrap_or_else(|_| git_dir.to_path_buf());
161
        let key = hash_object(ObjectType::Chunk, canonical.to_string_lossy().as_bytes()).short(16);
162
        let dir = self.oxy.join("git").join(key);
163
        fs::create_dir_all(&dir)?;
164
        Ok(dir)
165
    }
166
167
    fn read_oxy_marks(path: &Path) -> Result<BTreeMap<ObjectRef, u64>> {
168
        let mut out = BTreeMap::new();
169
        if let Ok(text) = fs::read_to_string(path) {
170
            for line in text.lines() {
171
                if let Some((h, m)) = line.split_once(' ') {
172
                    if let (Some(h), Ok(m)) = (ObjectRef::from_hex(h), m.parse()) {
173
                        out.insert(h, m);
174
                    }
175
                }
176
            }
177
        }
178
        Ok(out)
179
    }
180
181
    /// Export the current bookmark to the Git repository at `git_dir`,
182
    /// creating a bare one if nothing is there. Each state the bookmark
183
    /// passed through (per the operation log) becomes one commit whose tree
184
    /// is that state, so the Git branch is a linear history of full trees.
185
    pub fn git_export(&self, git_dir: &Path) -> Result<GitExportReport> {
186
        if !git_dir.join("HEAD").exists() && !git_dir.join(".git").exists() {
187
            fs::create_dir_all(git_dir)?;
188
            git(git_dir, &["init", "--bare", "-q"])?;
189
        }
190
        let marks_dir = self.git_marks_dir(git_dir)?;
191
        let git_marks = marks_dir.join("git-marks");
192
        let oxy_marks_path = marks_dir.join("oxy-marks");
193
        let mut marks = Self::read_oxy_marks(&oxy_marks_path)?;
194
        let mut next_mark = marks.values().copied().max().unwrap_or(0) + 1;
195
196
        let bookmark = self.config.bookmark.clone();
197
        let mut stream: Vec<u8> = Vec::new();
198
        let mut report = GitExportReport { branch: bookmark.clone(), heads: self.bookmark_heads()?.len(), ..Default::default() };
199
200
        // States along the bookmark, oldest first, skipping unchanged and pruned ones.
201
        let mut ops = self.op_log()?;
202
        ops.reverse();
203
        let mut prev: Option<(ObjectRef, Vec<ChangeHash>)> = None;
204
        let mut last_mark: Option<u64> = None;
205
        for (_, op) in &ops {
206
            let Some(state_ref) = op.view.bookmarks.get(&bookmark).copied() else { continue };
207
            if prev.as_ref().is_some_and(|(p, _)| *p == state_ref) {
208
                continue;
209
            }
210
            let Ok(state) = self.store.get_state(&state_ref) else { continue };
211
            if state.heads.is_empty() {
212
                prev = Some((state_ref, vec![]));
213
                continue;
214
            }
215
            if let Some(m) = marks.get(&state_ref) {
216
                report.skipped += 1;
217
                last_mark = Some(*m);
218
                prev = Some((state_ref, state.heads.clone()));
219
                continue;
220
            }
221
            let prev_heads = prev.as_ref().map(|(_, h)| h.clone()).unwrap_or_default();
222
            let added = self.negotiate(&prev_heads, &state.heads).unwrap_or_default();
223
            let loaded = self.load_state(&state.heads)?;
224
            let snapshot = loaded.materialize();
225
            let (author, secs, mut message) = match added.as_slice() {
226
                [h] => {
227
                    let c = self.store.get_change(h)?;
228
                    let mut m = c.message.clone();
229
                    if !m.ends_with('\n') {
230
                        m.push('\n');
231
                    }
232
                    m.push_str(&format!("\nChange-Id: {}\nOxy-Hash: {}\n", c.change_id.to_hex(), h.to_hex()));
233
                    (self.git_author(&c), c.hlc.wall_ms / 1000, m)
234
                }
235
                [] => ("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, format!("oxy: {}\n\n", op.command)),
236
                many => {
237
                    let mut m = format!("oxy: {} ({} changes)\n\n", op.command, many.len());
238
                    for h in many {
239
                        let c = self.store.get_change(h)?;
240
                        m.push_str(&format!("- {} {}\n", c.change_id.short(8), c.message.lines().next().unwrap_or("")));
241
                    }
242
                    m.push_str(&format!("\nOxy-Changes: {}\n", many.iter().map(|h| h.to_hex()).collect::<Vec<_>>().join(" ")));
243
                    ("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, m)
244
                }
245
            };
246
            message.push_str(&format!("Oxy-State: {}\n", state_ref.to_hex()));
247
            if snapshot.is_conflicted() {
248
                message.push_str("Oxy-Conflicted: true\n");
249
            }
250
            let mark = next_mark;
251
            next_mark += 1;
252
            let parents: Vec<u64> = last_mark.into_iter().collect();
253
            self.write_commit(&mut stream, "refs/oxy/export", mark, &author, secs, &message, &parents, &snapshot)?;
254
            marks.insert(state_ref, mark);
255
            last_mark = Some(mark);
256
            prev = Some((state_ref, state.heads.clone()));
257
            report.exported += 1;
258
        }
132b884b Andy McCall ↶ 259
        if let Some(m) = last_mark {
260
            stream.extend_from_slice(format!("reset refs/heads/{bookmark}\nfrom :{m}\n\n").as_bytes());
79ab8528 Andy McCall ↶ 261
        }
132b884b Andy McCall ↶ 262
        // Tags: a tag whose state was exported as a bookmark state points at
263
        // that commit; otherwise its state gets a commit of its own.
264
        for t in self.tags()? {
265
            let mark = match marks.get(&t.state) {
266
                Some(m) => *m,
267
                None => {
268
                    let loaded = self.load_state(&t.heads)?;
269
                    let snapshot = loaded.materialize();
270
                    let mark = next_mark;
271
                    next_mark += 1;
272
                    let message = format!("oxy: tag {}\n\n{}Oxy-State: {}\n", t.name, if t.message.is_empty() { String::new() } else { format!("{}\n\n", t.message) }, t.state.to_hex());
273
                    self.write_commit(&mut stream, "refs/oxy/export", mark, "oxy <oxy@localhost>", Hlc::now().wall_ms / 1000, &message, &[], &snapshot)?;
274
                    marks.insert(t.state, mark);
275
                    report.exported += 1;
276
                    mark
277
                }
278
            };
279
            stream.extend_from_slice(format!("reset refs/tags/{}\nfrom :{mark}\n\n", t.name).as_bytes());
280
        }
281
        stream.extend_from_slice(b"done\n");
79ab8528 Andy McCall ↶ 282
283
        let mut cmd = Command::new("git");
284
        // `--force`: the scratch ref may jump between unrelated states.
285
        cmd.arg("-C").arg(git_dir).args(["fast-import", "--quiet", "--done", "--force"]).arg(format!("--export-marks={}", git_marks.display()));
286
        if git_marks.exists() {
287
            cmd.arg(format!("--import-marks={}", git_marks.display()));
288
        }
289
        let mut child = cmd.stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::piped()).spawn().map_err(|e| RepoError::Msg(format!("running git fast-import: {e}")))?;
290
        child.stdin.take().unwrap().write_all(&stream)?;
291
        let out = child.wait_with_output()?;
292
        if !out.status.success() {
293
            return fail(format!("git fast-import failed: {}", String::from_utf8_lossy(&out.stderr).trim()));
294
        }
295
        let text: String = marks.iter().map(|(h, m)| format!("{} {m}\n", h.to_hex())).collect();
296
        write_atomic(&oxy_marks_path, text.as_bytes())?;
297
        let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]);
298
        Ok(report)
299
    }
300
c3069537 Andy McCall ↶ 301
    /// Import branch `branch` of the Git repository at `git_dir` into this
302
    /// (empty) repository's bookmark (spec §23): one change per commit, its
303
    /// edits the difference between the union of its parents' states and its
304
    /// tree, so merge commits become resolutions. Submodules are skipped.
305
    pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result<GitImportReport> {
306
        if !self.bookmark_heads()?.is_empty() {
307
            return fail("git import needs an empty bookmark; import into a fresh repository");
308
        }
309
        let commits = list_commits(git_dir, branch)?;
310
        let mut cat = CatFile::start(git_dir)?;
311
        let mut report = GitImportReport::default();
312
        let mut change_of: HashMap<String, ChangeHash> = HashMap::new();
313
        // The state after the most recently imported commit, reused when the
314
        // next commit's first parent is that commit.
315
        let mut last: Option<(String, State)> = None;
316
        let mut blob_cache: HashMap<String, Vec<u8>> = HashMap::new();
317
        let mut cache_bytes: usize = 0;
318
319
        for gc in &commits {
320
            // Base: union of every parent's state.
321
            let mut base = match (&gc.parents.first(), &last) {
322
                (Some(p1), Some((sha, st))) if *p1 == sha => st.clone(),
323
                (Some(p1), _) => self.git_ancestor_state(git_dir, &[p1.as_str()], &change_of)?,
324
                (None, _) => State::new(),
325
            };
326
            if gc.parents.len() > 1 {
327
                report.merges += 1;
328
                let mut args: Vec<String> = gc.parents[1..].to_vec();
329
                args.push(format!("^{}", gc.parents[0]));
330
                let extra = git(git_dir, &["rev-list".to_string()].iter().map(String::as_str).chain(args.iter().map(String::as_str)).collect::<Vec<_>>())?;
331
                for sha in extra.lines() {
332
                    if let Some(h) = change_of.get(sha) {
333
                        base.add_unchecked(self.store.get_change(h)?);
334
                    }
335
                }
336
            }
337
            let base_snapshot = base.materialize();
338
339
            // The commit's tree as in-memory entries.
340
            let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();
341
            for (path, (mode, sha)) in list_tree(git_dir, &gc.sha)? {
342
                let (kind, exec) = match mode.as_str() {
343
                    "100644" | "100664" => (NodeKind::File, false),
344
                    "100755" => (NodeKind::File, true),
345
                    "120000" => (NodeKind::Symlink, false),
346
                    "160000" => {
347
                        report.skipped_submodules += 1;
348
                        continue;
349
                    }
350
                    _ => continue,
351
                };
352
                let content = match blob_cache.get(&sha) {
353
                    Some(c) => c.clone(),
354
                    None => {
355
                        let c = cat.read(&sha)?;
356
                        if cache_bytes + c.len() > 256 * 1024 * 1024 {
357
                            blob_cache.clear();
358
                            cache_bytes = 0;
359
                        }
360
                        cache_bytes += c.len();
361
                        blob_cache.insert(sha.clone(), c.clone());
362
                        c
363
                    }
364
                };
365
                entries.insert(path, FsEntry { kind, exec, content });
366
            }
367
368
            let built = self.build_from_entries(&base, &base_snapshot, HashMap::new(), &entries)?;
369
            let mut deps: BTreeSet<ChangeHash> = built.contributors.iter().copied().collect();
370
            let mut message = gc.message.clone();
371
            if !message.ends_with('\n') {
372
                message.push('\n');
373
            }
374
            message.push_str(&format!("\nGit-Commit: {}\nGit-Author: {}\n", gc.sha, gc.author));
375
            let mut change = Change {
376
                change_id: change_id_for(&gc.sha, &gc.message),
377
                deps: vec![],
378
                author: ObjectRef::ZERO,
379
                hlc: Hlc { wall_ms: gc.author_secs * 1000, logical: 0 },
380
                message,
381
                tree_ops: built.tree_ops,
382
                edits: built.edits,
383
                kind: if built.resolves { ChangeKind::Resolution } else { ChangeKind::Normal },
384
            };
385
            deps.extend(referenced_changes(&change, &base));
386
            change.deps = deps.into_iter().collect();
387
            change.normalize();
388
            let hash = self.store.put_change(&change)?;
389
            base.add_unchecked(change);
390
            change_of.insert(gc.sha.clone(), hash);
391
            last = Some((gc.sha.clone(), base));
392
            report.commits += 1;
393
        }
394
395
        // The bookmark: heads of the tip's state; the working copy follows.
396
        let Some((_, tip_state)) = last else { return Ok(report) };
397
        let heads = tip_state.heads();
398
        report.heads = heads.len();
399
        let (op_ref, op) = self.head_op()?;
400
        let mut view = op.view.clone();
401
        view.bookmarks.insert(self.config.bookmark.clone(), self.store.put_state(&StateObject::new(heads.clone()))?);
402
        self.record_op(&[op_ref], &format!("oxy git import {} {branch}", git_dir.display()), view)?;
403
        self.checkout(&heads, true, "oxy git import (checkout)")?;
404
        Ok(report)
405
    }
406
407
    /// The state holding the changes of every Git ancestor of `shas`.
408
    fn git_ancestor_state(&self, git_dir: &Path, shas: &[&str], change_of: &HashMap<String, ChangeHash>) -> Result<State> {
409
        let mut args = vec!["rev-list"];
410
        args.extend(shas);
411
        let out = git(git_dir, &args)?;
412
        let mut st = State::new();
413
        for sha in out.lines() {
414
            if let Some(h) = change_of.get(sha) {
415
                st.add_unchecked(self.store.get_change(h)?);
416
            }
417
        }
418
        Ok(st)
419
    }
420
79ab8528 Andy McCall ↶ 421
    fn git_author(&self, c: &Change) -> String {
422
        match self.store.get_identity(&c.author) {
423
            Ok(id) if !id.name.is_empty() => format!("{} <{}>", id.name.replace(['<', '>', '\n'], " "), id.email.replace(['<', '>', '\n', ' '], "")),
424
            _ => "oxy <oxy@localhost>".to_string(),
425
        }
426
    }
427
428
    #[allow(clippy::too_many_arguments)]
429
    fn write_commit(&self, out: &mut Vec<u8>, reference: &str, mark: u64, author: &str, secs: u64, message: &str, parents: &[u64], snapshot: &Snapshot) -> Result<()> {
430
        out.extend_from_slice(format!("commit {reference}\nmark :{mark}\nauthor {author} {secs} +0000\ncommitter {author} {secs} +0000\ndata {}\n{message}\n", message.len()).as_bytes());
431
        for (i, p) in parents.iter().enumerate() {
432
            out.extend_from_slice(format!("{} :{p}\n", if i == 0 { "from" } else { "merge" }).as_bytes());
433
        }
434
        out.extend_from_slice(b"deleteall\n");
435
        for (path, f) in &snapshot.files {
436
            let (mode, bytes): (&str, Vec<u8>) = match (&f.kind, &f.content) {
437
                (NodeKind::Dir, _) => continue,
438
                (NodeKind::Symlink, FileContent::Opaque { blobs }) => ("120000", blobs.first().map(|b| self.read_blob(b)).transpose()?.unwrap_or_default()),
439
                (_, FileContent::Text { bytes, .. }) => (if f.mode.exec { "100755" } else { "100644" }, bytes.clone()),
440
                (_, FileContent::Opaque { blobs }) => (if f.mode.exec { "100755" } else { "100644" }, blobs.first().map(|b| self.read_blob(b)).transpose()?.unwrap_or_default()),
441
                (_, FileContent::Dir) => continue,
442
            };
443
            out.extend_from_slice(format!("M {mode} inline {path}\ndata {}\n", bytes.len()).as_bytes());
444
            out.extend_from_slice(&bytes);
445
            out.push(b'\n');
446
        }
447
        out.push(b'\n');
448
        Ok(())
449
    }
450
}