| 79ab8528 Andy McCall ↶ |
1 |
|
| 2 |
|
| 3 |
|
| 4 |
|
| 5 |
|
| 6 |
|
| 7 |
|
| 8 |
|
| 9 |
|
| 10 |
use super::*; |
| 11 |
use std::process::{Command, Stdio}; |
| 12 |
|
| 13 |
#[derive(Debug, Clone, Default, PartialEq, Eq)] |
| c3069537 Andy McCall ↶ |
14 |
pub struct GitImportReport { |
| 15 |
pub commits: usize, |
| 16 |
pub merges: usize, |
| 17 |
pub skipped_submodules: usize, |
| 18 |
pub heads: usize, |
| 19 |
} |
| 20 |
|
| 21 |
|
| 22 |
struct CatFile { |
| 23 |
child: std::process::Child, |
| 24 |
stdin: std::process::ChildStdin, |
| 25 |
stdout: std::io::BufReader<std::process::ChildStdout>, |
| 26 |
} |
| 27 |
|
| 28 |
impl CatFile { |
| 29 |
fn start(git_dir: &Path) -> Result<CatFile> { |
| 30 |
let mut child = Command::new("git") |
| 31 |
.arg("-C") |
| 32 |
.arg(git_dir) |
| 33 |
.args(["cat-file", "--batch"]) |
| 34 |
.stdin(Stdio::piped()) |
| 35 |
.stdout(Stdio::piped()) |
| 36 |
.stderr(Stdio::null()) |
| 37 |
.spawn() |
| 38 |
.map_err(|e| RepoError::Msg(format!("running git cat-file: {e}")))?; |
| 39 |
let stdin = child.stdin.take().unwrap(); |
| 40 |
let stdout = std::io::BufReader::new(child.stdout.take().unwrap()); |
| 41 |
Ok(CatFile { child, stdin, stdout }) |
| 42 |
} |
| 43 |
|
| 44 |
fn read(&mut self, sha: &str) -> Result<Vec<u8>> { |
| 45 |
use std::io::{BufRead, Read}; |
| 46 |
writeln!(self.stdin, "{sha}")?; |
| 47 |
self.stdin.flush()?; |
| 48 |
let mut header = String::new(); |
| 49 |
self.stdout.read_line(&mut header)?; |
| 50 |
let parts: Vec<&str> = header.trim_end().split(' ').collect(); |
| 51 |
if parts.len() != 3 { |
| 52 |
return fail(format!("git cat-file: object {sha} not found")); |
| 53 |
} |
| 54 |
let size: usize = parts[2].parse().map_err(|_| RepoError::Msg("git cat-file: bad size".into()))?; |
| 55 |
let mut body = vec![0u8; size]; |
| 56 |
self.stdout.read_exact(&mut body)?; |
| 57 |
let mut nl = [0u8; 1]; |
| 58 |
self.stdout.read_exact(&mut nl)?; |
| 59 |
Ok(body) |
| 60 |
} |
| 61 |
} |
| 62 |
|
| 63 |
impl Drop for CatFile { |
| 64 |
fn drop(&mut self) { |
| 65 |
let _ = self.child.kill(); |
| 66 |
let _ = self.child.wait(); |
| 67 |
} |
| 68 |
} |
| 69 |
|
| 70 |
|
| 71 |
struct GitCommit { |
| 72 |
sha: String, |
| 73 |
parents: Vec<String>, |
| 74 |
author_secs: u64, |
| 75 |
author: String, |
| 76 |
message: String, |
| 77 |
} |
| 78 |
|
| 79 |
fn list_commits(git_dir: &Path, branch: &str) -> Result<Vec<GitCommit>> { |
| 80 |
let out = Command::new("git") |
| 81 |
.arg("-C") |
| 82 |
.arg(git_dir) |
| 83 |
.args(["log", "--reverse", "--topo-order", "--format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e", branch]) |
| 84 |
.output() |
| 85 |
.map_err(|e| RepoError::Msg(format!("running git log: {e}")))?; |
| 86 |
if !out.status.success() { |
| 87 |
return fail(format!("git log {branch} failed: {}", String::from_utf8_lossy(&out.stderr).trim())); |
| 88 |
} |
| 89 |
let text = String::from_utf8_lossy(&out.stdout); |
| 90 |
let mut commits = Vec::new(); |
| 91 |
for record in text.split('\x1e') { |
| 92 |
let record = record.trim_start_matches('\n'); |
| 93 |
if record.trim().is_empty() { |
| 94 |
continue; |
| 95 |
} |
| 96 |
let f: Vec<&str> = record.splitn(5, '\0').collect(); |
| 97 |
if f.len() != 5 { |
| 98 |
return fail("unexpected git log record"); |
| 99 |
} |
| 100 |
commits.push(GitCommit { |
| 101 |
sha: f[0].to_string(), |
| 102 |
parents: f[1].split_whitespace().map(str::to_string).collect(), |
| 103 |
author_secs: f[2].parse().unwrap_or(0), |
| 104 |
author: f[3].to_string(), |
| 105 |
message: f[4].trim_end().to_string(), |
| 106 |
}); |
| 107 |
} |
| 108 |
Ok(commits) |
| 109 |
} |
| 110 |
|
| 111 |
|
| 112 |
fn list_tree(git_dir: &Path, sha: &str) -> Result<BTreeMap<String, (String, String)>> { |
| 113 |
let text = git(git_dir, &["ls-tree", "-r", "-z", sha])?; |
| 114 |
let mut out = BTreeMap::new(); |
| 115 |
for entry in text.split('\0') { |
| 116 |
if entry.is_empty() { |
| 117 |
continue; |
| 118 |
} |
| 119 |
let Some((meta, path)) = entry.split_once('\t') else { continue }; |
| 120 |
let m: Vec<&str> = meta.split(' ').collect(); |
| 121 |
if m.len() == 3 { |
| 122 |
out.insert(path.to_string(), (m[0].to_string(), m[2].to_string())); |
| 123 |
} |
| 124 |
} |
| 125 |
Ok(out) |
| 126 |
} |
| 127 |
|
| 128 |
|
| 129 |
|
| 130 |
fn change_id_for(sha: &str, message: &str) -> ChangeId { |
| 131 |
for line in message.lines().rev() { |
| 132 |
if let Some(v) = line.strip_prefix("Change-Id:") { |
| 133 |
if let Some(id) = ChangeId::from_hex(v.trim()) { |
| 134 |
return id; |
| 135 |
} |
| 136 |
} |
| 137 |
} |
| 138 |
let h = blake3::hash(sha.as_bytes()); |
| 139 |
ChangeId(h.as_bytes()[..16].try_into().unwrap()) |
| 140 |
} |
| 141 |
|
| 142 |
#[derive(Debug, Clone, Default, PartialEq, Eq)] |
| 79ab8528 Andy McCall ↶ |
143 |
pub struct GitExportReport { |
| 144 |
pub exported: usize, |
| 145 |
pub skipped: usize, |
| 146 |
pub branch: String, |
| 147 |
pub heads: usize, |
| 148 |
} |
| 149 |
|
| 150 |
fn git(dir: &Path, args: &[&str]) -> Result<String> { |
| 151 |
let out = Command::new("git").arg("-C").arg(dir).args(args).output().map_err(|e| RepoError::Msg(format!("running git: {e}")))?; |
| 152 |
if !out.status.success() { |
| 153 |
return fail(format!("git {} failed: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim())); |
| 154 |
} |
| 155 |
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string()) |
| 156 |
} |
| 157 |
|
| 158 |
impl Repo { |
| 159 |
fn git_marks_dir(&self, git_dir: &Path) -> Result<PathBuf> { |
| 160 |
let canonical = git_dir.canonicalize().unwrap_or_else(|_| git_dir.to_path_buf()); |
| 161 |
let key = hash_object(ObjectType::Chunk, canonical.to_string_lossy().as_bytes()).short(16); |
| 162 |
let dir = self.oxy.join("git").join(key); |
| 163 |
fs::create_dir_all(&dir)?; |
| 164 |
Ok(dir) |
| 165 |
} |
| 166 |
|
| 167 |
fn read_oxy_marks(path: &Path) -> Result<BTreeMap<ObjectRef, u64>> { |
| 168 |
let mut out = BTreeMap::new(); |
| 169 |
if let Ok(text) = fs::read_to_string(path) { |
| 170 |
for line in text.lines() { |
| 171 |
if let Some((h, m)) = line.split_once(' ') { |
| 172 |
if let (Some(h), Ok(m)) = (ObjectRef::from_hex(h), m.parse()) { |
| 173 |
out.insert(h, m); |
| 174 |
} |
| 175 |
} |
| 176 |
} |
| 177 |
} |
| 178 |
Ok(out) |
| 179 |
} |
| 180 |
|
| 181 |
|
| 182 |
|
| 183 |
|
| 184 |
|
| 185 |
pub fn git_export(&self, git_dir: &Path) -> Result<GitExportReport> { |
| 186 |
if !git_dir.join("HEAD").exists() && !git_dir.join(".git").exists() { |
| 187 |
fs::create_dir_all(git_dir)?; |
| 188 |
git(git_dir, &["init", "--bare", "-q"])?; |
| 189 |
} |
| 190 |
let marks_dir = self.git_marks_dir(git_dir)?; |
| 191 |
let git_marks = marks_dir.join("git-marks"); |
| 192 |
let oxy_marks_path = marks_dir.join("oxy-marks"); |
| 193 |
let mut marks = Self::read_oxy_marks(&oxy_marks_path)?; |
| 194 |
let mut next_mark = marks.values().copied().max().unwrap_or(0) + 1; |
| 195 |
|
| 196 |
let bookmark = self.config.bookmark.clone(); |
| 197 |
let mut stream: Vec<u8> = Vec::new(); |
| 198 |
let mut report = GitExportReport { branch: bookmark.clone(), heads: self.bookmark_heads()?.len(), ..Default::default() }; |
| 199 |
|
| 200 |
|
| 201 |
let mut ops = self.op_log()?; |
| 202 |
ops.reverse(); |
| 203 |
let mut prev: Option<(ObjectRef, Vec<ChangeHash>)> = None; |
| 204 |
let mut last_mark: Option<u64> = None; |
| 205 |
for (_, op) in &ops { |
| 206 |
let Some(state_ref) = op.view.bookmarks.get(&bookmark).copied() else { continue }; |
| 207 |
if prev.as_ref().is_some_and(|(p, _)| *p == state_ref) { |
| 208 |
continue; |
| 209 |
} |
| 210 |
let Ok(state) = self.store.get_state(&state_ref) else { continue }; |
| 211 |
if state.heads.is_empty() { |
| 212 |
prev = Some((state_ref, vec![])); |
| 213 |
continue; |
| 214 |
} |
| 215 |
if let Some(m) = marks.get(&state_ref) { |
| 216 |
report.skipped += 1; |
| 217 |
last_mark = Some(*m); |
| 218 |
prev = Some((state_ref, state.heads.clone())); |
| 219 |
continue; |
| 220 |
} |
| 221 |
let prev_heads = prev.as_ref().map(|(_, h)| h.clone()).unwrap_or_default(); |
| 222 |
let added = self.negotiate(&prev_heads, &state.heads).unwrap_or_default(); |
| 223 |
let loaded = self.load_state(&state.heads)?; |
| 224 |
let snapshot = loaded.materialize(); |
| 225 |
let (author, secs, mut message) = match added.as_slice() { |
| 226 |
[h] => { |
| 227 |
let c = self.store.get_change(h)?; |
| 228 |
let mut m = c.message.clone(); |
| 229 |
if !m.ends_with('\n') { |
| 230 |
m.push('\n'); |
| 231 |
} |
| 232 |
m.push_str(&format!("\nChange-Id: {}\nOxy-Hash: {}\n", c.change_id.to_hex(), h.to_hex())); |
| 233 |
(self.git_author(&c), c.hlc.wall_ms / 1000, m) |
| 234 |
} |
| 235 |
[] => ("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, format!("oxy: {}\n\n", op.command)), |
| 236 |
many => { |
| 237 |
let mut m = format!("oxy: {} ({} changes)\n\n", op.command, many.len()); |
| 238 |
for h in many { |
| 239 |
let c = self.store.get_change(h)?; |
| 240 |
m.push_str(&format!("- {} {}\n", c.change_id.short(8), c.message.lines().next().unwrap_or(""))); |
| 241 |
} |
| 242 |
m.push_str(&format!("\nOxy-Changes: {}\n", many.iter().map(|h| h.to_hex()).collect::<Vec<_>>().join(" "))); |
| 243 |
("oxy <oxy@localhost>".to_string(), op.hlc.wall_ms / 1000, m) |
| 244 |
} |
| 245 |
}; |
| 246 |
message.push_str(&format!("Oxy-State: {}\n", state_ref.to_hex())); |
| 247 |
if snapshot.is_conflicted() { |
| 248 |
message.push_str("Oxy-Conflicted: true\n"); |
| 249 |
} |
| 250 |
let mark = next_mark; |
| 251 |
next_mark += 1; |
| 252 |
let parents: Vec<u64> = last_mark.into_iter().collect(); |
| 253 |
self.write_commit(&mut stream, "refs/oxy/export", mark, &author, secs, &message, &parents, &snapshot)?; |
| 254 |
marks.insert(state_ref, mark); |
| 255 |
last_mark = Some(mark); |
| 256 |
prev = Some((state_ref, state.heads.clone())); |
| 257 |
report.exported += 1; |
| 258 |
} |
| 259 |
match last_mark { |
| 260 |
Some(m) => stream.extend_from_slice(format!("reset refs/heads/{bookmark}\nfrom :{m}\n\ndone\n").as_bytes()), |
| 261 |
None => stream.extend_from_slice(b"done\n"), |
| 262 |
} |
| 263 |
|
| 264 |
let mut cmd = Command::new("git"); |
| 265 |
|
| 266 |
cmd.arg("-C").arg(git_dir).args(["fast-import", "--quiet", "--done", "--force"]).arg(format!("--export-marks={}", git_marks.display())); |
| 267 |
if git_marks.exists() { |
| 268 |
cmd.arg(format!("--import-marks={}", git_marks.display())); |
| 269 |
} |
| 270 |
let mut child = cmd.stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::piped()).spawn().map_err(|e| RepoError::Msg(format!("running git fast-import: {e}")))?; |
| 271 |
child.stdin.take().unwrap().write_all(&stream)?; |
| 272 |
let out = child.wait_with_output()?; |
| 273 |
if !out.status.success() { |
| 274 |
return fail(format!("git fast-import failed: {}", String::from_utf8_lossy(&out.stderr).trim())); |
| 275 |
} |
| 276 |
let text: String = marks.iter().map(|(h, m)| format!("{} {m}\n", h.to_hex())).collect(); |
| 277 |
write_atomic(&oxy_marks_path, text.as_bytes())?; |
| 278 |
let _ = git(git_dir, &["update-ref", "-d", "refs/oxy/export"]); |
| 279 |
Ok(report) |
| 280 |
} |
| 281 |
|
| c3069537 Andy McCall ↶ |
282 |
|
| 283 |
|
| 284 |
|
| 285 |
|
| 286 |
pub fn import_git(&self, git_dir: &Path, branch: &str) -> Result<GitImportReport> { |
| 287 |
if !self.bookmark_heads()?.is_empty() { |
| 288 |
return fail("git import needs an empty bookmark; import into a fresh repository"); |
| 289 |
} |
| 290 |
let commits = list_commits(git_dir, branch)?; |
| 291 |
let mut cat = CatFile::start(git_dir)?; |
| 292 |
let mut report = GitImportReport::default(); |
| 293 |
let mut change_of: HashMap<String, ChangeHash> = HashMap::new(); |
| 294 |
|
| 295 |
|
| 296 |
let mut last: Option<(String, State)> = None; |
| 297 |
let mut blob_cache: HashMap<String, Vec<u8>> = HashMap::new(); |
| 298 |
let mut cache_bytes: usize = 0; |
| 299 |
|
| 300 |
for gc in &commits { |
| 301 |
|
| 302 |
let mut base = match (&gc.parents.first(), &last) { |
| 303 |
(Some(p1), Some((sha, st))) if *p1 == sha => st.clone(), |
| 304 |
(Some(p1), _) => self.git_ancestor_state(git_dir, &[p1.as_str()], &change_of)?, |
| 305 |
(None, _) => State::new(), |
| 306 |
}; |
| 307 |
if gc.parents.len() > 1 { |
| 308 |
report.merges += 1; |
| 309 |
let mut args: Vec<String> = gc.parents[1..].to_vec(); |
| 310 |
args.push(format!("^{}", gc.parents[0])); |
| 311 |
let extra = git(git_dir, &["rev-list".to_string()].iter().map(String::as_str).chain(args.iter().map(String::as_str)).collect::<Vec<_>>())?; |
| 312 |
for sha in extra.lines() { |
| 313 |
if let Some(h) = change_of.get(sha) { |
| 314 |
base.add_unchecked(self.store.get_change(h)?); |
| 315 |
} |
| 316 |
} |
| 317 |
} |
| 318 |
let base_snapshot = base.materialize(); |
| 319 |
|
| 320 |
|
| 321 |
let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new(); |
| 322 |
for (path, (mode, sha)) in list_tree(git_dir, &gc.sha)? { |
| 323 |
let (kind, exec) = match mode.as_str() { |
| 324 |
"100644" | "100664" => (NodeKind::File, false), |
| 325 |
"100755" => (NodeKind::File, true), |
| 326 |
"120000" => (NodeKind::Symlink, false), |
| 327 |
"160000" => { |
| 328 |
report.skipped_submodules += 1; |
| 329 |
continue; |
| 330 |
} |
| 331 |
_ => continue, |
| 332 |
}; |
| 333 |
let content = match blob_cache.get(&sha) { |
| 334 |
Some(c) => c.clone(), |
| 335 |
None => { |
| 336 |
let c = cat.read(&sha)?; |
| 337 |
if cache_bytes + c.len() > 256 * 1024 * 1024 { |
| 338 |
blob_cache.clear(); |
| 339 |
cache_bytes = 0; |
| 340 |
} |
| 341 |
cache_bytes += c.len(); |
| 342 |
blob_cache.insert(sha.clone(), c.clone()); |
| 343 |
c |
| 344 |
} |
| 345 |
}; |
| 346 |
entries.insert(path, FsEntry { kind, exec, content }); |
| 347 |
} |
| 348 |
|
| 349 |
let built = self.build_from_entries(&base, &base_snapshot, HashMap::new(), &entries)?; |
| 350 |
let mut deps: BTreeSet<ChangeHash> = built.contributors.iter().copied().collect(); |
| 351 |
let mut message = gc.message.clone(); |
| 352 |
if !message.ends_with('\n') { |
| 353 |
message.push('\n'); |
| 354 |
} |
| 355 |
message.push_str(&format!("\nGit-Commit: {}\nGit-Author: {}\n", gc.sha, gc.author)); |
| 356 |
let mut change = Change { |
| 357 |
change_id: change_id_for(&gc.sha, &gc.message), |
| 358 |
deps: vec![], |
| 359 |
author: ObjectRef::ZERO, |
| 360 |
hlc: Hlc { wall_ms: gc.author_secs * 1000, logical: 0 }, |
| 361 |
message, |
| 362 |
tree_ops: built.tree_ops, |
| 363 |
edits: built.edits, |
| 364 |
kind: if built.resolves { ChangeKind::Resolution } else { ChangeKind::Normal }, |
| 365 |
}; |
| 366 |
deps.extend(referenced_changes(&change, &base)); |
| 367 |
change.deps = deps.into_iter().collect(); |
| 368 |
change.normalize(); |
| 369 |
let hash = self.store.put_change(&change)?; |
| 370 |
base.add_unchecked(change); |
| 371 |
change_of.insert(gc.sha.clone(), hash); |
| 372 |
last = Some((gc.sha.clone(), base)); |
| 373 |
report.commits += 1; |
| 374 |
} |
| 375 |
|
| 376 |
|
| 377 |
let Some((_, tip_state)) = last else { return Ok(report) }; |
| 378 |
let heads = tip_state.heads(); |
| 379 |
report.heads = heads.len(); |
| 380 |
let (op_ref, op) = self.head_op()?; |
| 381 |
let mut view = op.view.clone(); |
| 382 |
view.bookmarks.insert(self.config.bookmark.clone(), self.store.put_state(&StateObject::new(heads.clone()))?); |
| 383 |
self.record_op(&[op_ref], &format!("oxy git import {} {branch}", git_dir.display()), view)?; |
| 384 |
self.checkout(&heads, true, "oxy git import (checkout)")?; |
| 385 |
Ok(report) |
| 386 |
} |
| 387 |
|
| 388 |
|
| 389 |
fn git_ancestor_state(&self, git_dir: &Path, shas: &[&str], change_of: &HashMap<String, ChangeHash>) -> Result<State> { |
| 390 |
let mut args = vec!["rev-list"]; |
| 391 |
args.extend(shas); |
| 392 |
let out = git(git_dir, &args)?; |
| 393 |
let mut st = State::new(); |
| 394 |
for sha in out.lines() { |
| 395 |
if let Some(h) = change_of.get(sha) { |
| 396 |
st.add_unchecked(self.store.get_change(h)?); |
| 397 |
} |
| 398 |
} |
| 399 |
Ok(st) |
| 400 |
} |
| 401 |
|
| 79ab8528 Andy McCall ↶ |
402 |
fn git_author(&self, c: &Change) -> String { |
| 403 |
match self.store.get_identity(&c.author) { |
| 404 |
Ok(id) if !id.name.is_empty() => format!("{} <{}>", id.name.replace(['<', '>', '\n'], " "), id.email.replace(['<', '>', '\n', ' '], "")), |
| 405 |
_ => "oxy <oxy@localhost>".to_string(), |
| 406 |
} |
| 407 |
} |
| 408 |
|
| 409 |
#[allow(clippy::too_many_arguments)] |
| 410 |
fn write_commit(&self, out: &mut Vec<u8>, reference: &str, mark: u64, author: &str, secs: u64, message: &str, parents: &[u64], snapshot: &Snapshot) -> Result<()> { |
| 411 |
out.extend_from_slice(format!("commit {reference}\nmark :{mark}\nauthor {author} {secs} +0000\ncommitter {author} {secs} +0000\ndata {}\n{message}\n", message.len()).as_bytes()); |
| 412 |
for (i, p) in parents.iter().enumerate() { |
| 413 |
out.extend_from_slice(format!("{} :{p}\n", if i == 0 { "from" } else { "merge" }).as_bytes()); |
| 414 |
} |
| 415 |
out.extend_from_slice(b"deleteall\n"); |
| 416 |
for (path, f) in &snapshot.files { |
| 417 |
let (mode, bytes): (&str, Vec<u8>) = match (&f.kind, &f.content) { |
| 418 |
(NodeKind::Dir, _) => continue, |
| 419 |
(NodeKind::Symlink, FileContent::Opaque { blobs }) => ("120000", blobs.first().map(|b| self.read_blob(b)).transpose()?.unwrap_or_default()), |
| 420 |
(_, FileContent::Text { bytes, .. }) => (if f.mode.exec { "100755" } else { "100644" }, bytes.clone()), |
| 421 |
(_, FileContent::Opaque { blobs }) => (if f.mode.exec { "100755" } else { "100644" }, blobs.first().map(|b| self.read_blob(b)).transpose()?.unwrap_or_default()), |
| 422 |
(_, FileContent::Dir) => continue, |
| 423 |
}; |
| 424 |
out.extend_from_slice(format!("M {mode} inline {path}\ndata {}\n", bytes.len()).as_bytes()); |
| 425 |
out.extend_from_slice(&bytes); |
| 426 |
out.push(b'\n'); |
| 427 |
} |
| 428 |
out.push(b'\n'); |
| 429 |
Ok(()) |
| 430 |
} |
| 431 |
} |