79ab8528 Andy McCall 1 month ago ↶
1
2
3
4
5
6
7
8
9
10
use super :: * ;
11
use std:: process:: { Command, Stdio} ;
12
13
# [ derive ( Debug, Clone, Default, PartialEq, Eq ) ]
c3069537 Andy McCall 1 month ago ↶
14
pub struct GitImportReport {
15
pub commits : usize ,
16
pub merges : usize ,
17
pub skipped_submodules : usize ,
18
pub heads : usize ,
19
}
20
21
22
struct CatFile {
23
child : std:: process:: Child,
24
stdin : std:: process:: ChildStdin,
25
stdout : std:: io:: BufReader< std:: process:: ChildStdout> ,
26
}
27
28
impl CatFile {
29
fn start ( git_dir : & Path) -> Result< CatFile> {
30
let mut child = Command:: new( " git" )
31
.arg ( " -C" )
32
.arg ( git_dir)
33
.args ( [ " cat-file" , " --batch" ] )
34
.stdin ( Stdio:: piped( ) )
35
.stdout ( Stdio:: piped( ) )
36
.stderr ( Stdio:: null( ) )
37
.spawn ( )
38
.map_err ( | e | RepoError:: Msg( format! ( " running git cat-file: {e} " ) ) ) ? ;
39
let stdin = child.stdin.take ( ) .unwrap ( ) ;
40
let stdout = std:: io:: BufReader:: new( child.stdout.take ( ) .unwrap ( ) ) ;
41
Ok ( CatFile { child, stdin, stdout } )
42
}
43
44
fn read ( & mut self , sha : & str ) -> Result< Vec< u8 > > {
45
use std:: io:: { BufRead, Read} ;
46
writeln! ( self.stdin, " {sha} " ) ? ;
47
self .stdin.flush ( ) ? ;
48
let mut header = String :: new( ) ;
49
self .stdout.read_line ( & mut header) ? ;
50
let parts: Vec< & str > = header.trim_end ( ) .split ( ' ' ) .collect ( ) ;
51
if parts.len ( ) ! = 3 {
52
return fail ( format! ( " git cat-file: object {sha} not found" ) ) ;
53
}
54
let size: usize = parts[ 2 ] .parse ( ) .map_err ( | _| RepoError:: Msg( " git cat-file: bad size" .into ( ) ) ) ? ;
55
let mut body = vec! [ 0 u8 ; size] ;
56
self .stdout.read_exact ( & mut body) ? ;
57
let mut nl = [ 0 u8 ; 1 ] ;
58
self .stdout.read_exact ( & mut nl) ? ;
59
Ok ( body)
60
}
61
}
62
63
impl Drop for CatFile {
64
fn drop ( & mut self ) {
65
let _ = self .child.kill ( ) ;
66
let _ = self .child.wait ( ) ;
67
}
68
}
69
70
71
struct GitCommit {
72
sha : String,
73
parents : Vec< String> ,
74
author_secs : u64 ,
75
author : String,
76
message : String,
77
}
78
79
fn list_commits ( git_dir : & Path, branch : & str ) -> Result< Vec< GitCommit> > {
80
let out = Command:: new( " git" )
81
.arg ( " -C" )
82
.arg ( git_dir)
83
.args ( [ " log" , " --reverse" , " --topo-order" , " --format=%H%x00%P%x00%at%x00%an <%ae>%x00%B%x1e" , branch] )
84
.output ( )
85
.map_err ( | e | RepoError:: Msg( format! ( " running git log: {e} " ) ) ) ? ;
86
if ! out.status.success ( ) {
87
return fail ( format! ( " git log {branch} failed: {} " , String :: from_utf8_lossy( & out.stderr) .trim ( ) ) ) ;
88
}
89
let text = String :: from_utf8_lossy( & out.stdout) ;
90
let mut commits = Vec :: new( ) ;
91
for record in text.split ( ' \x1e ' ) {
92
let record = record.trim_start_matches ( ' \n ' ) ;
93
if record.trim ( ) .is_empty ( ) {
94
continue ;
95
}
96
let f: Vec< & str > = record.splitn ( 5 , ' \0 ' ) .collect ( ) ;
97
if f.len ( ) ! = 5 {
98
return fail ( " unexpected git log record" ) ;
99
}
100
commits.push ( GitCommit {
101
sha: f[ 0 ] .to_string ( ) ,
102
parents: f[ 1 ] .split_whitespace ( ) .map ( str :: to_string) .collect ( ) ,
103
author_secs: f[ 2 ] .parse ( ) .unwrap_or ( 0 ) ,
104
author: f[ 3 ] .to_string ( ) ,
105
message: f[ 4 ] .trim_end ( ) .to_string ( ) ,
106
} ) ;
107
}
108
Ok ( commits)
109
}
110
111
112
fn list_tree ( git_dir : & Path, sha : & str ) -> Result< BTreeMap< String, ( String, String) > > {
113
let text = git ( git_dir, & [ " ls-tree" , " -r" , " -z" , sha] ) ? ;
114
let mut out = BTreeMap:: new( ) ;
115
for entry in text.split ( ' \0 ' ) {
116
if entry.is_empty ( ) {
117
continue ;
118
}
119
let Some ( ( meta, path) ) = entry.split_once ( ' \t ' ) else { continue } ;
120
let m: Vec< & str > = meta.split ( ' ' ) .collect ( ) ;
121
if m.len ( ) = = 3 {
122
out.insert ( path.to_string ( ) , ( m[ 0 ] .to_string ( ) , m[ 2 ] .to_string ( ) ) ) ;
123
}
124
}
125
Ok ( out)
126
}
127
128
129
130
fn change_id_for ( sha : & str , message : & str ) -> ChangeId {
131
for line in message.lines ( ) .rev ( ) {
132
if let Some ( v) = line.strip_prefix ( " Change-Id:" ) {
133
if let Some ( id) = ChangeId:: from_hex( v.trim ( ) ) {
134
return id;
135
}
136
}
137
}
138
let h = blake3:: hash( sha.as_bytes ( ) ) ;
139
ChangeId( h.as_bytes ( ) [ .. 16 ] .try_into ( ) .unwrap ( ) )
140
}
141
142
# [ derive ( Debug, Clone, Default, PartialEq, Eq ) ]
79ab8528 Andy McCall 1 month ago ↶
143
pub struct GitExportReport {
144
pub exported : usize ,
145
pub skipped : usize ,
146
pub branch : String,
147
pub heads : usize ,
148
}
149
150
fn git ( dir : & Path, args : & [& str ]) -> Result< String> {
151
let out = Command:: new( " git" ) .arg ( " -C" ) .arg ( dir) .args ( args) .output ( ) .map_err ( | e | RepoError:: Msg( format! ( " running git: {e} " ) ) ) ? ;
152
if ! out.status.success ( ) {
153
return fail ( format! ( " git {} failed: {} " , args.join ( " " ) , String :: from_utf8_lossy( & out.stderr) .trim ( ) ) ) ;
154
}
155
Ok ( String :: from_utf8_lossy( & out.stdout) .trim ( ) .to_string ( ) )
156
}
157
158
impl Repo {
159
fn git_marks_dir ( & self , git_dir : & Path) -> Result< PathBuf> {
160
let canonical = git_dir.canonicalize ( ) .unwrap_or_else ( | _| git_dir.to_path_buf ( ) ) ;
161
let key = hash_object ( ObjectType:: Chunk, canonical.to_string_lossy ( ) .as_bytes ( ) ) .short ( 16 ) ;
162
let dir = self .oxy.join ( " git" ) .join ( key) ;
163
fs:: create_dir_all( & dir) ? ;
164
Ok ( dir)
165
}
166
167
fn read_oxy_marks ( path : & Path) -> Result< BTreeMap< ObjectRef, u64 > > {
168
let mut out = BTreeMap:: new( ) ;
169
if let Ok ( text) = fs:: read_to_string( path) {
170
for line in text.lines ( ) {
171
if let Some ( ( h, m) ) = line.split_once ( ' ' ) {
172
if let ( Some ( h) , Ok ( m) ) = ( ObjectRef:: from_hex( h) , m.parse ( ) ) {
173
out.insert ( h, m) ;
174
}
175
}
176
}
177
}
178
Ok ( out)
179
}
180
181
182
183
184
185
pub fn git_export ( & self , git_dir : & Path) -> Result< GitExportReport> {
186
if ! git_dir.join ( " HEAD" ) .exists ( ) & & ! git_dir.join ( " .git" ) .exists ( ) {
187
fs:: create_dir_all( git_dir) ? ;
188
git ( git_dir, & [ " init" , " --bare" , " -q" ] ) ? ;
189
}
190
let marks_dir = self .git_marks_dir ( git_dir) ? ;
191
let git_marks = marks_dir.join ( " git-marks" ) ;
192
let oxy_marks_path = marks_dir.join ( " oxy-marks" ) ;
193
let mut marks = Self :: read_oxy_marks( & oxy_marks_path) ? ;
194
let mut next_mark = marks.values ( ) .copied ( ) .max ( ) .unwrap_or ( 0 ) + 1 ;
195
196
let bookmark = self .config.bookmark.clone ( ) ;
197
let mut stream: Vec< u8 > = Vec :: new( ) ;
198
let mut report = GitExportReport { branch: bookmark.clone ( ) , heads: self .bookmark_heads ( ) ? .len ( ) , .. Default :: default( ) } ;
199
200
201
let mut ops = self .op_log ( ) ? ;
202
ops.reverse ( ) ;
203
let mut prev: Option< ( ObjectRef, Vec< ChangeHash> ) > = None ;
204
let mut last_mark: Option< u64 > = None ;
205
for ( _ , op) in & ops {
206
let Some ( state_ref) = op.view.bookmarks.get ( & bookmark) .copied ( ) else { continue } ;
207
if prev.as_ref ( ) .is_some_and ( | ( p , _) | * p = = state_ref) {
208
continue ;
209
}
210
let Ok ( state) = self .store.get_state ( & state_ref) else { continue } ;
211
if state.heads.is_empty ( ) {
212
prev = Some ( ( state_ref, vec! [ ] ) ) ;
213
continue ;
214
}
215
if let Some ( m) = marks.get ( & state_ref) {
216
report.skipped + = 1 ;
217
last_mark = Some ( * m) ;
218
prev = Some ( ( state_ref, state.heads.clone ( ) ) ) ;
219
continue ;
220
}
221
let prev_heads = prev.as_ref ( ) .map ( | ( _, h ) | h.clone ( ) ) .unwrap_or_default ( ) ;
222
let added = self .negotiate ( & prev_heads, & state.heads) .unwrap_or_default ( ) ;
223
let loaded = self .load_state ( & state.heads) ? ;
224
let snapshot = loaded.materialize ( ) ;
225
let ( author, secs, mut message) = match added.as_slice ( ) {
226
[ h] => {
227
let c = self .store.get_change ( h) ? ;
228
let mut m = c.message.clone ( ) ;
229
if ! m.ends_with ( ' \n ' ) {
230
m.push ( ' \n ' ) ;
231
}
232
m.push_str ( & format! ( " \n Change-Id: {} \n Oxy-Hash: {} \n " , c.change_id.to_hex ( ) , h.to_hex ( ) ) ) ;
233
( self .git_author ( & c) , c.hlc.wall_ms / 1000 , m)
234
}
235
[ ] => ( " oxy <oxy@localhost>" .to_string ( ) , op.hlc.wall_ms / 1000 , format! ( " oxy: {} \n \n " , op.command) ) ,
236
many => {
237
let mut m = format! ( " oxy: {} ({} changes)\n \n " , op.command, many.len ( ) ) ;
238
for h in many {
239
let c = self .store.get_change ( h) ? ;
240
m.push_str ( & format! ( " - {} {} \n " , c.change_id.short ( 8 ) , c.message.lines ( ) .next ( ) .unwrap_or ( " " ) ) ) ;
241
}
242
m.push_str ( & format! ( " \n Oxy-Changes: {} \n " , many.iter ( ) .map ( | h | h.to_hex ( ) ) .collect:: < Vec< _ > > ( ) .join ( " " ) ) ) ;
243
( " oxy <oxy@localhost>" .to_string ( ) , op.hlc.wall_ms / 1000 , m)
244
}
245
} ;
246
message.push_str ( & format! ( " Oxy-State: {} \n " , state_ref.to_hex ( ) ) ) ;
247
if snapshot.is_conflicted ( ) {
248
message.push_str ( " Oxy-Conflicted: true\n " ) ;
249
}
250
let mark = next_mark;
251
next_mark + = 1 ;
252
let parents: Vec< u64 > = last_mark.into_iter ( ) .collect ( ) ;
253
self .write_commit ( & mut stream, " refs/oxy/export" , mark, & author, secs, & message, & parents, & snapshot) ? ;
254
marks.insert ( state_ref, mark) ;
255
last_mark = Some ( mark) ;
256
prev = Some ( ( state_ref, state.heads.clone ( ) ) ) ;
257
report.exported + = 1 ;
258
}
132b884b Andy McCall 1 month ago ↶
259
if let Some ( m) = last_mark {
260
stream.extend_from_slice ( format! ( " reset refs/heads/{bookmark} \n from :{m} \n \n " ) .as_bytes ( ) ) ;
79ab8528 Andy McCall 1 month ago ↶
261
}
132b884b Andy McCall 1 month ago ↶
262
263
264
for t in self .tags ( ) ? {
265
let mark = match marks.get ( & t.state) {
266
Some ( m) => * m,
267
None => {
268
let loaded = self .load_state ( & t.heads) ? ;
269
let snapshot = loaded.materialize ( ) ;
270
let mark = next_mark;
271
next_mark + = 1 ;
272
let message = format! ( " oxy: tag {} \n \n {} Oxy-State: {} \n " , t.name, if t.message.is_empty ( ) { String :: new( ) } else { format! ( " {} \n \n " , t.message) } , t.state.to_hex ( ) ) ;
273
self .write_commit ( & mut stream, " refs/oxy/export" , mark, " oxy <oxy@localhost>" , Hlc:: now( ) .wall_ms / 1000 , & message, & [ ] , & snapshot) ? ;
274
marks.insert ( t.state, mark) ;
275
report.exported + = 1 ;
276
mark
277
}
278
} ;
279
stream.extend_from_slice ( format! ( " reset refs/tags/{} \n from :{mark} \n \n " , t.name) .as_bytes ( ) ) ;
280
}
281
stream.extend_from_slice ( b " done\n " ) ;
79ab8528 Andy McCall 1 month ago ↶
282
283
let mut cmd = Command:: new( " git" ) ;
284
285
cmd.arg ( " -C" ) .arg ( git_dir) .args ( [ " fast-import" , " --quiet" , " --done" , " --force" ] ) .arg ( format! ( " --export-marks={} " , git_marks.display ( ) ) ) ;
286
if git_marks.exists ( ) {
287
cmd.arg ( format! ( " --import-marks={} " , git_marks.display ( ) ) ) ;
288
}
289
let mut child = cmd.stdin ( Stdio:: piped( ) ) .stdout ( Stdio:: null( ) ) .stderr ( Stdio:: piped( ) ) .spawn ( ) .map_err ( | e | RepoError:: Msg( format! ( " running git fast-import: {e} " ) ) ) ? ;
290
child.stdin.take ( ) .unwrap ( ) .write_all ( & stream) ? ;
291
let out = child.wait_with_output ( ) ? ;
292
if ! out.status.success ( ) {
293
return fail ( format! ( " git fast-import failed: {} " , String :: from_utf8_lossy( & out.stderr) .trim ( ) ) ) ;
294
}
295
let text: String = marks.iter ( ) .map ( | ( h , m ) | format! ( " {} {m} \n " , h.to_hex ( ) ) ) .collect ( ) ;
296
write_atomic ( & oxy_marks_path, text.as_bytes ( ) ) ? ;
297
let _ = git ( git_dir, & [ " update-ref" , " -d" , " refs/oxy/export" ] ) ;
298
Ok ( report)
299
}
300
c3069537 Andy McCall 1 month ago ↶
301
302
303
304
305
pub fn import_git ( & self , git_dir : & Path, branch : & str ) -> Result< GitImportReport> {
306
if ! self .bookmark_heads ( ) ? .is_empty ( ) {
307
return fail ( " git import needs an empty bookmark; import into a fresh repository" ) ;
308
}
309
let commits = list_commits ( git_dir, branch) ? ;
310
let mut cat = CatFile:: start( git_dir) ? ;
311
let mut report = GitImportReport:: default( ) ;
312
let mut change_of: HashMap< String, ChangeHash> = HashMap:: new( ) ;
313
314
315
let mut last: Option< ( String, State) > = None ;
316
let mut blob_cache: HashMap< String, Vec< u8 > > = HashMap:: new( ) ;
317
let mut cache_bytes: usize = 0 ;
318
319
for gc in & commits {
320
321
let mut base = match ( & gc.parents.first ( ) , & last) {
322
( Some ( p1) , Some ( ( sha, st) ) ) if * p1 = = sha => st.clone ( ) ,
323
( Some ( p1) , _ ) => self .git_ancestor_state ( git_dir, & [ p1.as_str ( ) ] , & change_of) ? ,
324
( None , _ ) => State:: new( ) ,
325
} ;
326
if gc.parents.len ( ) > 1 {
327
report.merges + = 1 ;
328
let mut args: Vec< String> = gc.parents[ 1 .. ] .to_vec ( ) ;
329
args.push ( format! ( " ^{} " , gc.parents[ 0 ] ) ) ;
330
let extra = git ( git_dir, & [ " rev-list" .to_string ( ) ] .iter ( ) .map ( String :: as_str) .chain ( args.iter ( ) .map ( String :: as_str) ) .collect:: < Vec< _ > > ( ) ) ? ;
331
for sha in extra.lines ( ) {
332
if let Some ( h) = change_of.get ( sha) {
333
base.add_unchecked ( self .store.get_change ( h) ? ) ;
334
}
335
}
336
}
337
let base_snapshot = base.materialize ( ) ;
338
339
340
let mut entries: BTreeMap< String, FsEntry> = BTreeMap:: new( ) ;
341
for ( path, ( mode, sha) ) in list_tree ( git_dir, & gc.sha) ? {
342
let ( kind, exec) = match mode.as_str ( ) {
343
" 100644" | " 100664" => ( NodeKind:: File, false ) ,
344
" 100755" => ( NodeKind:: File, true ) ,
345
" 120000" => ( NodeKind:: Symlink, false ) ,
346
" 160000" => {
347
report.skipped_submodules + = 1 ;
348
continue ;
349
}
350
_ => continue ,
351
} ;
352
let content = match blob_cache.get ( & sha) {
353
Some ( c) => c.clone ( ) ,
354
None => {
355
let c = cat.read ( & sha) ? ;
356
if cache_bytes + c.len ( ) > 256 * 1024 * 1024 {
357
blob_cache.clear ( ) ;
358
cache_bytes = 0 ;
359
}
360
cache_bytes + = c.len ( ) ;
361
blob_cache.insert ( sha.clone ( ) , c.clone ( ) ) ;
362
c
363
}
364
} ;
365
entries.insert ( path, FsEntry { kind, exec, content } ) ;
366
}
367
368
let built = self .build_from_entries ( & base, & base_snapshot, HashMap:: new( ) , & entries) ? ;
369
let mut deps: BTreeSet< ChangeHash> = built.contributors.iter ( ) .copied ( ) .collect ( ) ;
370
let mut message = gc.message.clone ( ) ;
371
if ! message.ends_with ( ' \n ' ) {
372
message.push ( ' \n ' ) ;
373
}
374
message.push_str ( & format! ( " \n Git-Commit: {} \n Git-Author: {} \n " , gc.sha, gc.author) ) ;
375
let mut change = Change {
376
change_id: change_id_for ( & gc.sha, & gc.message) ,
377
deps: vec! [ ] ,
378
author: ObjectRef:: ZERO ,
379
hlc: Hlc { wall_ms: gc.author_secs * 1000 , logical: 0 } ,
380
message,
381
tree_ops: built.tree_ops,
382
edits: built.edits,
383
kind: if built.resolves { ChangeKind:: Resolution } else { ChangeKind:: Normal } ,
384
} ;
385
deps.extend ( referenced_changes ( & change, & base) ) ;
386
change.deps = deps.into_iter ( ) .collect ( ) ;
387
change.normalize ( ) ;
388
let hash = self .store.put_change ( & change) ? ;
389
base.add_unchecked ( change) ;
390
change_of.insert ( gc.sha.clone ( ) , hash) ;
391
last = Some ( ( gc.sha.clone ( ) , base) ) ;
392
report.commits + = 1 ;
393
}
394
395
396
let Some ( ( _ , tip_state) ) = last else { return Ok ( report) } ;
397
let heads = tip_state.heads ( ) ;
398
report.heads = heads.len ( ) ;
399
let ( op_ref, op) = self .head_op ( ) ? ;
400
let mut view = op.view.clone ( ) ;
401
view.bookmarks.insert ( self .config.bookmark.clone ( ) , self .store.put_state ( & StateObject:: new( heads.clone ( ) ) ) ? ) ;
402
self .record_op ( & [ op_ref] , & format! ( " oxy git import {} {branch} " , git_dir.display ( ) ) , view) ? ;
403
self .checkout ( & heads, true , " oxy git import (checkout)" ) ? ;
404
Ok ( report)
405
}
406
407
408
fn git_ancestor_state ( & self , git_dir : & Path, shas : & [& str ], change_of : & HashMap< String, ChangeHash> ) -> Result< State> {
409
let mut args = vec! [ " rev-list" ] ;
410
args.extend ( shas) ;
411
let out = git ( git_dir, & args) ? ;
412
let mut st = State:: new( ) ;
413
for sha in out.lines ( ) {
414
if let Some ( h) = change_of.get ( sha) {
415
st.add_unchecked ( self .store.get_change ( h) ? ) ;
416
}
417
}
418
Ok ( st)
419
}
420
79ab8528 Andy McCall 1 month ago ↶
421
fn git_author ( & self , c : & Change) -> String {
422
match self .store.get_identity ( & c.author) {
423
Ok ( id) if ! id.name.is_empty ( ) => format! ( " {} <{} >" , id.name.replace ( [ ' <' , ' >' , ' \n ' ] , " " ) , id.email.replace ( [ ' <' , ' >' , ' \n ' , ' ' ] , " " ) ) ,
424
_ => " oxy <oxy@localhost>" .to_string ( ) ,
425
}
426
}
427
428
# [ allow ( clippy::too_many_arguments ) ]
429
fn write_commit ( & self , out : & mut Vec< u8 > , reference : & str , mark : u64 , author : & str , secs : u64 , message : & str , parents : & [u64 ], snapshot : & Snapshot) -> Result< ( ) > {
430
out.extend_from_slice ( format! ( " commit {reference} \n mark :{mark} \n author {author} {secs} +0000\n committer {author} {secs} +0000\n data {} \n {message} \n " , message.len ( ) ) .as_bytes ( ) ) ;
431
for ( i, p) in parents.iter ( ) .enumerate ( ) {
432
out.extend_from_slice ( format! ( " {} :{p} \n " , if i = = 0 { " from" } else { " merge" } ) .as_bytes ( ) ) ;
433
}
434
out.extend_from_slice ( b " deleteall\n " ) ;
435
for ( path, f) in & snapshot.files {
436
let ( mode, bytes) : ( & str , Vec< u8 > ) = match ( & f.kind, & f.content) {
437
( NodeKind:: Dir, _ ) => continue ,
438
( NodeKind:: Symlink, FileContent:: Opaque { blobs } ) => ( " 120000" , blobs.first ( ) .map ( | b | self .read_blob ( b) ) .transpose ( ) ? .unwrap_or_default ( ) ) ,
439
( _ , FileContent:: Text { bytes, .. } ) => ( if f.mode.exec { " 100755" } else { " 100644" } , bytes.clone ( ) ) ,
440
( _ , FileContent:: Opaque { blobs } ) => ( if f.mode.exec { " 100755" } else { " 100644" } , blobs.first ( ) .map ( | b | self .read_blob ( b) ) .transpose ( ) ? .unwrap_or_default ( ) ) ,
441
( _ , FileContent:: Dir) => continue ,
442
} ;
443
out.extend_from_slice ( format! ( " M {mode} inline {path} \n data {} \n " , bytes.len ( ) ) .as_bytes ( ) ) ;
444
out.extend_from_slice ( & bytes) ;
445
out.push ( b ' \n ' ) ;
446
}
447
out.push ( b ' \n ' ) ;
448
Ok ( ( ) )
449
}
450
}