git import: checkpoint progress so a stopped import resumes 9935d490ddfb
An import moved its bookmark once, at the end, and imported commits are found from the bookmark's history, so stopping a long import lost all of it. It now moves the bookmark to the heads so far every 2000 commits (OXY_IMPORT_CHECKPOINT overrides, for testing), and oxy git fetch picks up from there. A clean working copy with an empty base, as such an import leaves, now checks out the result. oxy git fetch with a relative path ran it from inside the mirror and found no branch; local paths are made absolute first. Checked: a 3000-commit repository, the import killed after 2000, then git fetch ../repo: 3000 commits verified and the files checked out. closes #77
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-10-06 09:45 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
9935d490ddfbccef8cf6a702ef91e10ae9cd847f3bfcc1aa64c8a5e0603d89f3- Depends on
- 30ea9c1e70fa 58dfe0e134eb 79ab8528ff62 c3069537d7c5 d45dab0484ea fb5b6181618a fd280228bdc2
- Depended on by
- 544d0541a7e5 810691ca6ce6 54e2e6309aa3
- In bookmarks
- bare-hashes carried-not-removed carry-identical doctor-trust identical-sides keyless-tags main merge-remote-bookmark only-help-103 paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state switch-says
- In tags
- v0.10.0 v0.11.0 v0.4.1 v0.4.2 v0.4.3 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
- Edits
- 7 atoms, 0 tree operations, 1 file
Browse the tree at this change
modified crates/oxy-core/src/repo/git.rs +40 −1
| … | 293 unchanged lines |
||
| 294 | 294 | const KEPT_ATOMS: usize = 1_500_000; |
|
| 295 | 295 | /// A kept state with more built graphs than this sheds them. |
|
| 296 | 296 | const KEPT_GRAPHS: usize = 2000; |
|
| 297 | + | /// An import moves its bookmark to the commits done so far every this |
|
| 298 | + | /// many commits, so one that stops part way resumes from there: imported |
|
| 299 | + | /// commits are found from the bookmark's history (`imported_commits`). |
|
| 300 | + | const CHECKPOINT_COMMITS: usize = 2000; |
|
| 297 | 301 | ||
| 298 | 302 | fn git(dir: &Path, args: &[&str]) -> Result<String> { |
|
| 299 | 303 | let out = Command::new("git") |
|
| … | 103 unchanged lines |
||
| 403 | 407 | /// (spec §23). Works on a repository that already has history: the |
|
| 404 | 408 | /// new changes join the bookmark like a pull. |
|
| 405 | 409 | pub fn git_fetch(&self, url: &str, branch: &str) -> Result<GitImportReport> { |
|
| 410 | + | // A local path, made absolute: git runs it from inside the mirror. |
|
| 411 | + | let local = Path::new(url) |
|
| 412 | + | .canonicalize() |
|
| 413 | + | .ok() |
|
| 414 | + | .filter(|_| !url.contains("://")); |
|
| 415 | + | let absolute = local.as_ref().map(|p| p.to_string_lossy().into_owned()); |
|
| 416 | + | let url = absolute.as_deref().unwrap_or(url); |
|
| 406 | 417 | let mirror = self.git_mirror(url)?; |
|
| 407 | 418 | self.git_mirror_fetch(&mirror, url, branch)?; |
|
| 408 | 419 | if git( |
|
| … | 388 unchanged lines |
||
| 797 | 808 | *clock = now; |
|
| 798 | 809 | }; |
|
| 799 | 810 | ||
| 811 | + | // OXY_IMPORT_CHECKPOINT overrides it, for testing. |
|
| 812 | + | let checkpoint_every: usize = std::env::var("OXY_IMPORT_CHECKPOINT") |
|
| 813 | + | .ok() |
|
| 814 | + | .and_then(|v| v.parse().ok()) |
|
| 815 | + | .filter(|n| *n > 0) |
|
| 816 | + | .unwrap_or(CHECKPOINT_COMMITS); |
|
| 800 | 817 | let total = commits.len(); |
|
| 801 | 818 | for (index, gc) in commits.iter().enumerate() { |
|
| 802 | 819 | if known.imported.contains_key(&gc.sha) || known.exported.contains_key(&gc.sha) { |
|
| … | 405 unchanged lines |
||
| 1208 | 1225 | report.commits += 1; |
|
| 1209 | 1226 | lap(5, &mut clock); |
|
| 1210 | 1227 | done += 1; |
|
| 1228 | + | if done.is_multiple_of(checkpoint_every) { |
|
| 1229 | + | self.checkpoint_import(&heads_now, done)?; |
|
| 1230 | + | } |
|
| 1211 | 1231 | // A merge renders thousands of files on many threads and frees |
|
| 1212 | 1232 | // it all; the allocator keeps those pages unless asked to give |
|
| 1213 | 1233 | // them back, and the next merge piles on top (#25). |
|
| … | 74 unchanged lines |
||
| 1288 | 1308 | base.sort(); |
|
| 1289 | 1309 | let mut sorted_old = old_heads.clone(); |
|
| 1290 | 1310 | sorted_old.sort(); |
|
| 1291 | - | if clean && base == sorted_old { |
|
| 1311 | + | // An empty base too: an import that stopped after a checkpoint |
|
| 1312 | + | // never checked anything out, and resuming it should. |
|
| 1313 | + | if clean && (base == sorted_old || base.is_empty()) { |
|
| 1292 | 1314 | self.checkout(&heads, false, &format!("{command} (checkout)"))?; |
|
| 1293 | 1315 | } |
|
| 1294 | 1316 | } |
|
| … | 64 unchanged lines |
||
| 1359 | 1381 | } |
|
| 1360 | 1382 | } |
|
| 1361 | 1383 | } |
|
| 1384 | + | Ok(()) |
|
| 1385 | + | } |
|
| 1386 | + | ||
| 1387 | + | /// Point the bookmark at an import's heads so far (see |
|
| 1388 | + | /// `CHECKPOINT_COMMITS`); the working copy is left alone until the end. |
|
| 1389 | + | fn checkpoint_import(&self, heads: &BTreeSet<ChangeHash>, done: usize) -> Result<()> { |
|
| 1390 | + | let heads: Vec<ChangeHash> = heads.iter().copied().collect(); |
|
| 1391 | + | let (op_ref, op) = self.head_op()?; |
|
| 1392 | + | let mut view = op.view.clone(); |
|
| 1393 | + | let state_ref = self.store.put_state(&StateObject::new(heads))?; |
|
| 1394 | + | view.bookmarks |
|
| 1395 | + | .insert(self.config.bookmark.clone(), state_ref); |
|
| 1396 | + | self.record_op( |
|
| 1397 | + | &[op_ref], |
|
| 1398 | + | &format!("oxy git import (checkpoint after {done} commits)"), |
|
| 1399 | + | view, |
|
| 1400 | + | )?; |
|
| 1362 | 1401 | Ok(()) |
|
| 1363 | 1402 | } |
|
| 1364 | 1403 | ||
| … | 405 unchanged lines |