OXYGENHUB
# 091fef80755d oxygen / crates / oxy-core / src / repo / checkout.rs
22.4 KiB · Rust · last changed Checkout: files added once are written from their text, new files written in parallel (refs #41) 1 month ago source blame history raw
1
//! Moving the working copy to another state (spec §12.3, §7.7).
2
3
use super::*;
4
5
#[derive(Debug, Default, Clone, PartialEq, Eq)]
6
pub struct CheckoutReport {
7
    pub written: usize,
8
    pub removed: usize,
9
    pub conflicted: bool,
10
    /// Nested repositories that could not be moved, and other remarks.
11
    pub notes: Vec<String>,
12
}
13
14
/// Text files rendered at a time by a checkout: bounds its memory.
15
const CHECKOUT_BATCH: usize = 2048;
16
17
impl Repo {
18
    /// Move the working copy to the state whose heads are `heads` (spec §12.3):
19
    /// the working directory is rewritten to that state and a fresh
20
    /// working-copy change is started on it. The bookmark does not move.
21
    pub fn checkout(
22
        &self,
23
        heads: &[ChangeHash],
24
        discard: bool,
25
        command: &str,
26
    ) -> Result<CheckoutReport> {
27
        self.checkout_keeping(heads, discard, command, &BTreeSet::new())
28
    }
29
30
    /// [`Repo::checkout`], leaving the nested repositories at
31
    /// `keep_subrepos` as they are (their pins are carried work).
32
    pub(crate) fn checkout_keeping(
33
        &self,
34
        heads: &[ChangeHash],
35
        discard: bool,
36
        command: &str,
37
        keep_subrepos: &BTreeSet<String>,
38
    ) -> Result<CheckoutReport> {
39
        let _wc = self.lock_working_copy()?;
40
        // A status snapshot: files the manifest proves unchanged are neither
41
        // read nor rendered, and its base is history only (oxygen#35).
42
        let snap = self.snapshot_status(command)?;
43
        if !discard && (!snap.wc.tree_ops.is_empty() || !snap.wc.edits.is_empty()) {
44
            let list: Vec<String> = snap
45
                .statuses
46
                .iter()
47
                .take(8)
48
                .map(|s| format!("{} {}", status_word(s.status), s.path))
49
                .collect();
50
            return fail(format!(
51
                "the working copy has changes ({}{}); commit them, take them along with --carry, or drop them with --discard",
52
                list.join(", "),
53
                if snap.statuses.len() > 8 { ", …" } else { "" }
54
            ));
55
        }
56
        let old_manifest = self.read_manifest()?.map(|(_, _, m)| m).unwrap_or_default();
57
        let base = &snap.base;
58
        let base_files = &snap.base_snapshot.files;
59
        let changed: HashSet<&str> = snap.statuses.iter().map(|s| s.path.as_str()).collect();
60
        // The target's history; its file graphs are built a batch at a time
61
        // for the files that must be written, and dropped after.
62
        let mut target = self.load_state_history(heads)?;
63
        let heads = target.maximal(heads);
64
        let tview = target.tree().view();
65
        let mut report = CheckoutReport::default();
66
        // Paths this file system cannot hold side by side (spec §7.7).
67
        report.notes.extend(
68
            self.platform
69
                .problems(tview.entries.keys().map(String::as_str)),
70
        );
71
        report.conflicted = tview.is_conflicted();
72
73
        // Remove what the base or the working copy has and the target does
74
        // not. A nested repository is never deleted (spec §7.8).
75
        let mut dirs_to_prune: BTreeSet<PathBuf> = BTreeSet::new();
76
        let gone: BTreeSet<&str> = base_files
77
            .iter()
78
            .filter(|(_, f)| f.kind != NodeKind::Dir)
79
            .map(|(p, _)| p.as_str())
80
            .chain(changed.iter().copied())
81
            .filter(|p| !tview.entries.contains_key(*p))
82
            .collect();
83
        for path in gone {
84
            if base_files
85
                .get(path)
86
                .is_some_and(|f| f.kind == NodeKind::Subrepo)
87
            {
88
                report
89
                    .notes
90
                    .push(format!("{path}: nested repository left in place"));
91
                continue;
92
            }
93
            let full = self.safe_path(path)?;
94
            if fs::symlink_metadata(&full).is_err() {
95
                continue;
96
            }
97
            fs::remove_file(&full)?;
98
            report.removed += 1;
99
            if let Some(parent) = full.parent() {
100
                dirs_to_prune.insert(parent.to_path_buf());
101
            }
102
        }
103
104
        // The manifest after the checkout, built from what is known.
105
        let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();
106
        let hex = |b: &[u8]| blake3::hash(b).to_hex().to_string();
107
        // Text files to render, attribute files first (the others need them).
108
        let mut texts: Vec<(String, NodeId, Mode)> = Vec::new();
109
        for (path, e) in &tview.entries {
110
            if e.kind == NodeKind::Dir || !self.in_scope(path) {
111
                continue;
112
            }
113
            let untouched = !changed.contains(path.as_str())
114
                && base_files
115
                    .get(path)
116
                    .is_some_and(|b| b.node == e.node && b.kind == e.kind && b.mode == e.mode);
117
            if e.kind == NodeKind::File && !e.mode.opaque {
118
                // Unchanged on disk (unrendered by the snapshot) and the same
119
                // graph in both states: nothing to read, render or write.
120
                let same_graph = untouched
121
                    && snap.base_snapshot.unrendered.contains(path)
122
                    && !is_attributes(path)
123
                    && base.graph_key(&e.node).is_some()
124
                    && base.graph_key(&e.node) == target.graph_key(&e.node);
125
                match old_manifest.entries.get(path).filter(|_| same_graph) {
126
                    Some(m) => {
127
                        entries.insert(
128
                            path.clone(),
129
                            FsEntry {
130
                                content_hash: Some(m.hash.clone()),
131
                                graph_key: m.graph.clone(),
132
                                unchanged: true,
133
                                ..FsEntry::new(NodeKind::File, e.mode.exec, Vec::new())
134
                            },
135
                        );
136
                    }
137
                    None => texts.push((path.clone(), e.node, e.mode)),
138
                }
139
                continue;
140
            }
141
            // Opaque files, symlinks and subrepos: their blobs.
142
            let blobs = target.live_blobs(&e.node);
143
            if blobs.len() > 1 {
144
                report.conflicted = true;
145
            }
146
            let Some(blob) = blobs.first().copied() else {
147
                continue;
148
            };
149
            if e.kind == NodeKind::Subrepo {
150
                if keep_subrepos.contains(path) {
151
                    continue;
152
                }
153
                let pin = crate::subrepo::Pin::decode(&self.read_blob(&blob)?)?;
154
                let sync = self.sync_subrepo(path, &pin, false)?;
155
                if matches!(
156
                    sync,
157
                    SubrepoSync::Updated | SubrepoSync::KeptBookmark { .. }
158
                ) {
159
                    report.written += 1;
160
                }
161
                report.notes.extend(sync.note(path));
162
                continue;
163
            }
164
            let same_blob = untouched && base.live_blobs(&e.node) == blobs;
165
            if let Some(m) = old_manifest
166
                .entries
167
                .get(path)
168
                .filter(|_| same_blob && blobs.len() == 1)
169
            {
170
                entries.insert(
171
                    path.clone(),
172
                    FsEntry {
173
                        content_hash: Some(m.hash.clone()),
174
                        blob: (e.kind == NodeKind::File).then_some(blob),
175
                        ..FsEntry::new(e.kind, e.mode.exec, Vec::new())
176
                    },
177
                );
178
                continue;
179
            }
180
            let bytes = self.read_blob(&blob)?;
181
            let full = self.root.join(path);
182
            if e.kind == NodeKind::Symlink {
183
                let same = fs::read_link(&full)
184
                    .is_ok_and(|t| t.to_string_lossy().as_bytes() == bytes.as_slice());
185
                if !same {
186
                    self.write_symlink(path, &bytes)?;
187
                    report.written += 1;
188
                }
189
                entries.insert(
190
                    path.clone(),
191
                    FsEntry {
192
                        content_hash: Some(hex(&bytes)),
193
                        ..FsEntry::new(NodeKind::Symlink, false, Vec::new())
194
                    },
195
                );
196
            } else {
197
                if !self.same_on_disk(&full, &bytes, e.mode.exec) {
198
                    self.write_file(path, &bytes, e.mode.exec)?;
199
                    report.written += 1;
200
                }
201
                entries.insert(
202
                    path.clone(),
203
                    FsEntry {
204
                        content_hash: Some(hex(&bytes)),
205
                        blob: (blobs.len() == 1).then_some(blob),
206
                        ..FsEntry::new(NodeKind::File, e.mode.exec, Vec::new())
207
                    },
208
                );
209
            }
210
            for (i, b) in blobs.iter().enumerate().skip(1) {
211
                self.write_file(
212
                    &format!("{path}.oxy-conflict-{}", i + 1),
213
                    &self.read_blob(b)?,
214
                    e.mode.exec,
215
                )?;
216
            }
217
        }
218
219
        // Render and write text files a batch at a time; attribute files
220
        // come first, since the others' line endings and locks follow them.
221
        texts.sort_by_key(|(p, _, _)| !is_attributes(p));
222
        let keys: HashMap<NodeId, [u8; 32]> = texts
223
            .iter()
224
            .filter_map(|(_, n, _)| Some((*n, target.graph_key(n)?)))
225
            .collect();
226
        let mut attrs = crate::attributes::Attributes::default();
227
        let attr_count = texts.iter().filter(|(p, _, _)| is_attributes(p)).count();
228
        let mut done = 0usize;
229
        let mut batches: Vec<&[(String, NodeId, Mode)]> = Vec::new();
230
        if attr_count > 0 {
231
            batches.push(&texts[..attr_count]);
232
        }
233
        batches.extend(texts[attr_count..].chunks(CHECKOUT_BATCH));
234
        for batch in batches {
235
            use rayon::prelude::*;
236
            let nodes: Vec<NodeId> = batch.iter().map(|(_, n, _)| *n).collect();
237
            // A file added once and never edited is its text: no graph.
238
            let single: Vec<Option<crate::objects::TextRef>> =
239
                nodes.iter().map(|n| target.single_text(n)).collect();
240
            let graphs: Vec<NodeId> = nodes
241
                .iter()
242
                .zip(&single)
243
                .filter(|(_, s)| s.is_none())
244
                .map(|(n, _)| *n)
245
                .collect();
246
            let mut texts = {
247
                let _t = crate::timing::phase("checkout: read single texts");
248
                let wanted: Vec<crate::objects::TextRef> =
249
                    single.iter().flatten().copied().collect();
250
                self.single_texts(&wanted)?.into_iter()
251
            };
252
            {
253
                let _t = crate::timing::phase("checkout: build graphs");
254
                self.load_graphs_of(&mut target, &graphs)?;
255
            }
256
            let rendered: Vec<(Vec<u8>, bool)> = {
257
                let _t = crate::timing::phase("checkout: render");
258
                let mut out: Vec<Option<(Vec<u8>, bool)>> = single
259
                    .iter()
260
                    .map(|s| s.as_ref().and_then(|_| texts.next().map(|t| (t, false))))
261
                    .collect();
262
                let from_graphs: Vec<(Vec<u8>, bool)> = graphs
263
                    .par_iter()
264
                    .map(|n| {
265
                        let (bytes, m) = target.render_file(n);
266
                        (bytes, m.is_conflicted())
267
                    })
268
                    .collect();
269
                let mut from_graphs = from_graphs.into_iter();
270
                for o in out.iter_mut().filter(|o| o.is_none()) {
271
                    *o = from_graphs.next();
272
                }
273
                out.into_iter()
274
                    .map(|o| o.expect("every file rendered"))
275
                    .collect()
276
            };
277
            #[cfg(test)]
278
            {
279
                let singles: Vec<NodeId> = nodes
280
                    .iter()
281
                    .zip(&single)
282
                    .filter(|(_, s)| s.is_some())
283
                    .map(|(n, _)| *n)
284
                    .collect();
285
                self.load_graphs_of(&mut target, &singles)?;
286
                for (n, (bytes, _)) in nodes.iter().zip(&rendered) {
287
                    assert_eq!(
288
                        *bytes,
289
                        target.render_file(n).0,
290
                        "a single text is its file's rendering"
291
                    );
292
                }
293
            }
294
            target.drop_graphs(&nodes);
295
            let _t = crate::timing::phase("checkout: write");
296
            if done == 0 && attr_count > 0 {
297
                attrs = crate::attributes::Attributes::from_files(
298
                    batch
299
                        .iter()
300
                        .zip(&rendered)
301
                        .map(|((p, _, _), (b, _))| (p.as_str(), b.as_slice())),
302
                );
303
            }
304
            done += batch.len();
305
            let on_disk: Vec<Vec<u8>> = batch
306
                .iter()
307
                .zip(rendered.iter())
308
                .map(|((path, _, _), (bytes, _))| {
309
                    if attrs.lookup(path).eol == Some(crate::attributes::Eol::Crlf) {
310
                        crate::attributes::to_crlf(bytes)
311
                    } else {
312
                        bytes.clone()
313
                    }
314
                })
315
                .collect();
316
            // Each directory checked (no symlink on the way) and made once,
317
            // then the files compared and written on every core.
318
            let mut dirs: BTreeSet<&str> = BTreeSet::new();
319
            for (path, _, _) in batch {
320
                if let Some((dir, _)) = path.rsplit_once('/') {
321
                    dirs.insert(dir);
322
                }
323
            }
324
            for dir in dirs {
325
                let full = self.safe_path(dir)?;
326
                if !full.is_dir() {
327
                    if fs::symlink_metadata(&full).is_ok() {
328
                        remove_any(&full)?;
329
                    }
330
                    fs::create_dir_all(&full)?;
331
                }
332
            }
333
            for (path, _, _) in batch {
334
                check_path_name(path)?;
335
            }
336
            let (root, exec_bits) = (&self.root, self.platform.exec_bits);
337
            // Compared on every core; new files written on every core too,
338
            // but files that exist rewritten one at a time: on ext4,
339
            // rewriting in parallel was six times slower (each truncation
340
            // starts writing back the old blocks).
341
            let plans: Vec<Disk> = batch
342
                .par_iter()
343
                .zip(on_disk.par_iter())
344
                .map(|((path, _, mode), bytes)| {
345
                    compare_on_disk(&root.join(path), bytes, mode.exec, exec_bits)
346
                })
347
                .collect();
348
            let fresh: Vec<usize> = (0..batch.len())
349
                .filter(|&i| plans[i] == Disk::Absent)
350
                .collect();
351
            fresh
352
                .par_iter()
353
                .map(|&i| write_file_at(&root.join(&batch[i].0), &on_disk[i], batch[i].2.exec))
354
                .collect::<std::io::Result<()>>()?;
355
            for (i, plan) in plans.iter().enumerate() {
356
                if *plan == Disk::Differs {
357
                    write_file_at(&root.join(&batch[i].0), &on_disk[i], batch[i].2.exec)?;
358
                }
359
            }
360
            report.written += plans.iter().filter(|p| **p != Disk::Same).count();
361
            for (((path, node, mode), (_, conflicted)), on_disk) in
362
                batch.iter().zip(rendered).zip(on_disk)
363
            {
364
                let a = attrs.lookup(path);
365
                report.conflicted |= conflicted;
366
                entries.insert(
367
                    path.clone(),
368
                    FsEntry {
369
                        content_hash: Some(hex(&on_disk)),
370
                        graph_key: (!conflicted && a.eol.is_none())
371
                            .then(|| keys.get(node).map(|k| ObjectRef(*k).to_hex()))
372
                            .flatten(),
373
                        ..FsEntry::new(NodeKind::File, mode.exec, Vec::new())
374
                    },
375
                );
376
            }
377
        }
378
        // Lockable files are read-only until locked (spec §18), whether
379
        // written now or left as they were.
380
        for (path, e) in &entries {
381
            if e.kind == NodeKind::File && attrs.lookup(path).lockable && !self.is_locked(path) {
382
                let _ = set_readonly(&self.root.join(path), true);
383
            }
384
        }
385
        // Prune directories left empty, deepest first.
386
        for d in dirs_to_prune.iter().rev() {
387
            let mut d = d.clone();
388
            while d != self.root
389
                && fs::read_dir(&d)
390
                    .map(|mut i| i.next().is_none())
391
                    .unwrap_or(false)
392
            {
393
                fs::remove_dir(&d)?;
394
                let Some(p) = d.parent() else { break };
395
                d = p.to_path_buf();
396
            }
397
        }
398
399
        let (op_ref, op) = self.head_op()?;
400
        let wc = self.new_working_copy(heads.clone(), "");
401
        let wc_hash = self.store.put_change(&wc)?;
402
        let mut view = op.view.clone();
403
        view.workspaces.insert(self.workspace, wc_hash);
404
        self.record_op(&[op_ref], command, view)?;
405
        self.write_manifest_entries(report.conflicted, false, &entries, &HashMap::new())?;
406
        let _ = self.run_hook(
407
            "post-checkout",
408
            &format!(
409
                "{{\"event\":\"post-checkout\",\"heads\":{},\"written\":{},\"removed\":{}}}",
410
                json_array(heads.iter().map(|h| json_str(&h.to_hex()))),
411
                report.written,
412
                report.removed
413
            ),
414
        );
415
        Ok(report)
416
    }
417
418
    /// Whether the file at `full` already holds `bytes` with this executable
419
    /// bit, so writing it again can be skipped.
420
    fn same_on_disk(&self, full: &Path, bytes: &[u8], exec: bool) -> bool {
421
        match fs::symlink_metadata(full) {
422
            Ok(m) if m.is_file() && m.len() == bytes.len() as u64 => {
423
                (!self.platform.exec_bits || is_executable(&m) == exec)
424
                    && fs::read(full).is_ok_and(|b| b == bytes)
425
            }
426
            _ => false,
427
        }
428
    }
429
430
    /// The absolute path for a repository path, refusing anything that would
431
    /// escape the workspace or write through a symlink (spec §7.7).
432
    fn safe_path(&self, rel: &str) -> Result<PathBuf> {
433
        let mut full = self.root.clone();
434
        let comps: Vec<&str> = rel.split('/').collect();
435
        for (i, comp) in comps.iter().enumerate() {
436
            if comp.is_empty() || *comp == "." || *comp == ".." || comp.contains('\\') {
437
                return fail(format!("refusing unsafe path {rel:?}"));
438
            }
439
            if i == 0 && *comp == OXY_DIR {
440
                return fail(format!("refusing to write inside {OXY_DIR}: {rel:?}"));
441
            }
442
            if i + 1 < comps.len() {
443
                full.push(comp);
444
                if let Ok(m) = fs::symlink_metadata(&full) {
445
                    if m.file_type().is_symlink() {
446
                        return fail(format!(
447
                            "refusing to write through symlink {}",
448
                            full.display()
449
                        ));
450
                    }
451
                }
452
            } else {
453
                full.push(comp);
454
            }
455
        }
456
        Ok(full)
457
    }
458
459
    pub(crate) fn write_file(&self, rel: &str, bytes: &[u8], exec: bool) -> Result<()> {
460
        let full = self.safe_path(rel)?;
461
        if let Some(parent) = full.parent() {
462
            fs::create_dir_all(parent)?;
463
        }
464
        if fs::symlink_metadata(&full).is_ok_and(|m| m.file_type().is_symlink() || m.is_dir()) {
465
            remove_any(&full)?;
466
        }
467
        if fs::metadata(&full).is_ok_and(|m| m.permissions().readonly()) {
468
            set_readonly(&full, false)?;
469
        }
470
        fs::write(&full, bytes)?;
471
        set_executable(&full, exec)?;
472
        Ok(())
473
    }
474
475
    pub(crate) fn write_symlink(&self, rel: &str, target: &[u8]) -> Result<()> {
476
        let full = self.safe_path(rel)?;
477
        if let Some(parent) = full.parent() {
478
            fs::create_dir_all(parent)?;
479
        }
480
        if fs::symlink_metadata(&full).is_ok() {
481
            remove_any(&full)?;
482
        }
483
        #[cfg(unix)]
484
        {
485
            std::os::unix::fs::symlink(String::from_utf8_lossy(target).as_ref(), &full)?;
486
            Ok(())
487
        }
488
        #[cfg(not(unix))]
489
        {
490
            // Spec §7.7: without symlink support, write the target as a plain file.
491
            fs::write(&full, target)?;
492
            Ok(())
493
        }
494
    }
495
}
496
497
/// How a refusal names a path's change: "removed" is a file of the working
498
/// copy's base that is missing on disk.
499
fn status_word(s: ChangeStatus) -> &'static str {
500
    match s {
501
        ChangeStatus::Added => "added",
502
        ChangeStatus::Modified => "modified",
503
        ChangeStatus::Removed => "removed",
504
        ChangeStatus::Renamed => "renamed",
505
        ChangeStatus::ModeChanged => "mode changed",
506
        ChangeStatus::ConflictEdited => "conflict edited",
507
        ChangeStatus::Resolved => "resolved",
508
    }
509
}
510
511
/// Refuse a path that could escape the workspace (spec §7.7): an empty,
512
/// `.` or `..` component, a backslash, or a first component `.oxy`.
513
fn check_path_name(rel: &str) -> Result<()> {
514
    for (i, comp) in rel.split('/').enumerate() {
515
        if comp.is_empty() || comp == "." || comp == ".." || comp.contains('\\') {
516
            return fail(format!("refusing unsafe path {rel:?}"));
517
        }
518
        if i == 0 && comp == OXY_DIR {
519
            return fail(format!("refusing to write inside {OXY_DIR}: {rel:?}"));
520
        }
521
    }
522
    Ok(())
523
}
524
525
/// A file on disk against what a checkout would write there.
526
#[derive(Clone, Copy, PartialEq, Eq)]
527
enum Disk {
528
    Same,
529
    Differs,
530
    Absent,
531
}
532
533
/// Whether the file at `full` already holds `bytes` with the right
534
/// executable bit. Touches no repository state, so it runs in parallel.
535
fn compare_on_disk(full: &Path, bytes: &[u8], exec: bool, exec_bits: bool) -> Disk {
536
    match fs::symlink_metadata(full) {
537
        Ok(m)
538
            if m.is_file()
539
                && m.len() == bytes.len() as u64
540
                && (!exec_bits || is_executable(&m) == exec)
541
                && fs::read(full).is_ok_and(|b| b == bytes) =>
542
        {
543
            Disk::Same
544
        }
545
        Ok(_) => Disk::Differs,
546
        Err(_) => Disk::Absent,
547
    }
548
}
549
550
/// Write a file whose directory exists, replacing whatever is there.
551
fn write_file_at(full: &Path, bytes: &[u8], exec: bool) -> std::io::Result<()> {
552
    match fs::symlink_metadata(full) {
553
        Ok(m) if m.is_file() => {
554
            if m.permissions().readonly() {
555
                set_readonly(full, false)?;
556
            }
557
        }
558
        Ok(_) => remove_any(full)?,
559
        Err(_) => {}
560
    }
561
    fs::write(full, bytes)?;
562
    set_executable(full, exec)
563
}