checkout.rs 91.9%

464 of 505 lines run, 87 of 97 functions · run #398 on subrepo-pin-state · run missed
//! Moving the working copy to another state (spec §12.3, §7.7).

use super::*;

#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct CheckoutReport {
    pub written: usize,
    pub removed: usize,
    /// The first files removed, to name them (oxygen#102).
    pub removed_paths: Vec<String>,
    pub conflicted: bool,
    /// Nested repositories that could not be moved, and other remarks.
    pub notes: Vec<String>,
    /// How many changes of the state left behind the new one lacks.
    pub lacks: usize,
    /// Union files whose adjacent insertions were settled by keeping both
    /// sides (oxygen#145).
    pub kept_both: Vec<KeptBoth>,
}

/// Text files rendered at a time by a checkout: bounds its memory.
const CHECKOUT_BATCH: usize = 2048;

impl Repo {
    /// Move the working copy to the state whose heads are `heads` (spec §12.3):
    /// the working directory is rewritten to that state and a fresh
    /// working-copy change is started on it. The bookmark does not move.
    pub fn checkout(
        &self,
        heads: &[ChangeHash],
        discard: bool,
        command: &str,
    ) -> Result<CheckoutReport> {
        self.checkout_keeping(heads, discard, command, &BTreeSet::new())
    }

    /// [`Repo::checkout`], leaving the nested repositories at
    /// `keep_subrepos` as they are (their pins are carried work).
    pub(crate) fn checkout_keeping(
        &self,
        heads: &[ChangeHash],
        discard: bool,
        command: &str,
        keep_subrepos: &BTreeSet<String>,
    ) -> Result<CheckoutReport> {
        let _wc = self.lock_working_copy()?;
        let _all = crate::timing::phase("checkout");
        // A working copy clean by the manifest needs no snapshot; otherwise a
        // status snapshot records what changed, and rewrites the manifest
        // with the graph key of every file it proves unchanged. Either way
        // the manifest then says, for each file on disk, which graph it
        // renders, so the base state itself is not needed (oxygen#35).
        let changed_paths: Vec<String> = if self.clean_by_manifest()? {
            Vec::new()
        } else {
            let snap = self.snapshot_status(command)?;
            self.refuse_dirty(&snap, discard)?;
            snap.statuses.iter().map(|s| s.path.clone()).collect()
        };
        let (_, wc_now) = self.working_copy()?;
        let old_manifest = self.read_manifest()?.map(|(_, _, m)| m).unwrap_or_default();
        let changed: HashSet<&str> = changed_paths.iter().map(String::as_str).collect();
        // The target's history; its file graphs are built a batch at a time
        // for the files that must be written, and dropped after. It becomes
        // the working copy's base, which the state index holds.
        let mut target = self.load_state_indexed(heads)?;
        let heads = target.maximal(heads);
        let tv = crate::timing::phase("checkout: target view");
        let tview = self.tree_view(&mut target)?;
        drop(tv);
        let mut report = CheckoutReport::default();
        // Paths this file system cannot hold side by side (spec §7.7).
        report.notes.extend(
            self.platform
                .problems(tview.entries.keys().map(String::as_str)),
        );
        report.conflicted = tview.is_conflicted();
        report.lacks = self.count_lacking(&wc_now.deps, &target)?;

        // Remove what the base or the working copy has and the target does
        // not. A nested repository is never deleted (spec §7.8).
        let mut dirs_to_prune: BTreeSet<PathBuf> = BTreeSet::new();
        let gone: BTreeSet<&str> = old_manifest
            .entries
            .keys()
            .map(String::as_str)
            .chain(changed.iter().copied())
            .filter(|p| !tview.entries.contains_key(*p))
            .collect();
        for path in gone {
            let dir = self.root.join(path);
            if dir.join(OXY_DIR).exists() || dir.join(".git").exists() {
                report
                    .notes
                    .push(format!("{path}: nested repository left in place"));
                continue;
            }
            let full = self.safe_path(path)?;
            if fs::symlink_metadata(&full).is_err() {
                continue;
            }
            fs::remove_file(&full)?;
            report.removed += 1;
            if report.removed_paths.len() < 5 {
                report.removed_paths.push(path.to_string());
            }
            if let Some(parent) = full.parent() {
                dirs_to_prune.insert(parent.to_path_buf());
            }
        }

        // The manifest after the checkout, built from what is known.
        let plan = crate::timing::phase("checkout: plan");
        let mut entries: BTreeMap<String, FsEntry> = BTreeMap::new();
        let hex = |b: &[u8]| *blake3::hash(b).as_bytes();
        // Text files to render, attribute files first (the others need them).
        let mut texts: Vec<(String, NodeId, Mode)> = Vec::new();
        for (path, e) in tview.entries.iter() {
            if e.kind == NodeKind::Dir || !self.in_scope(path) {
                continue;
            }
            // The manifest entry of a file the working copy has not changed,
            // with the right executable bit.
            let exec_ok =
                |m: &prompt::ManifestEntry| !self.platform.exec_bits || m.exec == e.mode.exec;
            let entry = old_manifest
                .entries
                .get(path)
                .filter(|m| !changed.contains(path.as_str()) && exec_ok(m));
            if e.kind == NodeKind::File && !e.mode.opaque {
                // The manifest proves the file on disk equal to the rendering
                // of a graph whose key is the target's: nothing to read,
                // render or write.
                let key = target.graph_key(&e.node);
                let proven = |m: &&prompt::ManifestEntry| {
                    !is_attributes(path) && key.is_some() && m.graph == key
                };
                match entry.filter(proven) {
                    Some(m) => {
                        entries.insert(
                            path.clone(),
                            FsEntry {
                                content_hash: Some(m.hash),
                                graph_key: m.graph,
                                unchanged: true,
                                stat: Some((m.size, m.mtime_ns)),
                                ..FsEntry::new(NodeKind::File, e.mode.exec, Vec::new())
                            },
                        );
                    }
                    None => texts.push((path.clone(), e.node, e.mode)),
                }
                continue;
            }
            // Opaque files, symlinks and subrepos: their blobs.
            let blobs = target.live_blobs(&e.node);
            if blobs.len() > 1 {
                report.conflicted = true;
            }
            let Some(blob) = blobs.first().copied() else {
                continue;
            };
            if e.kind == NodeKind::Subrepo {
                if keep_subrepos.contains(path) {
                    continue;
                }
                let pin = crate::subrepo::Pin::decode(&self.read_blob(&blob)?)?;
                let sync = self.sync_subrepo(path, &pin, false)?;
                if matches!(
                    sync,
                    SubrepoSync::Updated | SubrepoSync::KeptBookmark { .. }
                ) {
                    report.written += 1;
                }
                report.notes.extend(sync.note(path));
                continue;
            }
            if let Some(m) = entry
                .filter(|m| e.kind == NodeKind::File && blobs.len() == 1 && m.blob == Some(blob))
            {
                entries.insert(
                    path.clone(),
                    FsEntry {
                        content_hash: Some(m.hash),
                        blob: (e.kind == NodeKind::File).then_some(blob),
                        stat: Some((m.size, m.mtime_ns)),
                        ..FsEntry::new(e.kind, e.mode.exec, Vec::new())
                    },
                );
                continue;
            }
            let bytes = self.read_blob(&blob)?;
            let full = self.root.join(path);
            if e.kind == NodeKind::Symlink {
                let same = fs::read_link(&full)
                    .is_ok_and(|t| t.to_string_lossy().as_bytes() == bytes.as_slice());
                if !same {
                    self.write_symlink(path, &bytes)?;
                    report.written += 1;
                }
                entries.insert(
                    path.clone(),
                    FsEntry {
                        content_hash: Some(hex(&bytes)),
                        ..FsEntry::new(NodeKind::Symlink, false, Vec::new())
                    },
                );
            } else {
                if !self.same_on_disk(&full, &bytes, e.mode.exec) {
                    self.write_file(path, &bytes, e.mode.exec)?;
                    report.written += 1;
                }
                entries.insert(
                    path.clone(),
                    FsEntry {
                        content_hash: Some(hex(&bytes)),
                        blob: (blobs.len() == 1).then_some(blob),
                        ..FsEntry::new(NodeKind::File, e.mode.exec, Vec::new())
                    },
                );
            }
            for (i, b) in blobs.iter().enumerate().skip(1) {
                self.write_file(
                    &format!("{path}.oxy-conflict-{}", i + 1),
                    &self.read_blob(b)?,
                    e.mode.exec,
                )?;
            }
        }

        drop(plan);
        let _render = crate::timing::phase("checkout: render and write");
        // Render and write text files a batch at a time; attribute files
        // come first, since the others' line endings and locks follow them.
        texts.sort_by_key(|(p, _, _)| !is_attributes(p));
        let keys: HashMap<NodeId, [u8; 32]> = texts
            .iter()
            .filter_map(|(_, n, _)| Some((*n, target.graph_key(n)?)))
            .collect();
        let mut attrs = crate::attributes::Attributes::default();
        let attr_count = texts.iter().filter(|(p, _, _)| is_attributes(p)).count();
        let mut done = 0usize;
        let mut batches: Vec<&[(String, NodeId, Mode)]> = Vec::new();
        if attr_count > 0 {
            batches.push(&texts[..attr_count]);
        }
        batches.extend(texts[attr_count..].chunks(CHECKOUT_BATCH));
        for batch in batches {
            use rayon::prelude::*;
            let nodes: Vec<NodeId> = batch.iter().map(|(_, n, _)| *n).collect();
            // A file added once and never edited is its text: no graph.
            let single: Vec<Option<crate::objects::TextRef>> =
                nodes.iter().map(|n| target.single_text(n)).collect();
            let graphs: Vec<NodeId> = nodes
                .iter()
                .zip(&single)
                .filter(|(_, s)| s.is_none())
                .map(|(n, _)| *n)
                .collect();
            let mut texts = {
                let _t = crate::timing::phase("checkout: read single texts");
                let wanted: Vec<crate::objects::TextRef> =
                    single.iter().flatten().copied().collect();
                self.single_texts(&wanted)?.into_iter()
            };
            {
                let _t = crate::timing::phase("checkout: build graphs");
                self.load_graphs_of(&mut target, &graphs)?;
            }
            let rendered: Vec<(Vec<u8>, bool)> = {
                let _t = crate::timing::phase("checkout: render");
                let mut out: Vec<Option<(Vec<u8>, bool)>> = single
                    .iter()
                    .map(|s| s.as_ref().and_then(|_| texts.next().map(|t| (t, false))))
                    .collect();
                let from_graphs: Vec<(Vec<u8>, bool)> = graphs
                    .par_iter()
                    .map(|n| {
                        let (bytes, m) = target.render_file(n);
                        (bytes, m.is_conflicted())
                    })
                    .collect();
                let mut from_graphs = from_graphs.into_iter();
                for o in out.iter_mut().filter(|o| o.is_none()) {
                    *o = from_graphs.next();
                }
                out.into_iter()
                    .map(|o| o.expect("every file rendered"))
                    .collect()
            };
            #[cfg(test)]
            {
                let singles: Vec<NodeId> = nodes
                    .iter()
                    .zip(&single)
                    .filter(|(_, s)| s.is_some())
                    .map(|(n, _)| *n)
                    .collect();
                self.load_graphs_of(&mut target, &singles)?;
                for (n, (bytes, _)) in nodes.iter().zip(&rendered) {
                    assert_eq!(
                        *bytes,
                        target.render_file(n).0,
                        "a single text is its file's rendering"
                    );
                }
            }
            target.drop_graphs(&nodes);
            let _t = crate::timing::phase("checkout: write");
            if done == 0 && attr_count > 0 {
                attrs = crate::attributes::Attributes::from_files(
                    batch
                        .iter()
                        .zip(&rendered)
                        .map(|((p, _, _), (b, _))| (p.as_str(), b.as_slice())),
                );
            }
            done += batch.len();
            let on_disk: Vec<Vec<u8>> = batch
                .iter()
                .zip(rendered.iter())
                .map(|((path, _, _), (bytes, _))| {
                    if attrs.lookup(path).eol == Some(crate::attributes::Eol::Crlf) {
                        crate::attributes::to_crlf(bytes)
                    } else {
                        bytes.clone()
                    }
                })
                .collect();
            // Each directory checked (no symlink on the way) and made once,
            // then the files compared and written on every core.
            let mut dirs: BTreeSet<&str> = BTreeSet::new();
            for (path, _, _) in batch {
                if let Some((dir, _)) = path.rsplit_once('/') {
                    dirs.insert(dir);
                }
            }
            for dir in dirs {
                let full = self.safe_path(dir)?;
                if !full.is_dir() {
                    if fs::symlink_metadata(&full).is_ok() {
                        remove_any(&full)?;
                    }
                    fs::create_dir_all(&full)?;
                }
            }
            for (path, _, _) in batch {
                check_path_name(path)?;
            }
            let (root, exec_bits) = (&self.root, self.platform.exec_bits);
            // Compared on every core; new files written on every core too,
            // but files that exist rewritten one at a time: on ext4,
            // rewriting in parallel was six times slower (each truncation
            // starts writing back the old blocks).
            let plans: Vec<Disk> = batch
                .par_iter()
                .zip(on_disk.par_iter())
                .map(|((path, _, mode), bytes)| {
                    compare_on_disk(&root.join(path), bytes, mode.exec, exec_bits)
                })
                .collect();
            let fresh: Vec<usize> = (0..batch.len())
                .filter(|&i| plans[i] == Disk::Absent)
                .collect();
            fresh
                .par_iter()
                .map(|&i| write_file_at(&root.join(&batch[i].0), &on_disk[i], batch[i].2.exec))
                .collect::<std::io::Result<()>>()?;
            for (i, plan) in plans.iter().enumerate() {
                if *plan == Disk::Differs {
                    write_file_at(&root.join(&batch[i].0), &on_disk[i], batch[i].2.exec)?;
                }
            }
            report.written += plans.iter().filter(|p| **p != Disk::Same).count();
            for (((path, node, mode), (_, conflicted)), on_disk) in
                batch.iter().zip(rendered).zip(on_disk)
            {
                let a = attrs.lookup(path);
                report.conflicted |= conflicted;
                entries.insert(
                    path.clone(),
                    FsEntry {
                        content_hash: Some(hex(&on_disk)),
                        graph_key: (!conflicted && a.eol.is_none())
                            .then(|| keys.get(node).copied())
                            .flatten(),
                        ..FsEntry::new(NodeKind::File, mode.exec, Vec::new())
                    },
                );
            }
        }
        // Lockable files are read-only until locked (spec §18), whether
        // written now or left as they were.
        for (path, e) in &entries {
            if e.kind == NodeKind::File && attrs.lookup(path).lockable && !self.is_locked(path) {
                let _ = set_readonly(&self.root.join(path), true);
            }
        }
        // Prune directories left empty, deepest first.
        for d in dirs_to_prune.iter().rev() {
            let mut d = d.clone();
            while d != self.root
                && fs::read_dir(&d)
                    .map(|mut i| i.next().is_none())
                    .unwrap_or(false)
            {
                fs::remove_dir(&d)?;
                let Some(p) = d.parent() else { break };
                d = p.to_path_buf();
            }
        }

        drop(_render);
        // How the files rendered: which way each graph was ordered, summed
        // over every core (`OXY_TIMINGS`).
        crate::timing::report_totals();
        let recording = crate::timing::phase("checkout: record");
        let (op_ref, op) = self.head_op()?;
        let wc = self.new_working_copy(heads.clone(), "");
        let wc_hash = self.store.put_change(&wc)?;
        let mut view = op.view.clone();
        view.workspaces.insert(self.workspace, wc_hash);
        self.record_op(&[op_ref], command, view)?;
        drop(recording);
        let _t = crate::timing::phase("checkout: manifest");
        // The directories the new tree has, with any the old manifest kept
        // that still exist (holding only ignored files, say), statted now,
        // after the writes: the next check then need not list the tree. A
        // directory written just now is within the racy window, so for a
        // moment the check lists anyway.
        if !old_manifest.dirs.is_empty() {
            let mut dirs: BTreeSet<String> =
                old_manifest.dirs.iter().map(|(d, _)| d.clone()).collect();
            dirs.insert(String::new());
            for p in entries.keys() {
                let mut at = p.as_str();
                while let Some((parent, _)) = at.rsplit_once('/') {
                    if !dirs.insert(parent.to_string()) {
                        break;
                    }
                    at = parent;
                }
            }
            use rayon::prelude::*;
            let root = &self.root;
            let dirs: Vec<(String, u128)> = dirs
                .into_par_iter()
                .filter_map(|d| {
                    let m = fs::metadata(root.join(&d)).ok().filter(|m| m.is_dir())?;
                    Some((d, prompt::mtime_ns(&m)))
                })
                .collect();
            let mut dirs = dirs;
            dirs.sort_unstable();
            *self.walked_dirs.borrow_mut() = Some((dirs, old_manifest.nested.clone()));
        }
        self.write_manifest_entries(report.conflicted, false, &entries, &HashMap::new())?;
        if report.conflicted && !self.bare {
            report.kept_both = self.settle_unions(command)?;
        }
        let _ = self.run_hook(
            "post-checkout",
            &format!(
                "{{\"event\":\"post-checkout\",\"heads\":{},\"written\":{},\"removed\":{}}}",
                json_array(heads.iter().map(|h| json_str(&h.to_hex()))),
                report.written,
                report.removed
            ),
        );
        Ok(report)
    }

    /// How many changes of the state at `from` the state `to` lacks: a walk
    /// down from `from` that stops at what `to` holds.
    fn count_lacking(&self, from: &[ChangeHash], to: &State) -> Result<usize> {
        let mut seen: HashSet<ChangeHash> = HashSet::new();
        let mut stack: Vec<ChangeHash> = from.to_vec();
        while let Some(h) = stack.pop() {
            if to.contains(&h) || !seen.insert(h) {
                continue;
            }
            stack.extend(self.store.get_change_lazy(&h)?.0.deps);
        }
        Ok(seen.len())
    }

    /// Refuse to leave uncommitted work behind unless `discard`.
    pub(crate) fn refuse_dirty(&self, snap: &SnapshotResult, discard: bool) -> Result<()> {
        if discard || (snap.wc.tree_ops.is_empty() && snap.wc.edits.is_empty()) {
            return Ok(());
        }
        let list: Vec<String> = snap
            .statuses
            .iter()
            .take(8)
            .map(|s| format!("{} {}", status_word(s.status), s.path))
            .collect();
        fail(format!(
            "the working copy has changes ({}{}); commit them, take them along with --carry, or drop them with --discard",
            list.join(", "),
            if snap.statuses.len() > 8 { ", …" } else { "" }
        ))
    }

    /// Whether the file at `full` already holds `bytes` with this executable
    /// bit, so writing it again can be skipped.
    fn same_on_disk(&self, full: &Path, bytes: &[u8], exec: bool) -> bool {
        match fs::symlink_metadata(full) {
            Ok(m) if m.is_file() && m.len() == bytes.len() as u64 => {
                (!self.platform.exec_bits || is_executable(&m) == exec)
                    && fs::read(full).is_ok_and(|b| b == bytes)
            }
            _ => false,
        }
    }

    /// The absolute path for a repository path, refusing anything that would
    /// escape the workspace or write through a symlink (spec §7.7).
    fn safe_path(&self, rel: &str) -> Result<PathBuf> {
        let mut full = self.root.clone();
        let comps: Vec<&str> = rel.split('/').collect();
        for (i, comp) in comps.iter().enumerate() {
            if comp.is_empty() || *comp == "." || *comp == ".." || comp.contains('\\') {
                return fail(format!("refusing unsafe path {rel:?}"));
            }
            if i == 0 && *comp == OXY_DIR {
                return fail(format!("refusing to write inside {OXY_DIR}: {rel:?}"));
            }
            if i + 1 < comps.len() {
                full.push(comp);
                if let Ok(m) = fs::symlink_metadata(&full) {
                    if m.file_type().is_symlink() {
                        return fail(format!(
                            "refusing to write through symlink {}",
                            full.display()
                        ));
                    }
                }
            } else {
                full.push(comp);
            }
        }
        Ok(full)
    }

    pub(crate) fn write_file(&self, rel: &str, bytes: &[u8], exec: bool) -> Result<()> {
        let full = self.safe_path(rel)?;
        if let Some(parent) = full.parent() {
            fs::create_dir_all(parent)?;
        }
        if fs::symlink_metadata(&full).is_ok_and(|m| m.file_type().is_symlink() || m.is_dir()) {
            remove_any(&full)?;
        }
        if fs::metadata(&full).is_ok_and(|m| m.permissions().readonly()) {
            set_readonly(&full, false)?;
        }
        fs::write(&full, bytes)?;
        set_executable(&full, exec)?;
        Ok(())
    }

    pub(crate) fn write_symlink(&self, rel: &str, target: &[u8]) -> Result<()> {
        let full = self.safe_path(rel)?;
        if let Some(parent) = full.parent() {
            fs::create_dir_all(parent)?;
        }
        if fs::symlink_metadata(&full).is_ok() {
            remove_any(&full)?;
        }
        #[cfg(unix)]
        {
            std::os::unix::fs::symlink(String::from_utf8_lossy(target).as_ref(), &full)?;
            Ok(())
        }
        #[cfg(not(unix))]
        {
            // Spec §7.7: without symlink support, write the target as a plain file.
            fs::write(&full, target)?;
            Ok(())
        }
    }
}

/// How a refusal names a path's change: "removed" is a file of the working
/// copy's base that is missing on disk.
fn status_word(s: ChangeStatus) -> &'static str {
    match s {
        ChangeStatus::Added => "added",
        ChangeStatus::Modified => "modified",
        ChangeStatus::Removed => "removed",
        ChangeStatus::Renamed => "renamed",
        ChangeStatus::ModeChanged => "mode changed",
        ChangeStatus::ConflictEdited => "conflict edited",
        ChangeStatus::Resolved => "resolved",
    }
}

/// Refuse a path that could escape the workspace (spec §7.7): an empty,
/// `.` or `..` component, a backslash, or a first component `.oxy`.
fn check_path_name(rel: &str) -> Result<()> {
    for (i, comp) in rel.split('/').enumerate() {
        if comp.is_empty() || comp == "." || comp == ".." || comp.contains('\\') {
            return fail(format!("refusing unsafe path {rel:?}"));
        }
        if i == 0 && comp == OXY_DIR {
            return fail(format!("refusing to write inside {OXY_DIR}: {rel:?}"));
        }
    }
    Ok(())
}

/// A file on disk against what a checkout would write there.
#[derive(Clone, Copy, PartialEq, Eq)]
enum Disk {
    Same,
    Differs,
    Absent,
}

/// Whether the file at `full` already holds `bytes` with the right
/// executable bit. Touches no repository state, so it runs in parallel.
fn compare_on_disk(full: &Path, bytes: &[u8], exec: bool, exec_bits: bool) -> Disk {
    match fs::symlink_metadata(full) {
        Ok(m)
            if m.is_file()
                && m.len() == bytes.len() as u64
                && (!exec_bits || is_executable(&m) == exec)
                && fs::read(full).is_ok_and(|b| b == bytes) =>
        {
            Disk::Same
        }
        Ok(_) => Disk::Differs,
        Err(_) => Disk::Absent,
    }
}

/// Write a file whose directory exists, replacing whatever is there.
fn write_file_at(full: &Path, bytes: &[u8], exec: bool) -> std::io::Result<()> {
    match fs::symlink_metadata(full) {
        Ok(m) if m.is_file() => {
            if m.permissions().readonly() {
                set_readonly(full, false)?;
            }
        }
        Ok(_) => remove_any(full)?,
        Err(_) => {}
    }
    fs::write(full, bytes)?;
    set_executable(full, exec)
}
`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet
Coverage / crates/oxy-core/src/repo/checkout.rs 91.9%

Keyboard
