Git import: a large merge renders its tree in batches c3d2ef6104c2

/// Text files of a merge rendered this many at a time (oxygen#25): a

/// batch's graphs are built beside the state and dropped once rendered,

/// so what is held for the whole tree is the renderings, not the graphs

/// with every deleted line of their history.

const MERGE_BATCH: usize = 500;

thread_local! {

    /// A smaller batch for this thread's imports: tests set 1, so a small

    /// merge takes the batched path.

    static MERGE_BATCH_OVERRIDE: std::cell::Cell<Option<usize>> = const { std::cell::Cell::new(None) };

}

#[cfg(test)]

pub(crate) fn set_merge_batch_for_tests(n: Option<usize>) {

    MERGE_BATCH_OVERRIDE.with(|c| c.set(n));

}

fn merge_batch() -> usize {

    MERGE_BATCH_OVERRIDE

        .with(|c| c.get())

        .filter(|n| *n > 0)

        .unwrap_or(MERGE_BATCH)

}

impl Repo {

    /// As `State::materialize_except`, for a tree whose graphs would not

    /// fit in memory at once. The few files the change builder reads

    /// graphs for (conflicted ones, and renderings whose lines are not

    /// one per live atom) are built in the state afterwards.

    fn materialize_batched(

        &self,

        base: &mut State,

        need: &[crate::ids::NodeId],

        skip: &HashSet<crate::ids::NodeId>,

    ) -> Result<crate::state::Snapshot> {

        use crate::graph::Materialized;

        use crate::state::{FileContent, FileState};

        use rayon::prelude::*;

        let view = base.tree().view();

        let mut rendered: HashMap<crate::ids::NodeId, (Vec<u8>, Materialized)> =

            HashMap::with_capacity(need.len());

        let mut keep_built: Vec<crate::ids::NodeId> = Vec::new();

        for batch in need.chunks(merge_batch()) {

            let t = std::time::Instant::now();

            let graphs = self.build_graphs_detached(base, batch)?;

            crate::timing::add("render: build graphs, batch", t.elapsed());

            let t = std::time::Instant::now();

            let batch_rendered: Vec<(crate::ids::NodeId, Vec<u8>, Materialized)> = graphs

                .par_iter()

                .map(|(node, g)| {

                    let (bytes, m) = base.render_graph(g);

                    (*node, bytes, m)

                })

                .collect();

            drop(graphs);

            crate::timing::add("render: render, batch", t.elapsed());

            for (node, bytes, m) in batch_rendered {

                if m.is_conflicted()

                    || m.live_atoms().len() != crate::diff::split_lines(&bytes).len()

                {

                    keep_built.push(node);

                }

                rendered.insert(node, (bytes, m));

            }

        }

        if !keep_built.is_empty() {

            self.load_graphs_of(base, &keep_built)?;

        }

        let mut files = BTreeMap::new();

        let mut unrendered = BTreeSet::new();

        for (path, e) in view.entries.iter() {

            let f = if e.kind == NodeKind::File && !e.mode.opaque {

                match rendered.remove(&e.node) {

                    Some((bytes, materialized)) => FileState {

                        node: e.node,

                        kind: e.kind,

                        mode: e.mode,

                        content: FileContent::Text {

                            bytes,

                            materialized,

                        },

                    },

                    None if skip.contains(&e.node) => {

                        unrendered.insert(path.clone());

                        FileState {

                            node: e.node,

                            kind: e.kind,

                            mode: e.mode,

                            content: FileContent::Text {

                                bytes: Vec::new(),

                                materialized: Materialized::default(),

                            },

                        }

                    }

                    None => {

                        // Neither asked for nor known unchanged: render it

                        // from a built graph, as the whole-tree path does.

                        self.load_graphs_of(base, &[e.node])?;

                        base.entry_state(e)

                    }

                }

            } else {

                base.entry_state(e)

            };

            files.insert(path.clone(), f);

        }

        Ok(crate::state::Snapshot {

            tree: view,

            files,

            unrendered,

        })

    }

}

                let snap = if need.len() > merge_batch() {

                    // A merge onto a base with no built graphs: thousands

                    // of files, whose graphs would not fit at once.

                    self.materialize_batched(&mut base, &need, &skip)?

                } else {

                    self.load_graphs_of(&mut base, &need)?;

                    base.materialize_except(&skip)

                };

                crate::timing::add("render: whole tree", t.elapsed());

    git_import_reproduces_trees_with(None);

}

/// The same import with every merge rendered in batches of one file

/// (oxygen#25): what a merge onto a base of thousands of files takes.

#[test]

fn git_import_reproduces_trees_in_batches() {

    git_import_reproduces_trees_with(Some(1));

}

fn git_import_reproduces_trees_with(batch: Option<usize>) {

    crate::repo::git::set_merge_batch_for_tests(batch);

    /// The state of one tree entry: a directory, an opaque file's live

    /// blobs, or a text file rendered from its graph (which must be built).

    pub fn entry_state(&self, e: &crate::tree::TreeEntry) -> FileState {

        self.file_state(e)

    }