git import: graphs shed before large merges and from kept states (refs #25) fd280228bdc2

/// A merge bringing more commits than this sheds its base's graphs first.

const MERGE_SHED_COMMITS: usize = 100;

/// A kept state with more built graphs than this sheds them.

const KEPT_GRAPHS: usize = 2000;

        // Graphs turned back into history (see `State::shed_graphs`).

        let mut shed = 0usize;

                // A merge of a long-lived branch brings thousands of

                // changes: added to built graphs, which kept states share,

                // they were copied and applied whole (vscode's import passed

                // 14 GB on such merges). Shed the graphs first, so the

                // changes go in as skeletons and only what this commit

                // touches is built again.

                if base.built_count() > 0 {

                    let mut count_args = vec!["rev-list", "--count"];

                    count_args.extend(refs.iter().copied());

                    let others: usize = git(git_dir, &count_args)?.trim().parse().unwrap_or(0);

                    if others > MERGE_SHED_COMMITS {

                        shed += base.shed_graphs(|b| self.store_blob(b))?;

                    }

                }

                // A kept state holds its graphs until its children come:

                // past a few thousand, they go back into its history.

                if base.built_count() > KEPT_GRAPHS {

                    shed += base.shed_graphs(|b| self.store_blob(b))?;

                }

                    "trace {done} {} parents={} base={base_how} merged={merged} built={built} kept={} shed={shed} rss_mb={}",

            eprintln!(

                "timing git import: {rebuilt} bases rebuilt from history, {shed} graphs shed"

            );

#[test]

fn shed_graphs_come_back_the_same() {

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let repo = Repo::init(root).unwrap();

    write(root, "a.txt", b"1\n2\n3\n");

    write(root, "b.txt", b"x\n");

    repo.commit("first", &[]).unwrap();

    write(root, "a.txt", b"1\ntwo\n3\n4\n");

    repo.commit("second", &[]).unwrap();

    let heads = repo.bookmark_heads().unwrap();

    let mut st = repo.load_state_history(&heads).unwrap();

    let view = st.tree().view();

    let nodes: Vec<NodeId> = ["a.txt", "b.txt"]

        .iter()

        .map(|p| view.entries[*p].node)

        .collect();

    repo.load_graphs_of(&mut st, &nodes).unwrap();

    let before: Vec<Vec<u8>> = nodes.iter().map(|n| st.render_file(n).0).collect();

    assert_eq!(st.built_count(), 2);

    let shed = st.shed_graphs(|b| repo.store_blob(b)).unwrap();

    assert_eq!((shed, st.built_count()), (2, 0));

    // A later change applies on top of the shed history.

    write(root, "a.txt", b"1\ntwo\n3\n4\n5\n");

    let h3 = repo.commit("third", &[]).unwrap();

    let (skeleton, texts) = repo.store().get_change_lazy(&h3).unwrap();

    st.add_known(h3, skeleton, texts);

    repo.load_graphs_of(&mut st, &nodes).unwrap();

    let after: Vec<Vec<u8>> = nodes.iter().map(|n| st.render_file(n).0).collect();

    assert_eq!(after[1], before[1]);

    assert_eq!(after[0], b"1\ntwo\n3\n4\n5\n");

}

    /// Turn this history-only state's built graphs back into deferred

    /// histories, each starting from its graph encoded as a checkpoint

    /// graph and stored by `store` (it returns the blob's reference): what

    /// they held stays, and memory goes back to what a history-only state

    /// costs. A graph built later again decodes that blob and applies only

    /// the edits that came after. Graph keys change (the base is now a

    /// blob), which only means a cached rendering is not reused.

    pub fn shed_graphs<E>(

        &mut self,

        mut store: impl FnMut(&[u8]) -> Result<ObjectRef, E>,

    ) -> Result<usize, E> {

        use rayon::prelude::*;

        let Some(deferred) = self.deferred.as_mut() else {

            return Ok(0);

        };

        let graphs: Vec<(NodeId, Arc<FileGraph>)> = self.files.drain().collect();

        let encoded: Vec<(NodeId, Vec<u8>)> = graphs

            .par_iter()

            .map(|(n, g)| (*n, cbor::encode(&g.to_value())))

            .collect();

        drop(graphs);

        let shed = encoded.len();

        for (node, bytes) in encoded {

            let blob = store(&bytes)?;

            deferred.insert(

                node,

                Arc::new(Deferred::from_body(DeferredBody {

                    base: Some(DeferredBase::Blob(blob)),

                    ..DeferredBody::default()

                })),

            );

        }

        Ok(shed)

    }