Fewer waits on the disk: batched loose writes, the commit in one batch, fan-outs kept (refs #41) b30a880fa409

        // The commit's objects are written together, made durable at once

        // before the operation that names them (spec §13.6).

        let batch = self.store.begin_batch()?;

        let out = self.commit_selection_batched(message, extra_deps, selection);

        batch.finish()?;

        out

    }

    fn commit_selection_batched(

        &self,

        message: &str,

        extra_deps: &[ChangeHash],

        selection: &Selection,

    ) -> Result<ChangeHash> {

            // The fan-out directory stays: recreating it costs a sync of

            // `loose` on the next write.

        // Fan-out directories stay, emptied: see `ensure_fanouts`.

                let objects: Vec<(ObjectType, &[u8])> = pending

                    .buffered

                    .iter()

                    .map(|(ty, body)| (*ty, body.as_slice()))

                    .collect();

                self.put_loose_many(&objects)

        if !path.parent().unwrap().is_dir() {

            self.ensure_fanouts()?;

    }

    /// Create every fan-out directory of `loose` (`00` to `ff`) and sync

    /// `loose` once. Syncing it is slow on a busy disk (130 ms seen), and

    /// creating the directories one at a time paid that on most writes

    /// after a gc had emptied them; they are kept from then on.

    fn ensure_fanouts(&self) -> io::Result<()> {

        let loose = self.dir.join("loose");

        for b in 0..=255u8 {

            match fs::create_dir(loose.join(format!("{b:02x}"))) {

                Ok(()) => {}

                Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {}

                Err(e) => return Err(e),

            }

        }

        crate::durable::sync_dir(&loose)

    }

    /// Write several loose objects with as few waits on the disk as the

    /// guarantees allow: every file written, then all synced together, then

    /// each renamed into place and each directory synced once. Durable when

    /// this returns, as `put_loose` one at a time would be (spec §13.6).

    fn put_loose_many(&self, objects: &[(ObjectType, &[u8])]) -> Result<()> {

        use rayon::prelude::*;

        if objects.len() <= 1 {

            for (ty, body) in objects {

                self.put_loose(*ty, body)?;

            }

            return Ok(());

        }

        let mut seen = std::collections::HashSet::new();

        let todo: Vec<(PathBuf, PathBuf, ObjectType, &[u8])> = objects

            .iter()

            .filter_map(|(ty, body)| {

                let r = hash_object(*ty, body);

                let path = self.path_for(&r);

                (seen.insert(r) && !path.is_file()).then(|| {

                    let mut nonce = [0u8; 8];

                    rand::rngs::OsRng.fill_bytes(&mut nonce);

                    let tmp = self

                        .dir

                        .join("loose")

                        .join(format!("tmp-{}", hex::encode(nonce)));

                    (path, tmp, *ty, *body)

                })

            })

            .collect();

        if todo.is_empty() {

            return Ok(());

        }

        let fanouts: std::collections::BTreeSet<&Path> =

            todo.iter().map(|(p, ..)| p.parent().unwrap()).collect();

        if fanouts.iter().any(|f| !f.is_dir()) {

            self.ensure_fanouts()?;

        }

        crate::durable::step("write loose objects")?;

        todo.par_iter()

            .try_for_each(|(_, tmp, ty, body)| -> io::Result<()> {

                let mut f = fs::File::create(tmp)?;

                f.write_all(&header(*ty, body.len()))?;

                f.write_all(body)?;

                f.sync_all()

            })?;

        crate::durable::step("rename loose objects")?;

        for (path, tmp, ..) in &todo {

            fs::rename(tmp, path)?;

        }

        fanouts

            .par_iter()

            .try_for_each(|d| crate::durable::sync_dir(d))?;

        Ok(())

Fewer waits on the disk (same evening): a batch's loose objects are

written together, synced in parallel, renamed and their directories

synced once each; a commit runs in one batch; packing keeps the emptied

fan-out directories, and a missing one has all 256 created with one sync

of `loose` (that sync alone took 130 ms on a busy disk). Serial syncs per

commit fall from 18 to about 7; each still waits on the same ext4 journal

commit, so the gain is modest. Old and new builds alternating, five

commits after a status each: median 0.39 s before, 0.34 s after (Git

0.14 to 0.16 s).