Read a packed object from its pack before asking for a loose copy 092457020692

    /// Read and verify an object, packed, loose, or pending in a batch.

    /// The packs' indexes are in memory, so they are asked first: asking

    /// for a loose copy of a packed object costs a request that finds

    /// nothing on a store in a bucket, every time (oxygenhub#228).

        if let Some((i, e)) = self.find_packed(r) {

            return self.packs.borrow()[i].fetch(self.backend.as_ref(), r, e);

        }

            None => self.pending_get(r).unwrap_or(Err(StoreError::NotFound(*r))),

    /// Many objects at once, read and verified in parallel: packed ones

    /// from their packs, the others loose (as [`Store::get`], packs first),

    /// then what is pending in a batch. Results are in the order of `refs`.

        let place: Vec<Option<(usize, PackEntry)>> = refs

            .map(|r| {

                    .find_map(|(p, pack)| pack.find(r).map(|e| (p, e)))

        let found: Vec<Option<Result<Object>>> = refs

            .zip(place.par_iter())

            .map(|(r, at)| match at {

                Some((p, e)) => Some(indexes[*p].fetch(backend, r, *e)),

                None => match get_optional(backend, &loose_key(r)) {

                    Ok(Some(bytes)) => Some(parse_loose(&bytes, r)),

                    Ok(None) => None,

                    Err(e) => Some(Err(e)),

                },

            })

            .map(|(f, r)| {

                f.unwrap_or_else(|| self.pending_get(r).unwrap_or(Err(StoreError::NotFound(*r))))

    }

    /// A packed object is read from its pack without first asking the

    /// backend for a loose copy (oxygenhub#228: on a bucket each such ask

    /// was a request that found nothing, every time).

    #[test]

    fn packed_objects_cost_no_loose_lookups() {

        use std::sync::atomic::{AtomicUsize, Ordering};

        struct Counting(MemoryStore, AtomicUsize);

        impl crate::strata::Store for Counting {

            fn describe(&self) -> String {

                "counting".into()

            }

            fn has(&self, k: &str) -> crate::strata::Result<bool> {

                self.0.has(k)

            }

            fn get(&self, k: &str) -> crate::strata::Result<Vec<u8>> {

                if k.starts_with("loose/") {

                    self.1.fetch_add(1, Ordering::Relaxed);

                }

                self.0.get(k)

            }

            fn get_range(

                &self,

                k: &str,

                r: std::ops::Range<u64>,

            ) -> crate::strata::Result<Vec<u8>> {

                self.0.get_range(k, r)

            }

            fn put_new(&self, k: &str, b: &[u8]) -> crate::strata::Result<()> {

                self.0.put_new(k, b)

            }

            fn list(&self, p: &str) -> crate::strata::Result<Vec<crate::strata::Entry>> {

                self.0.list(p)

            }

            fn delete(&self, k: &str) -> crate::strata::Result<()> {

                self.0.delete(k)

            }

            fn read_ref(

                &self,

                k: &str,

            ) -> crate::strata::Result<Option<(Vec<u8>, crate::strata::Version)>> {

                self.0.read_ref(k)

            }

            fn write_ref(

                &self,

                k: &str,

                b: &[u8],

                e: Option<&crate::strata::Version>,

            ) -> crate::strata::Result<crate::strata::Version> {

                self.0.write_ref(k, b, e)

            }

        }

        let dir = tempfile::tempdir().unwrap();

        let backend = Arc::new(Counting(MemoryStore::new(), AtomicUsize::new(0)));

        let s = Store::open_on(backend.clone(), dir.path().join("scratch")).unwrap();

        let refs: Vec<ObjectRef> = (0..10)

            .map(|i| {

                s.put(ObjectType::Chunk, format!("p {i}").as_bytes())

                    .unwrap()

            })

            .collect();

        s.pack_loose().unwrap();

        let s = Store::open_on(backend.clone(), dir.path().join("scratch")).unwrap();

        backend.1.store(0, Ordering::Relaxed);

        for r in &refs {

            assert!(s.get(r).is_ok());

        }

        assert!(s.get_many(&refs).iter().all(|g| g.is_ok()));

        assert_eq!(backend.1.load(Ordering::Relaxed), 0, "no loose lookups");

        // An object no pack holds is still looked for loose.

        let missing = hash_object(ObjectType::Chunk, b"never stored");

        assert!(matches!(s.get(&missing), Err(StoreError::NotFound(_))));

        assert!(backend.1.load(Ordering::Relaxed) > 0);