//! Object store (spec §13.1, §13.3, §13.4): loose objects, one file per

//! object under `objects/loose/<hh>/<rest>`, and packs, `pack-<id>.pack`

//! with a sorted `pack-<id>.idx`, holding zstd-compressed objects. Reads

//! check loose objects first, then every pack; every read re-hashes.

//!

//! Pack file: `OXYPACK1` then entries of

//! `type u8 | flags u8 (bit 0: zstd) | stored_len u32 | raw_len u32 | ref 32 | body`.

//! Index file: `OXYIDX01` then `count u32` then sorted entries of

//! `ref 32 | offset u64 | stored_len u32 | raw_len u32 | type u8 | flags u8`.

use crate::attest::Attest;

use crate::hash::{hash_object, header, ObjectRef, ObjectType};

use crate::identity::{Identity, Signature};

use crate::objects::{Blob, Change, ObjectError, StateObject};

use crate::receipt::Receipt;

use rand::RngCore;

use std::cell::RefCell;

use std::fs;

use std::io::{self, Read, Seek, SeekFrom, Write};

use std::path::{Path, PathBuf};

const PACK_MAGIC: &[u8; 8] = b"OXYPACK1";

const IDX_MAGIC: &[u8; 8] = b"OXYIDX01";

const IDX_ENTRY: usize = 32 + 8 + 4 + 4 + 1 + 1;

const FLAG_ZSTD: u8 = 1;

/// Compressed with the pack's trained dictionary (`pack-ID.dict`), which a

/// full repack makes from a sample of the objects it packs: small files

/// share most of their vocabulary, which each one alone cannot use.

const FLAG_DICT: u8 = 2;

/// The size of a pack's dictionary, and the least a repack must pack for

/// one to pay.

const DICT_BYTES: usize = 110 << 10;

const DICT_MIN_OBJECTS: usize = 1000;

/// Objects up to this size are compressed with the dictionary.

const DICT_OBJECT_MAX: usize = 64 << 10;

const ZSTD_LEVEL: i32 = 3;

/// The level of a full repack (`oxy gc`): slower, once, for a smaller store.

pub const REPACK_ZSTD_LEVEL: i32 = 9;

/// Largest object a pack entry may claim; anything bigger is corruption,

/// and refusing it keeps a bad index from driving huge allocations.

const MAX_OBJECT_LEN: u32 = 1 << 30;

/// An object read from the store: its type and body.

pub type Object = (ObjectType, Vec<u8>);

/// Where a long operation reports what it is doing (oxygen#66): a phase,

/// or a count within one, as a line for a spinner or a log.

pub type Progress<'a> = &'a mut dyn FnMut(&str);

fn silent() -> impl FnMut(&str) {

    |_| {}

}

#[derive(Clone, Copy, Debug)]

struct PackEntry {

    offset: u64,

    stored_len: u32,

    raw_len: u32,

    ty: ObjectType,

    flags: u8,

}

/// One pack's index, fully in memory and sorted by ObjectRef.

struct PackIndex {

    path: PathBuf,

    refs: Vec<ObjectRef>,

    entries: Vec<PackEntry>,

    /// The pack's dictionary, prepared once, when it has one.

    dict: Option<std::sync::Arc<zstd::dict::DecoderDictionary<'static>>>,

}

impl std::fmt::Debug for PackIndex {

    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

        f.debug_struct("PackIndex")

            .field("path", &self.path)

            .field("objects", &self.refs.len())

            .field("dict", &self.dict.is_some())

            .finish()

    }

}

impl PackIndex {

    fn load(idx_path: &Path, pack_path: &Path) -> Result<PackIndex> {

        let data = fs::read(idx_path)?;

        let bad =

            |m: &str| StoreError::Corrupt(ObjectRef::ZERO, format!("{}: {m}", idx_path.display()));

        if data.len() < 12 || &data[..8] != IDX_MAGIC {

            return Err(bad("bad index header"));

        }

        let count = u32::from_le_bytes(data[8..12].try_into().unwrap()) as usize;

        if data.len() != 12 + count * IDX_ENTRY {

            return Err(bad("index length mismatch"));

        }

        let pack_len = fs::metadata(pack_path)?.len();

        let mut refs = Vec::with_capacity(count);

        let mut entries = Vec::with_capacity(count);

        for i in 0..count {

            let e = &data[12 + i * IDX_ENTRY..12 + (i + 1) * IDX_ENTRY];

            let r = ObjectRef::from_slice(&e[..32]).unwrap();

            if let Some(prev) = refs.last() {

                if r <= *prev {

                    return Err(bad("index not sorted"));

                }

            }

            refs.push(r);

            let entry = PackEntry {

                offset: u64::from_le_bytes(e[32..40].try_into().unwrap()),

                stored_len: u32::from_le_bytes(e[40..44].try_into().unwrap()),

                raw_len: u32::from_le_bytes(e[44..48].try_into().unwrap()),

                ty: ObjectType::ALL

                    .get(e[48] as usize)

                    .copied()

                    .ok_or_else(|| bad("unknown object type"))?,

                flags: e[49],

            };

            if entry.stored_len > MAX_OBJECT_LEN || entry.raw_len > MAX_OBJECT_LEN {

                return Err(StoreError::Corrupt(

                    r,

                    format!(

                        "{}: the index claims an object of {} bytes ({} stored), more than the {MAX_OBJECT_LEN} an object may be",

                        idx_path.display(),

                        entry.raw_len,

                        entry.stored_len

                    ),

                ));

            }

            if entry.offset.saturating_add(entry.stored_len as u64) > pack_len {

                return Err(bad("index entry points outside the pack"));

            }

            entries.push(entry);

        }

        let dict = match fs::read(pack_path.with_extension("dict")) {

            Ok(d) => Some(std::sync::Arc::new(zstd::dict::DecoderDictionary::copy(&d))),

            Err(e) if e.kind() == io::ErrorKind::NotFound => None,

            Err(e) => return Err(e.into()),

        };

        Ok(PackIndex {

            path: pack_path.to_path_buf(),

            refs,

            entries,

            dict,

        })

    }

    fn find(&self, r: &ObjectRef) -> Option<PackEntry> {

        self.refs.binary_search(r).ok().map(|i| self.entries[i])

    }

    fn read(&self, r: &ObjectRef, e: PackEntry) -> Result<(ObjectType, Vec<u8>)> {

        let mut f = fs::File::open(&self.path)?;

        f.seek(SeekFrom::Start(e.offset))?;

        let mut stored = vec![0u8; e.stored_len as usize];

        f.read_exact(&mut stored)?;

        let body = if e.flags & FLAG_DICT != 0 {

            let dict = self.dict.as_ref().ok_or_else(|| {

                StoreError::Corrupt(*r, "packed with a dictionary the pack lacks".into())

            })?;

            zstd::bulk::Decompressor::with_prepared_dictionary(dict)

                .and_then(|mut d| d.decompress(&stored, e.raw_len as usize))

                .map_err(|err| StoreError::Corrupt(*r, format!("zstd: {err}")))?

        } else if e.flags & FLAG_ZSTD != 0 {

            zstd::bulk::decompress(&stored, e.raw_len as usize)

                .map_err(|err| StoreError::Corrupt(*r, format!("zstd: {err}")))?

        } else {

            stored

        };

        if body.len() != e.raw_len as usize || hash_object(e.ty, &body) != *r {

            return Err(StoreError::Corrupt(

                *r,

                "packed object does not match its reference".into(),

            ));

        }

        Ok((e.ty, body))

    }

}

fn type_index(ty: ObjectType) -> u8 {

    ObjectType::ALL.iter().position(|t| *t == ty).unwrap() as u8

}

/// What `pack_loose` did.

#[derive(Debug, Clone, Default, PartialEq, Eq)]

pub struct PackReport {

    pub objects: usize,

    pub raw_bytes: u64,

    pub packed_bytes: u64,

}

/// Packs tolerated before `merge_packs` folds them into one (spec §13.4).

pub const MAX_PACKS: usize = 16;

/// What `prune` did.

#[derive(Debug, Clone, Default, PartialEq, Eq)]

pub struct PruneReport {

    pub objects: usize,

    pub bytes: u64,

    /// Unreachable objects left alone because they are younger than the grace period.

    pub deferred: usize,

}

#[derive(Debug, thiserror::Error)]

pub enum StoreError {

    #[error("io: {0}")]

    Io(#[from] io::Error),

    #[error("corrupt object {0}: {1}")]

    Corrupt(ObjectRef, String),

    #[error("object {0}")]

    Object(#[from] ObjectError),

    #[error("object {0} is a {1}, expected {2}")]

    WrongType(ObjectRef, &'static str, &'static str),

    #[error("object {0} not found")]

    NotFound(ObjectRef),

    #[error("a {kind} object of {size} bytes is larger than the {limit} bytes an object may be; nothing was stored. A change this large adds or touches too many files at once: commit it in parts (a directory at a time)")]

    TooLarge {

        kind: &'static str,

        size: usize,

        limit: u32,

    },

}

pub type Result<T> = std::result::Result<T, StoreError>;

#[derive(Debug)]

pub struct Store {

    dir: PathBuf,

    packs: RefCell<Vec<PackIndex>>,

    /// An open write batch (spec §13.4): objects go here instead of to loose

    /// files, and are written out with one `fsync` when the batch finishes.

    pending: RefCell<Option<Pending>>,

    batch_depth: std::cell::Cell<u32>,

}

/// Objects written since `begin_batch`. Small batches stay in memory and

/// become loose objects; once `SPILL_OBJECTS` or `SPILL_BYTES` is exceeded

/// the batch streams into a pack file instead.

#[derive(Debug)]

struct Pending {

    buffered: Vec<(ObjectType, Vec<u8>)>,

    buffered_bytes: usize,

    lookup: std::collections::HashMap<ObjectRef, PendingEntry>,

    pack: Option<PendingPack>,

    report: PackReport,

}

#[derive(Debug, Clone, Copy)]

enum PendingEntry {

    Buffered(usize),

    Packed(PackEntry),

}

#[derive(Debug)]

struct PendingPack {

    id: String,

    tmp_path: PathBuf,

    writer: io::BufWriter<fs::File>,

    offset: u64,

    index: Vec<(ObjectRef, PackEntry)>,

}

const SPILL_OBJECTS: usize = 256;

const SPILL_BYTES: usize = 8 << 20;

/// Finishes the enclosing batch when dropped; call `finish` to see errors.

pub struct Batch<'a> {

    store: &'a Store,

    done: bool,

}

impl Batch<'_> {

    pub fn finish(mut self) -> Result<()> {

        self.done = true;

        self.store.finish_batch()

    }

}

impl Drop for Batch<'_> {

    fn drop(&mut self) {

        if !self.done {

            let _ = self.store.finish_batch();

        }

    }

}

/// Append one stored object to a pending pack and index it.

fn write_pack_entry(

    pack: &mut PendingPack,

    r: &ObjectRef,

    ty: ObjectType,

    raw_len: usize,

    flags: u8,

    stored: &[u8],

    report: &mut PackReport,

) -> Result<PackEntry> {

    let header_len = 1 + 1 + 4 + 4 + 32;

    pack.writer.write_all(&[type_index(ty), flags])?;

    pack.writer

        .write_all(&(stored.len() as u32).to_le_bytes())?;

    pack.writer.write_all(&(raw_len as u32).to_le_bytes())?;

    pack.writer.write_all(&r.0)?;

    pack.writer.write_all(stored)?;

    let entry = PackEntry {

        offset: pack.offset + header_len,

        stored_len: stored.len() as u32,

        raw_len: raw_len as u32,

        ty,

        flags,

    };

    pack.offset += header_len + stored.len() as u64;

    pack.index.push((*r, entry));

    report.objects += 1;

    report.raw_bytes += raw_len as u64;

    report.packed_bytes += stored.len() as u64;

    Ok(entry)

}

impl Store {

    pub fn open(dir: impl Into<PathBuf>) -> io::Result<Store> {

        let dir = dir.into();

        fs::create_dir_all(dir.join("loose"))?;

        let store = Store {

            dir,

            packs: RefCell::new(Vec::new()),

            pending: RefCell::new(None),

            batch_depth: std::cell::Cell::new(0),

        };

        store

            .reload_packs()

            .map_err(|e| io::Error::other(e.to_string()))?;

        Ok(store)

    }

    fn reload_packs(&self) -> Result<()> {

        let mut packs = Vec::new();

        let mut names: Vec<PathBuf> = fs::read_dir(&self.dir)?

            .filter_map(|e| e.ok().map(|e| e.path()))

            .filter(|p| {

                p.extension().is_some_and(|x| x == "idx")

                    && p.file_name()

                        .is_some_and(|n| n.to_string_lossy().starts_with("pack-"))

            })

            .collect();

        names.sort();

        for idx in names {

            let pack = idx.with_extension("pack");

            if pack.is_file() {

                packs.push(PackIndex::load(&idx, &pack)?);

            }

        }

        *self.packs.borrow_mut() = packs;

        Ok(())

    }

    pub fn pack_count(&self) -> usize {

        self.packs.borrow().len()

    }

    fn find_packed(&self, r: &ObjectRef) -> Option<(usize, PackEntry)> {

        self.packs

            .borrow()

            .iter()

            .enumerate()

            .find_map(|(i, p)| p.find(r).map(|e| (i, e)))

    }

    /// Loose objects on disk, with their paths.

    fn loose_objects(&self) -> io::Result<Vec<(ObjectRef, PathBuf)>> {

        let mut out = Vec::new();

        let loose = self.dir.join("loose");

        for d in fs::read_dir(&loose)? {

            let d = d?;

            if !d.file_type()?.is_dir() {

                continue;

            }

            let prefix = d.file_name().to_string_lossy().to_string();

            for f in fs::read_dir(d.path())? {

                let f = f?;

                let name = f.file_name().to_string_lossy().to_string();

                if let Some(r) = ObjectRef::from_hex(&format!("{prefix}{name}")) {

                    out.push((r, f.path()));

                }

            }

        }

        out.sort();

        Ok(out)

    }

    pub fn loose_count(&self) -> io::Result<usize> {

        Ok(self.loose_objects()?.len())

    }

    /// Move every loose object into a new pack (spec §13.4). Loose files are

    /// deleted only after the pack and its index are durable.

    pub fn pack_loose(&self) -> Result<PackReport> {

        let loose = self.loose_objects()?;

        self.write_pack(

            &loose.iter().map(|(r, _)| *r).collect::<Vec<_>>(),

            &loose.iter().map(|(_, p)| p.clone()).collect::<Vec<_>>(),

            &[],

            ZSTD_LEVEL,

            &mut silent(),

        )

    }

    /// Fold every pack and loose object into one pack compressed at

    /// `level` (spec §13.4): what an explicit `oxy gc` does, as `git gc`

    /// repacks. Returns the report and how many packs were folded in.

    pub fn repack_all(&self, level: i32) -> Result<(PackReport, usize)> {

        self.repack_all_with(level, &mut silent())

    }

    /// [`Store::repack_all`], reporting progress.

    pub fn repack_all_with(&self, level: i32, progress: Progress) -> Result<(PackReport, usize)> {

        let loose = self.loose_objects()?;

        let (mut refs, paths): (Vec<ObjectRef>, Vec<PathBuf>) = {

            let packs = self.packs.borrow();

            (

                packs.iter().flat_map(|p| p.refs.iter().copied()).collect(),

                packs.iter().map(|p| p.path.clone()).collect(),

            )

        };

        if loose.is_empty() && paths.len() <= 1 && level == ZSTD_LEVEL {

            return Ok((PackReport::default(), 0));

        }

        refs.extend(loose.iter().map(|(r, _)| *r));

        let loose_paths: Vec<PathBuf> = loose.into_iter().map(|(_, p)| p).collect();

        let n = paths.len();

        let report = self.write_pack(&refs, &loose_paths, &paths, level, progress)?;

        Ok((report, n))

    }

    /// Fold every pack into one when there are more than `MAX_PACKS`

    /// (spec §13.4). Returns how many packs were merged.

    pub fn merge_packs(&self) -> Result<usize> {

        let (refs, paths): (Vec<ObjectRef>, Vec<PathBuf>) = {

            let packs = self.packs.borrow();

            if packs.len() <= MAX_PACKS {

                return Ok(0);

            }

            (

                packs.iter().flat_map(|p| p.refs.iter().copied()).collect(),

                packs.iter().map(|p| p.path.clone()).collect(),

            )

        };

        let n = paths.len();

        self.write_pack(&refs, &[], &paths, ZSTD_LEVEL, &mut silent())?;

        Ok(n)

    }

    /// Delete every object not in `keep` (spec §21). Loose objects and packs

    /// younger than `grace` are left alone, since a concurrent writer may not

    /// have recorded its references yet. Packs holding unreachable objects

    /// are rewritten without them. Signature sidecars and seal-index entries

    /// of deleted objects go too.

    pub fn prune(

        &self,

        keep: &std::collections::HashSet<ObjectRef>,

        grace: std::time::Duration,

    ) -> Result<PruneReport> {

        self.prune_with(keep, grace, &mut silent())

    }