OXYGENHUB
# b288cd254343 oxygen / crates / oxy-core / src / store.rs
8.5 KiB · Rust · last changed Import the Oxygen specification and the first Rust implementation (root change) 1 month ago source blame history raw
1
//! Loose object store (spec §13.1): one file per object under
2
//! `objects/loose/<hh>/<rest>`, written atomically and verified on read.
3
//! Packs come later.
4
5
use crate::hash::{hash_object, header, ObjectRef, ObjectType};
6
use crate::identity::{Identity, Signature};
7
use crate::objects::{Change, ObjectError};
8
use rand::RngCore;
9
use std::fs;
10
use std::io::{self, Write};
11
use std::path::{Path, PathBuf};
12
13
#[derive(Debug, thiserror::Error)]
14
pub enum StoreError {
15
    #[error("io: {0}")]
16
    Io(#[from] io::Error),
17
    #[error("corrupt object {0}: {1}")]
18
    Corrupt(ObjectRef, String),
19
    #[error("object {0}")]
20
    Object(#[from] ObjectError),
21
    #[error("object {0} is a {1}, expected {2}")]
22
    WrongType(ObjectRef, &'static str, &'static str),
23
    #[error("object {0} not found")]
24
    NotFound(ObjectRef),
25
}
26
27
pub type Result<T> = std::result::Result<T, StoreError>;
28
29
#[derive(Debug, Clone)]
30
pub struct Store {
31
    dir: PathBuf,
32
}
33
34
impl Store {
35
    pub fn open(dir: impl Into<PathBuf>) -> io::Result<Store> {
36
        let dir = dir.into();
37
        fs::create_dir_all(dir.join("loose"))?;
38
        Ok(Store { dir })
39
    }
40
41
    pub fn dir(&self) -> &Path {
42
        &self.dir
43
    }
44
45
    fn path_for(&self, r: &ObjectRef) -> PathBuf {
46
        let h = r.to_hex();
47
        self.dir.join("loose").join(&h[..2]).join(&h[2..])
48
    }
49
50
    pub fn has(&self, r: &ObjectRef) -> bool {
51
        self.path_for(r).is_file()
52
    }
53
54
    /// Store `body` as an object of type `ty`; idempotent.
55
    pub fn put(&self, ty: ObjectType, body: &[u8]) -> Result<ObjectRef> {
56
        let r = hash_object(ty, body);
57
        let path = self.path_for(&r);
58
        if path.is_file() {
59
            return Ok(r);
60
        }
61
        fs::create_dir_all(path.parent().unwrap())?;
62
        let mut nonce = [0u8; 8];
63
        rand::rngs::OsRng.fill_bytes(&mut nonce);
64
        let tmp = self.dir.join("loose").join(format!("tmp-{}", hex::encode(nonce)));
65
        {
66
            let mut f = fs::File::create(&tmp)?;
67
            f.write_all(&header(ty, body.len()))?;
68
            f.write_all(body)?;
69
            f.sync_all()?;
70
        }
71
        fs::rename(&tmp, &path)?;
72
        Ok(r)
73
    }
74
75
    /// Read and verify an object.
76
    pub fn get(&self, r: &ObjectRef) -> Result<(ObjectType, Vec<u8>)> {
77
        let path = self.path_for(r);
78
        let data = match fs::read(&path) {
79
            Ok(d) => d,
80
            Err(e) if e.kind() == io::ErrorKind::NotFound => return Err(StoreError::NotFound(*r)),
81
            Err(e) => return Err(e.into()),
82
        };
83
        let nul = data.iter().position(|&b| b == 0).ok_or_else(|| StoreError::Corrupt(*r, "no header".into()))?;
84
        let head = std::str::from_utf8(&data[..nul]).map_err(|_| StoreError::Corrupt(*r, "header not UTF-8".into()))?;
85
        let mut parts = head.split(' ');
86
        let (Some("oxy1"), Some(ty), Some(len), None) = (parts.next(), parts.next(), parts.next(), parts.next()) else {
87
            return Err(StoreError::Corrupt(*r, format!("bad header {head:?}")));
88
        };
89
        let ty = ObjectType::from_name(ty).ok_or_else(|| StoreError::Corrupt(*r, format!("unknown type {ty}")))?;
90
        let len: usize = len.parse().map_err(|_| StoreError::Corrupt(*r, "bad length".into()))?;
91
        let body = &data[nul + 1..];
92
        if body.len() != len {
93
            return Err(StoreError::Corrupt(*r, format!("length {} != header {len}", body.len())));
94
        }
95
        if hash_object(ty, body) != *r {
96
            return Err(StoreError::Corrupt(*r, "hash mismatch".into()));
97
        }
98
        Ok((ty, body.to_vec()))
99
    }
100
101
    pub fn get_typed(&self, r: &ObjectRef, want: ObjectType) -> Result<Vec<u8>> {
102
        let (ty, body) = self.get(r)?;
103
        if ty != want {
104
            return Err(StoreError::WrongType(*r, ty.name(), want.name()));
105
        }
106
        Ok(body)
107
    }
108
109
    pub fn put_change(&self, c: &Change) -> Result<ObjectRef> {
110
        self.put(ObjectType::Change, &c.encode())
111
    }
112
113
    pub fn get_change(&self, r: &ObjectRef) -> Result<Change> {
114
        Ok(Change::decode(&self.get_typed(r, ObjectType::Change)?)?)
115
    }
116
117
    pub fn put_chunk(&self, bytes: &[u8]) -> Result<ObjectRef> {
118
        self.put(ObjectType::Chunk, bytes)
119
    }
120
121
    pub fn put_identity(&self, id: &Identity) -> Result<ObjectRef> {
122
        self.put(ObjectType::Identity, &id.encode())
123
    }
124
125
    pub fn get_identity(&self, r: &ObjectRef) -> Result<Identity> {
126
        Ok(Identity::decode(&self.get_typed(r, ObjectType::Identity)?)?)
127
    }
128
129
    fn sig_path(&self, r: &ObjectRef) -> PathBuf {
130
        let h = r.to_hex();
131
        self.dir.join("sigs").join(&h[..2]).join(&h[2..])
132
    }
133
134
    /// Signatures stored alongside object `r` (spec §5.4). Empty if none.
135
    pub fn signatures(&self, r: &ObjectRef) -> Result<Vec<Signature>> {
136
        match fs::read(self.sig_path(r)) {
137
            Ok(b) => Ok(Signature::decode_list(&b)?),
138
            Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(vec![]),
139
            Err(e) => Err(e.into()),
140
        }
141
    }
142
143
    /// Attach a signature to object `r`, replacing any earlier one by the same key.
144
    pub fn put_signature(&self, r: &ObjectRef, sig: &Signature) -> Result<()> {
145
        let mut sigs = self.signatures(r)?;
146
        sigs.retain(|s| s.key != sig.key);
147
        sigs.push(sig.clone());
148
        sigs.sort_by(|a, b| a.key.cmp(&b.key));
149
        let path = self.sig_path(r);
150
        fs::create_dir_all(path.parent().unwrap())?;
151
        let mut nonce = [0u8; 8];
152
        rand::rngs::OsRng.fill_bytes(&mut nonce);
153
        let tmp = self.dir.join(format!("sigs-tmp-{}", hex::encode(nonce)));
154
        {
155
            let mut f = fs::File::create(&tmp)?;
156
            f.write_all(&Signature::encode_list(&sigs))?;
157
            f.sync_all()?;
158
        }
159
        fs::rename(&tmp, &path)?;
160
        Ok(())
161
    }
162
163
    /// Every object reference in the store (slow; for gc and tests).
164
    pub fn list(&self) -> io::Result<Vec<ObjectRef>> {
165
        let mut out = Vec::new();
166
        let loose = self.dir.join("loose");
167
        for d in fs::read_dir(&loose)? {
168
            let d = d?;
169
            if !d.file_type()?.is_dir() {
170
                continue;
171
            }
172
            let prefix = d.file_name().to_string_lossy().to_string();
173
            for f in fs::read_dir(d.path())? {
174
                let f = f?;
175
                let name = f.file_name().to_string_lossy().to_string();
176
                if let Some(r) = ObjectRef::from_hex(&format!("{prefix}{name}")) {
177
                    out.push(r);
178
                }
179
            }
180
        }
181
        out.sort();
182
        Ok(out)
183
    }
184
}
185
186
#[cfg(test)]
187
mod tests {
188
    use super::*;
189
190
    #[test]
191
    fn put_get_verify() {
192
        let dir = tempfile::tempdir().unwrap();
193
        let s = Store::open(dir.path().join("objects")).unwrap();
194
        let r = s.put(ObjectType::Chunk, b"hello").unwrap();
195
        assert!(s.has(&r));
196
        assert_eq!(s.get(&r).unwrap(), (ObjectType::Chunk, b"hello".to_vec()));
197
        assert_eq!(s.put(ObjectType::Chunk, b"hello").unwrap(), r);
198
        assert_eq!(s.list().unwrap(), vec![r]);
199
        assert!(matches!(s.get_typed(&r, ObjectType::Blob), Err(StoreError::WrongType(..))));
200
        assert!(matches!(s.get(&ObjectRef([9; 32])), Err(StoreError::NotFound(_))));
201
202
        // Tamper: flip a byte, read must fail.
203
        let path = s.path_for(&r);
204
        let mut data = fs::read(&path).unwrap();
205
        let last = data.len() - 1;
206
        data[last] ^= 1;
207
        fs::write(&path, data).unwrap();
208
        assert!(matches!(s.get(&r), Err(StoreError::Corrupt(..))));
209
    }
210
211
    #[test]
212
    fn signatures_are_stored_beside_objects() {
213
        use crate::identity::{Domain, SecretKey};
214
        let dir = tempfile::tempdir().unwrap();
215
        let s = Store::open(dir.path()).unwrap();
216
        let r = s.put(ObjectType::Chunk, b"x").unwrap();
217
        assert!(s.signatures(&r).unwrap().is_empty());
218
        let sk = SecretKey { ed_seed: [1; 32], ml_seed: [2; 32] };
219
        let id = sk.identity("a", "b", 0);
220
        let key = s.put_identity(&id).unwrap();
221
        assert_eq!(s.get_identity(&key).unwrap(), id);
222
        let sig = sk.sign(Domain::Change, &key, &r);
223
        s.put_signature(&r, &sig).unwrap();
224
        s.put_signature(&r, &sig).unwrap();
225
        assert_eq!(s.signatures(&r).unwrap(), vec![sig]);
226
        // The object itself is untouched by its signatures.
227
        assert_eq!(s.get(&r).unwrap().1, b"x");
228
    }
229
230
    #[test]
231
    fn change_round_trip_through_store() {
232
        let dir = tempfile::tempdir().unwrap();
233
        let s = Store::open(dir.path()).unwrap();
234
        let c = crate::objects::tests::sample_change();
235
        let r = s.put_change(&c).unwrap();
236
        assert_eq!(r, c.hash());
237
        assert_eq!(s.get_change(&r).unwrap(), c);
238
    }
239
}