OXYGENHUB
# c30444e35f2e oxygenhub / crates / hub / src / repos / mod.rs
11.7 KiB · Rust · last changed Readers never wait for a push or gc 17 days ago source blame history raw
1
//! Repositories on disk (hub spec §4): naming, the registry of open
2
//! handles and their locks, and the page-ready views built from `oxy-core`.
3
//!
4
//! Every `oxy-core` call is synchronous and runs on the blocking pool under
5
//! the repository's lock (§16.2): pushes and other writes take the write
6
//! half, page reads the read half.
7
8
pub mod archive;
9
pub mod cache;
10
pub mod edits;
11
pub mod functions;
12
pub mod names;
13
pub mod proposals;
14
pub mod search;
15
pub mod view;
16
17
pub use cache::{Loaded, StateCache};
18
pub use names::RepoName;
19
20
use oxy_core::repo::Repo;
21
use std::collections::HashMap;
22
use std::path::{Path, PathBuf};
23
use std::sync::{Arc, Mutex};
24
use tokio::sync::{OwnedMutexGuard, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock};
25
26
/// A repository's locks (oxygenhub#249). Readers never wait for a push or
27
/// a gc: the store gives every handle one head and finds what a repack
28
/// moved (oxy-core, "readers never wait"). Writers take `writer` in turn;
29
/// everyone holds `life` shared, and only deleting or moving the
30
/// repository holds it alone.
31
#[derive(Default)]
32
struct Locks {
33
    life: Arc<RwLock<()>>,
34
    writer: Arc<tokio::sync::Mutex<()>>,
35
}
36
37
/// A reader's hold: the repository is not deleted or moved under it.
38
pub type ReadGuard = OwnedRwLockReadGuard<()>;
39
40
/// A writer's hold: no other writer runs, and the repository stays put.
41
/// Readers carry on.
42
pub struct WriteGuard {
43
    _writer: OwnedMutexGuard<()>,
44
    _life: OwnedRwLockReadGuard<()>,
45
}
46
47
/// Deleting or moving a repository: nothing else runs on it.
48
pub type ExclusiveGuard = OwnedRwLockWriteGuard<()>;
49
50
/// Where repositories live and how their access is serialised.
51
pub struct Repos {
52
    root: PathBuf,
53
    locks: Mutex<HashMap<String, Arc<Locks>>>,
54
    cache: Arc<StateCache>,
55
    pub searches: search::SearchCache,
56
}
57
58
#[derive(Debug, thiserror::Error)]
59
pub enum RepoError {
60
    #[error("no such repository")]
61
    NotFound,
62
    #[error("{0} already exists")]
63
    Exists(String),
64
    #[error("{0}")]
65
    Oxy(#[from] oxy_core::repo::RepoError),
66
    #[error("{0}")]
67
    Store(#[from] oxy_core::store::StoreError),
68
    #[error("{0}")]
69
    Io(#[from] std::io::Error),
70
}
71
72
impl Repos {
73
    pub fn new(root: PathBuf) -> Repos {
74
        // States are bounded by their estimated bytes (`cache::budget`); the
75
        // count only stops many tiny states piling up. At 64 it was always
76
        // full and evicting long before the bytes were (#145).
77
        let briefs = root.parent().unwrap_or(&root).join("cache").join("summaries").join(root.file_name().unwrap_or_default());
78
        let cache = Arc::new(StateCache::new(256).with_briefs(briefs));
79
        Repos { root, locks: Mutex::new(HashMap::new()), cache, searches: search::SearchCache::default() }
80
    }
81
82
    pub fn cache(&self) -> &Arc<StateCache> {
83
        &self.cache
84
    }
85
86
    pub fn root(&self) -> &Path {
87
        &self.root
88
    }
89
90
    /// `<root>/<owner>/<name>`, holding `.oxy/` and nothing else.
91
    pub fn dir(&self, name: &RepoName) -> PathBuf {
92
        self.root.join(&name.owner).join(&name.name)
93
    }
94
95
    pub fn exists(&self, name: &RepoName) -> bool {
96
        self.dir(name).join(oxy_core::repo::OXY_DIR).is_dir()
97
    }
98
99
    /// Open a hosted repository; it is always bare.
100
    pub fn open(&self, name: &RepoName) -> Result<Repo, RepoError> {
101
        if !self.exists(name) {
102
            return Err(RepoError::NotFound);
103
        }
104
        Ok(Repo::open(&self.dir(name))?)
105
    }
106
107
    /// Create an empty bare repository. With no `id` it is **unbound**: the
108
    /// first push binds it to the pusher's repository id, so an existing
109
    /// repository keeps its identity when it moves here (hub spec §4.1).
110
    pub fn create(&self, name: &RepoName, id: Option<[u8; 16]>) -> Result<Repo, RepoError> {
111
        let dir = self.dir(name);
112
        if dir.exists() {
113
            return Err(RepoError::Exists(name.to_string()));
114
        }
115
        Ok(match id {
116
            Some(id) => Repo::init_bare_with_id(&dir, id)?,
117
            None => Repo::init_bare_unbound(&dir)?,
118
        })
119
    }
120
121
    /// Create an empty, unbound bare repository whose objects and
122
    /// operation log are kept in the store at `url` (oxygenhub#96); the
123
    /// store's credentials must already be known to the opener.
124
    pub fn create_on(&self, name: &RepoName, url: &str) -> Result<Repo, RepoError> {
125
        let dir = self.dir(name);
126
        if dir.exists() {
127
            return Err(RepoError::Exists(name.to_string()));
128
        }
129
        let made = (|| -> Result<Repo, RepoError> {
130
            drop(Repo::init_bare_on(&dir, [0; 16], Some(url))?);
131
            // Unbound as `create` makes them: oxy-core's marker file in
132
            // place of the id, so the first push gives the id.
133
            let oxy = dir.join(oxy_core::repo::OXY_DIR);
134
            std::fs::remove_file(oxy.join("id"))?;
135
            std::fs::write(oxy.join("unbound"), b"")?;
136
            Ok(Repo::open(&dir)?)
137
        })();
138
        if made.is_err() {
139
            let _ = std::fs::remove_dir_all(&dir);
140
        }
141
        made
142
    }
143
144
    /// Fork `src` into `dest` (hub spec §4): a new bare repository with its
145
    /// own id, the source's objects hard-linked where the file system allows
146
    /// (copied otherwise; objects are immutable, so sharing them is safe),
147
    /// and the source's bookmarks. The trust policy is not copied: the
148
    /// fork's owner sets their own. Runs under the source's writer lock, so
149
    /// no push lands while its files are linked.
150
    pub async fn fork(&self, src: &RepoName, dest: &RepoName) -> Result<(), RepoError> {
151
        let guard = self.write_guard(src).await;
152
        let (from, to) = (self.dir(src), self.dir(dest));
153
        if !self.exists(src) {
154
            return Err(RepoError::NotFound);
155
        }
156
        if to.exists() {
157
            return Err(RepoError::Exists(dest.to_string()));
158
        }
159
        if Repo::open(&from).is_ok_and(|r| r.config.storage_url.is_some()) {
160
            return Err(RepoError::Io(std::io::Error::other(
161
                "a repository kept in a bucket cannot be forked yet: clone it and push the clone to a new repository",
162
            )));
163
        }
164
        tokio::task::spawn_blocking(move || {
165
            let _guard = guard;
166
            let made = (|| -> Result<(), RepoError> {
167
                drop(Repo::init_bare(&to)?);
168
                let objects = |d: &Path| d.join(oxy_core::repo::OXY_DIR).join("objects");
169
                link_tree(&objects(&from), &objects(&to))?;
170
                let source = Repo::open(&from)?;
171
                let fork = Repo::open(&to)?;
172
                let existing: Vec<String> = fork.bookmarks()?.into_iter().map(|b| b.name).collect();
173
                for b in source.bookmarks()? {
174
                    if existing.contains(&b.name) {
175
                        fork.bookmark_set(&b.name, &b.heads)?;
176
                    } else {
177
                        fork.bookmark_create(&b.name, Some(&b.heads))?;
178
                    }
179
                }
180
                Ok(())
181
            })();
182
            if made.is_err() {
183
                let _ = std::fs::remove_dir_all(&to);
184
            }
185
            made
186
        })
187
        .await
188
        .map_err(|e| RepoError::Io(std::io::Error::other(e.to_string())))?
189
    }
190
191
    /// Move a repository to another name (a transfer, #108): the directory,
192
    /// and the lock the old name held.
193
    pub fn rename(&self, from: &RepoName, to: &RepoName) -> Result<(), RepoError> {
194
        let (src, dest) = (self.dir(from), self.dir(to));
195
        if !src.exists() {
196
            return Err(RepoError::NotFound);
197
        }
198
        if dest.exists() {
199
            return Err(RepoError::Exists(to.to_string()));
200
        }
201
        if let Some(parent) = dest.parent() {
202
            std::fs::create_dir_all(parent)?;
203
        }
204
        std::fs::rename(&src, &dest)?;
205
        let mut locks = self.locks.lock().unwrap();
206
        if let Some(l) = locks.remove(&from.to_string()) {
207
            locks.insert(to.to_string(), l);
208
        }
209
        Ok(())
210
    }
211
212
    /// Move a repository to the trash (hub spec §4.1); a job empties it later.
213
    pub fn trash(&self, name: &RepoName) -> Result<PathBuf, RepoError> {
214
        let dir = self.dir(name);
215
        if !dir.exists() {
216
            return Err(RepoError::NotFound);
217
        }
218
        let trash = self.root.parent().unwrap_or(&self.root).join("trash").join(&name.owner);
219
        std::fs::create_dir_all(&trash)?;
220
        let dest = trash.join(format!("{}-{}", name.name, crate::db::now_ms()));
221
        std::fs::rename(&dir, &dest)?;
222
        Ok(dest)
223
    }
224
225
    fn lock(&self, name: &RepoName) -> Arc<Locks> {
226
        self.locks.lock().unwrap().entry(name.to_string()).or_default().clone()
227
    }
228
229
    /// Read while `f` runs on the blocking pool: never waiting for a
230
    /// writer, only for the repository being deleted or moved.
231
    pub async fn read<T: Send + 'static>(
232
        &self,
233
        name: &RepoName,
234
        f: impl FnOnce(&Repo, &StateCache) -> Result<T, RepoError> + Send + 'static,
235
    ) -> Result<T, RepoError> {
236
        let guard = self.read_guard(name).await;
237
        let dir = self.dir(name);
238
        let exists = self.exists(name);
239
        let cache = self.cache.clone();
240
        tokio::task::spawn_blocking(move || {
241
            let _guard = guard;
242
            if !exists {
243
                return Err(RepoError::NotFound);
244
            }
245
            let repo = Repo::open(&dir)?;
246
            f(&repo, &cache)
247
        })
248
        .await
249
        .map_err(|e| RepoError::Io(std::io::Error::other(e)))?
250
    }
251
252
    /// Write while `f` runs on the blocking pool, after the writers ahead.
253
    pub async fn write<T: Send + 'static>(
254
        &self,
255
        name: &RepoName,
256
        f: impl FnOnce(&Repo, &StateCache) -> Result<T, RepoError> + Send + 'static,
257
    ) -> Result<T, RepoError> {
258
        let guard = self.write_guard(name).await;
259
        let dir = self.dir(name);
260
        let exists = self.exists(name);
261
        let cache = self.cache.clone();
262
        tokio::task::spawn_blocking(move || {
263
            let _guard = guard;
264
            if !exists {
265
                return Err(RepoError::NotFound);
266
            }
267
            let repo = Repo::open(&dir)?;
268
            f(&repo, &cache)
269
        })
270
        .await
271
        .map_err(|e| RepoError::Io(std::io::Error::other(e)))?
272
    }
273
274
    /// The writer's hold alone, for callers that open the repository
275
    /// themselves (the protocol server's push, a backup's last pass). The
276
    /// writer lock first, so a writer queued behind others holds nothing a
277
    /// delete would wait for.
278
    pub async fn write_guard(&self, name: &RepoName) -> WriteGuard {
279
        let locks = self.lock(name);
280
        let writer = locks.writer.clone().lock_owned().await;
281
        WriteGuard { _writer: writer, _life: locks.life.clone().read_owned().await }
282
    }
283
284
    pub async fn read_guard(&self, name: &RepoName) -> ReadGuard {
285
        self.lock(name).life.clone().read_owned().await
286
    }
287
288
    /// Deleting or moving the repository: waits for every reader and
289
    /// writer, and holds off new ones.
290
    pub async fn exclusive_guard(&self, name: &RepoName) -> ExclusiveGuard {
291
        self.lock(name).life.clone().write_owned().await
292
    }
293
}
294
295
/// Hard-link every file under `from` into the same place under `to`,
296
/// copying when a link is not possible (another file system). Temporary
297
/// files of writers in progress are skipped.
298
fn link_tree(from: &Path, to: &Path) -> std::io::Result<()> {
299
    std::fs::create_dir_all(to)?;
300
    for e in std::fs::read_dir(from)? {
301
        let e = e?;
302
        let name = e.file_name();
303
        let n = name.to_string_lossy();
304
        if n.starts_with("tmp-") || n.starts_with("sigs-tmp-") || n.contains(".tmp") {
305
            continue;
306
        }
307
        let (src, dst) = (e.path(), to.join(&name));
308
        if e.file_type()?.is_dir() {
309
            link_tree(&src, &dst)?;
310
        } else if !dst.exists() && std::fs::hard_link(&src, &dst).is_err() {
311
            std::fs::copy(&src, &dst)?;
312
        }
313
    }
314
    Ok(())
315
}