OXYGENHUB

Coverage / crates/hub/src/repos/mod.rs 82.5%

165 of 200 lines run, 381 of 1040 functions · run #482 on proposal/333 · run missed

1
//! Repositories on disk (hub spec §4): naming, the registry of open
2
//! handles and their locks, and the page-ready views built from `oxy-core`.
3
//!
4
//! Every `oxy-core` call is synchronous and runs on the blocking pool under
5
//! the repository's lock (§16.2): pushes and other writes take the write
6
//! half, page reads the read half.
7
8
pub mod archive;
9
pub mod cache;
10
pub mod edits;
11
pub mod functions;
12
pub mod names;
13
pub mod proposals;
14
pub mod search;
15
pub mod view;
16
17
pub use cache::{Loaded, StateCache};
18
pub use names::RepoName;
19
20
use oxy_core::repo::Repo;
21
use std::collections::HashMap;
22
use std::path::{Path, PathBuf};
23
use std::sync::{Arc, Mutex};
24
use tokio::sync::{OwnedMutexGuard, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock};
25
26
/// A repository's locks (oxygenhub#249). Readers never wait for a push or
27
/// a gc: the store gives every handle one head and finds what a repack
28
/// moved (oxy-core, "readers never wait"). Writers take `writer` in turn;
29
/// everyone holds `life` shared, and only deleting or moving the
30
/// repository holds it alone.
31
#[derive(Default)]
32
struct Locks {
33
    life: Arc<RwLock<()>>,
34
    writer: Arc<tokio::sync::Mutex<()>>,
35
}
36
37
/// A reader's hold: the repository is not deleted or moved under it.
38
pub type ReadGuard = OwnedRwLockReadGuard<()>;
39
40
/// A writer's hold: no other writer runs, and the repository stays put.
41
/// Readers carry on.
42
pub struct WriteGuard {
43
    _writer: OwnedMutexGuard<()>,
44
    _life: OwnedRwLockReadGuard<()>,
45
}
46
47
/// Deleting or moving a repository: nothing else runs on it.
48
pub type ExclusiveGuard = OwnedRwLockWriteGuard<()>;
49
50
/// Where repositories live and how their access is serialised.
51
pub struct Repos {
52
    root: PathBuf,
53
    locks: Mutex<HashMap<String, Arc<Locks>>>,
54
    cache: Arc<StateCache>,
55
    pub searches: search::SearchCache,
56
}
57
58
#[derive(Debug, thiserror::Error)]
59
pub enum RepoError {
60
    #[error("no such repository")]
61
    NotFound,
62
    #[error("{0} already exists")]
63
    Exists(String),
64
    #[error("{0}")]
65
    Oxy(#[from] oxy_core::repo::RepoError),
66
    #[error("{0}")]
67
    Store(#[from] oxy_core::store::StoreError),
68
    #[error("{0}")]
69
    Io(#[from] std::io::Error),
70
}
71
72
impl Repos {
73386×
    pub fn new(root: PathBuf) -> Repos {
74
        // States are bounded by their estimated bytes (`cache::budget`); the
75
        // count only stops many tiny states piling up. At 64 it was always
76
        // full and evicting long before the bytes were (#145).
77386×
        let briefs = root.parent().unwrap_or(&root).join("cache").join("summaries").join(root.file_name().unwrap_or_default());
78386×
        let cache = Arc::new(StateCache::new(256).with_briefs(briefs));
79386×
        Repos { root, locks: Mutex::new(HashMap::new()), cache, searches: search::SearchCache::default() }
80386×
    }
81
829×
    pub fn cache(&self) -> &Arc<StateCache> {
839×
        &self.cache
849×
    }
85
86297×
    pub fn root(&self) -> &Path {
87297×
        &self.root
88297×
    }
89
90
    /// `<root>/<owner>/<name>`, holding `.oxy/` and nothing else.
9110813×
    pub fn dir(&self, name: &RepoName) -> PathBuf {
9210813×
        self.root.join(&name.owner).join(&name.name)
9310813×
    }
94
955353×
    pub fn exists(&self, name: &RepoName) -> bool {
965353×
        self.dir(name).join(oxy_core::repo::OXY_DIR).is_dir()
975353×
    }
98
99
    /// Open a hosted repository; it is always bare.
1001834×
    pub fn open(&self, name: &RepoName) -> Result<Repo, RepoError> {
1011834×
        if !self.exists(name) {
1021×
            return Err(RepoError::NotFound);
1031833×
        }
1041833×
        Ok(Repo::open(&self.dir(name))?)
1051834×
    }
106
107
    /// Create an empty bare repository. With no `id` it is **unbound**: the
108
    /// first push binds it to the pusher's repository id, so an existing
109
    /// repository keeps its identity when it moves here (hub spec §4.1).
110193×
    pub fn create(&self, name: &RepoName, id: Option<[u8; 16]>) -> Result<Repo, RepoError> {
111193×
        let dir = self.dir(name);
112193×
        if dir.exists() {
1130×
            return Err(RepoError::Exists(name.to_string()));
114193×
        }
115193×
        Ok(match id {
1168×
            Some(id) => Repo::init_bare_with_id(&dir, id)?,
117185×
            None => Repo::init_bare_unbound(&dir)?,
118
        })
119193×
    }
120
121
    /// Create an empty, unbound bare repository whose objects and
122
    /// operation log are kept in the store at `url` (oxygenhub#96); the
123
    /// store's credentials must already be known to the opener.
1240×
    pub fn create_on(&self, name: &RepoName, url: &str) -> Result<Repo, RepoError> {
1250×
        let dir = self.dir(name);
1260×
        if dir.exists() {
1270×
            return Err(RepoError::Exists(name.to_string()));
1280×
        }
1290×
        let made = (|| -> Result<Repo, RepoError> {
1300×
            drop(Repo::init_bare_on(&dir, [0; 16], Some(url))?);
131
            // Unbound as `create` makes them: oxy-core's marker file in
132
            // place of the id, so the first push gives the id.
1330×
            let oxy = dir.join(oxy_core::repo::OXY_DIR);
1340×
            std::fs::remove_file(oxy.join("id"))?;
1350×
            std::fs::write(oxy.join("unbound"), b"")?;
1360×
            Ok(Repo::open(&dir)?)
137
        })();
1380×
        if made.is_err() {
1390×
            let _ = std::fs::remove_dir_all(&dir);
1400×
        }
1410×
        made
1420×
    }
143
144
    /// Fork `src` into `dest` (hub spec §4): a new bare repository with its
145
    /// own id, the source's objects hard-linked where the file system allows
146
    /// (copied otherwise; objects are immutable, so sharing them is safe),
147
    /// and the source's bookmarks. The trust policy is not copied: the
148
    /// fork's owner sets their own. Runs under the source's writer lock, so
149
    /// no push lands while its files are linked.
1502×
    pub async fn fork(&self, src: &RepoName, dest: &RepoName) -> Result<(), RepoError> {
1512×
        let guard = self.write_guard(src).await;
1522×
        let (from, to) = (self.dir(src), self.dir(dest));
1532×
        if !self.exists(src) {
1540×
            return Err(RepoError::NotFound);
1552×
        }
1562×
        if to.exists() {
1570×
            return Err(RepoError::Exists(dest.to_string()));
1582×
        }
1592×
        if Repo::open(&from).is_ok_and(|r| r.config.storage_url.is_some()) {
1600×
            return Err(RepoError::Io(std::io::Error::other(
1610×
                "a repository kept in a bucket cannot be forked yet: clone it and push the clone to a new repository",
1620×
            )));
1632×
        }
1642×
        tokio::task::spawn_blocking(move || {
1652×
            let _guard = guard;
1662×
            let made = (|| -> Result<(), RepoError> {
1672×
                drop(Repo::init_bare(&to)?);
1684×
                let objects = |d: &Path| d.join(oxy_core::repo::OXY_DIR).join("objects");
1692×
                link_tree(&objects(&from), &objects(&to))?;
1702×
                let source = Repo::open(&from)?;
1712×
                let fork = Repo::open(&to)?;
1722×
                let existing: Vec<String> = fork.bookmarks()?.into_iter().map(|b| b.name).collect();
1732×
                for b in source.bookmarks()? {
1742×
                    if existing.contains(&b.name) {
1752×
                        fork.bookmark_set(&b.name, &b.heads)?;
176
                    } else {
1770×
                        fork.bookmark_create(&b.name, Some(&b.heads))?;
178
                    }
179
                }
1802×
                Ok(())
181
            })();
1822×
            if made.is_err() {
1830×
                let _ = std::fs::remove_dir_all(&to);
1842×
            }
1852×
            made
1862×
        })
1872×
        .await
1882×
        .map_err(|e| RepoError::Io(std::io::Error::other(e.to_string())))?
1892×
    }
190
191
    /// Move a repository to another name (a transfer, #108): the directory,
192
    /// and the lock the old name held.
1931×
    pub fn rename(&self, from: &RepoName, to: &RepoName) -> Result<(), RepoError> {
1941×
        let (src, dest) = (self.dir(from), self.dir(to));
1951×
        if !src.exists() {
1960×
            return Err(RepoError::NotFound);
1971×
        }
1981×
        if dest.exists() {
1990×
            return Err(RepoError::Exists(to.to_string()));
2001×
        }
2011×
        if let Some(parent) = dest.parent() {
2021×
            std::fs::create_dir_all(parent)?;
2030×
        }
2041×
        std::fs::rename(&src, &dest)?;
2051×
        let mut locks = self.locks.lock().unwrap();
2061×
        if let Some(l) = locks.remove(&from.to_string()) {
2071×
            locks.insert(to.to_string(), l);
2081×
        }
2091×
        Ok(())
2101×
    }
211
212
    /// Move a repository to the trash (hub spec §4.1); a job empties it later.
2136×
    pub fn trash(&self, name: &RepoName) -> Result<PathBuf, RepoError> {
2146×
        let dir = self.dir(name);
2156×
        if !dir.exists() {
2160×
            return Err(RepoError::NotFound);
2176×
        }
2186×
        let trash = self.root.parent().unwrap_or(&self.root).join("trash").join(&name.owner);
219
        // A drop's name is `drops/ID`: its own directory under the owner's.
2206×
        let dest = trash.join(format!("{}-{}", name.name, crate::db::now_ms()));
2216×
        std::fs::create_dir_all(dest.parent().unwrap_or(&trash))?;
2226×
        std::fs::rename(&dir, &dest)?;
2236×
        Ok(dest)
2246×
    }
225
2263958×
    fn lock(&self, name: &RepoName) -> Arc<Locks> {
2273958×
        self.locks.lock().unwrap().entry(name.to_string()).or_default().clone()
2283958×
    }
229
230
    /// Read while `f` runs on the blocking pool: never waiting for a
231
    /// writer, only for the repository being deleted or moved.
2322747×
    pub async fn read<T: Send + 'static>(
2332747×
        &self,
2342747×
        name: &RepoName,
2352747×
        f: impl FnOnce(&Repo, &StateCache) -> Result<T, RepoError> + Send + 'static,
2362747×
    ) -> Result<T, RepoError> {
2372747×
        let guard = self.read_guard(name).await;
2382747×
        let dir = self.dir(name);
2392747×
        let exists = self.exists(name);
2402747×
        let cache = self.cache.clone();
2412747×
        tokio::task::spawn_blocking(move || {
2422746×
            let _guard = guard;
2432746×
            if !exists {
2440×
                return Err(RepoError::NotFound);
2452746×
            }
2462746×
            let repo = Repo::open(&dir)?;
2472746×
            f(&repo, &cache)
2482746×
        })
2492747×
        .await
2502746×
        .map_err(|e| RepoError::Io(std::io::Error::other(e)))?
2512746×
    }
252
253
    /// Write while `f` runs on the blocking pool, after the writers ahead.
254100×
    pub async fn write<T: Send + 'static>(
255100×
        &self,
256100×
        name: &RepoName,
257100×
        f: impl FnOnce(&Repo, &StateCache) -> Result<T, RepoError> + Send + 'static,
258100×
    ) -> Result<T, RepoError> {
259100×
        let guard = self.write_guard(name).await;
260100×
        let dir = self.dir(name);
261100×
        let exists = self.exists(name);
262100×
        let cache = self.cache.clone();
263100×
        tokio::task::spawn_blocking(move || {
264100×
            let _guard = guard;
265100×
            if !exists {
2660×
                return Err(RepoError::NotFound);
267100×
            }
268100×
            let repo = Repo::open(&dir)?;
269100×
            f(&repo, &cache)
270100×
        })
271100×
        .await
272100×
        .map_err(|e| RepoError::Io(std::io::Error::other(e)))?
273100×
    }
274
275
    /// The writer's hold alone, for callers that open the repository
276
    /// themselves (the protocol server's push, a backup's last pass). The
277
    /// writer lock first, so a writer queued behind others holds nothing a
278
    /// delete would wait for.
279384×
    pub async fn write_guard(&self, name: &RepoName) -> WriteGuard {
280384×
        let locks = self.lock(name);
281384×
        let writer = locks.writer.clone().lock_owned().await;
282383×
        WriteGuard { _writer: writer, _life: locks.life.clone().read_owned().await }
283383×
    }
284
2853565×
    pub async fn read_guard(&self, name: &RepoName) -> ReadGuard {
2863565×
        self.lock(name).life.clone().read_owned().await
2873564×
    }
288
289
    /// Deleting or moving the repository: waits for every reader and
290
    /// writer, and holds off new ones.
2919×
    pub async fn exclusive_guard(&self, name: &RepoName) -> ExclusiveGuard {
2929×
        self.lock(name).life.clone().write_owned().await
2938×
    }
294
}
295
296
/// Hard-link every file under `from` into the same place under `to`,
297
/// copying when a link is not possible (another file system). Temporary
298
/// files of writers in progress are skipped.
29921×
fn link_tree(from: &Path, to: &Path) -> std::io::Result<()> {
30021×
    std::fs::create_dir_all(to)?;
30136×
    for e in std::fs::read_dir(from)? {
30236×
        let e = e?;
30336×
        let name = e.file_name();
30436×
        let n = name.to_string_lossy();
30536×
        if n.starts_with("tmp-") || n.starts_with("sigs-tmp-") || n.contains(".tmp") {
3060×
            continue;
30736×
        }
30836×
        let (src, dst) = (e.path(), to.join(&name));
30936×
        if e.file_type()?.is_dir() {
31019×
            link_tree(&src, &dst)?;
31117×
        } else if !dst.exists() && std::fs::hard_link(&src, &dst).is_err() {
3120×
            std::fs::copy(&src, &dst)?;
31317×
        }
314
    }
31521×
    Ok(())
31621×
}