OXYGENHUB
# c30444e35f2e oxygenhub / crates / hub / src / main.rs
14.5 KiB · Rust · last changed A stop no longer waits on blocking work past the drain 19 days ago source blame history raw
d23464c0 Andy McCall ↶ 1
//! `oxygenhub`: `init`, `serve`, `admin`, `check` (hub spec §17.3).
2
3
use clap::{Parser, Subcommand};
4
use hub_api::Visibility;
5
use oxygenhub::config::Config;
6
use oxygenhub::repos::RepoName;
7
use oxygenhub::{db, keys};
8
use std::path::PathBuf;
9
10
#[derive(Parser)]
11
#[command(name = "oxygenhub", version, about = "A forge for Oxygen repositories")]
12
struct Cli {
13
    /// The data directory (hub spec §3.2)
14
    #[arg(long, global = true, env = "OXYHUB_DATA", default_value = "data")]
15
    data: PathBuf,
16
    #[command(subcommand)]
17
    cmd: Cmd,
18
}
19
20
#[derive(Subcommand)]
21
enum Cmd {
22
    /// Create the data directory, keys, database and a default hub.toml
23
    Init {
24
        /// What links and clone URLs start with
25
        #[arg(long)]
26
        base_url: Option<String>,
27
        /// Generate a self-signed TLS certificate (developer edition)
28
        #[arg(long)]
29
        self_signed: bool,
30
    },
31
    /// Run the hub
32
    Serve,
33
    /// Offline administration
34
    Admin {
35
        #[command(subcommand)]
36
        cmd: AdminCmd,
37
    },
38
    /// Verify every repository's objects and signatures
39
    Check,
8a3334b7 Andy McCall ↶ 40
    /// Garbage-collect every repository now, as the nightly job does
41
    Gc,
d23464c0 Andy McCall ↶ 42
}
43
44
#[derive(Subcommand)]
45
enum AdminCmd {
46
    /// Repositories
47
    Repo {
48
        #[command(subcommand)]
49
        cmd: RepoCmd,
50
    },
19bb31ff Andy McCall ↶ 51
    /// Accounts
52
    User {
53
        #[command(subcommand)]
54
        cmd: UserCmd,
55
    },
d7d18246 Andy McCall ↶ 56
    /// Backups (docs/backups.md)
57
    Backup {
58
        #[command(subcommand)]
59
        cmd: BackupCmd,
60
    },
61
}
62
63
#[derive(Subcommand)]
64
enum BackupCmd {
65
    /// Back up a user's or organisation's repositories now, as its settings say
66
    Run { owner: String },
67
    /// Restore every repository of a whole run from a backup directory
68
    /// (the one holding LATEST, runs/ and blobs/) into this hub
69
    Restore {
70
        #[arg(long)]
71
        from: std::path::PathBuf,
72
        /// The run (runs/<RUN>); the latest whole one when not given
73
        #[arg(long)]
74
        run: Option<String>,
75
    },
d23464c0 Andy McCall ↶ 76
}
77
78
#[derive(Subcommand)]
19bb31ff Andy McCall ↶ 79
enum UserCmd {
80
    /// Create an account with a generated password, printed once
81
    Create {
82
        username: String,
83
        #[arg(long, default_value = "")]
84
        email: String,
85
        /// Make it a site administrator
86
        #[arg(long)]
87
        admin: bool,
88
    },
89
    /// Reset an account's password to a new generated one, printed once
90
    ResetPassword { username: String },
91
    /// List accounts
92
    List,
93
}
94
95
#[derive(Subcommand)]
d23464c0 Andy McCall ↶ 96
enum RepoCmd {
97
    /// Create an empty repository OWNER/NAME
98
    Create {
99
        slug: String,
100
        #[arg(long, default_value = "")]
101
        description: String,
102
        #[arg(long, default_value = "public")]
103
        visibility: String,
104
        /// Bind to this repository id now (32 hex characters, from `.oxy/id`);
105
        /// otherwise the first push binds it
106
        #[arg(long)]
107
        id: Option<String>,
108
    },
109
    /// List repositories
110
    List,
111
    /// Move a repository to the trash
112
    Delete { slug: String },
113
    /// Change the description, default bookmark or visibility
114
    Set {
115
        slug: String,
116
        #[arg(long)]
117
        description: Option<String>,
118
        #[arg(long)]
119
        default_bookmark: Option<String>,
120
        #[arg(long)]
121
        visibility: Option<String>,
122
    },
123
}
124
b2107275 Andy McCall ↶ 125
fn main() -> anyhow::Result<()> {
126
    tune_allocator();
127
    let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?;
128
    rt.spawn(return_freed_memory());
529c703d Andy McCall ↶ 129
    let out = rt.block_on(run());
130
    oxygenhub::shutdown::finish(rt);
131
    out
b2107275 Andy McCall ↶ 132
}
133
134
/// glibc's allocator gives each thread that allocates an arena of its own
135
/// and keeps what is freed in it: under the hub's worker threads, loaded
136
/// states that the cache had let go still held their pages, and the hub
137
/// grew to 12 GB with its state cache capped at 3 GiB (oxygenhub#117).
138
/// Two arenas, as oxy found for its own threads; MALLOC_ARENA_MAX in the
139
/// environment still has the last word. Called before any thread starts.
140
fn tune_allocator() {
141
    #[cfg(all(target_os = "linux", target_env = "gnu"))]
142
    if std::env::var_os("MALLOC_ARENA_MAX").is_none() {
143
        // SAFETY: mallopt is called once, before any other thread exists.
144
        unsafe {
145
            libc::mallopt(libc::M_ARENA_MAX, 2);
146
        }
147
    }
148
}
149
150
/// Hand freed memory back to the system every minute, so the hub's size
151
/// follows what it holds rather than its peak (oxygenhub#117).
152
async fn return_freed_memory() {
153
    #[cfg(all(target_os = "linux", target_env = "gnu"))]
154
    loop {
155
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
156
        // SAFETY: malloc_trim only releases free pages.
157
        let _ = tokio::task::spawn_blocking(|| unsafe { libc::malloc_trim(0) }).await;
158
    }
159
}
160
161
async fn run() -> anyhow::Result<()> {
d23464c0 Andy McCall ↶ 162
    tracing_subscriber::fmt()
163
        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env().add_directive("oxygenhub=info".parse()?))
164
        .init();
165
    let cli = Cli::parse();
166
    match cli.cmd {
167
        Cmd::Init { base_url, self_signed } => init(&cli.data, base_url, self_signed).await,
168
        Cmd::Serve => {
169
            let config = Config::load(&cli.data)?;
170
            let app = oxygenhub::build(config).await?;
171
            oxygenhub::serve(app).await
172
        }
173
        Cmd::Admin { cmd: AdminCmd::Repo { cmd } } => {
174
            let config = Config::load(&cli.data)?;
175
            let app = oxygenhub::build(config).await?;
176
            repo_admin(&app, cmd).await
d7d18246 Andy McCall ↶ 177
        }
178
        Cmd::Admin { cmd: AdminCmd::Backup { cmd } } => {
179
            let config = Config::load(&cli.data)?;
180
            let app = oxygenhub::build(config).await?;
181
            match cmd {
182
                BackupCmd::Run { owner } => {
183
                    let r = oxygenhub::backups::run(&app, &owner, "now").await?;
184
                    println!(
185
                        "{} {}: {} repositories, {} of {} files sent ({} KiB){}",
186
                        r.status,
187
                        r.at,
188
                        r.repos,
189
                        r.sent,
190
                        r.files,
191
                        r.bytes / 1024,
192
                        if r.detail.is_empty() { String::new() } else { format!(": {}", r.detail) }
193
                    );
194
                    if r.status == "failed" {
195
                        std::process::exit(1);
196
                    }
197
                    Ok(())
198
                }
199
                BackupCmd::Restore { from, run } => {
200
                    let dest: std::sync::Arc<dyn oxygenhub::backups::Destination> =
201
                        std::sync::Arc::new(oxygenhub::backups::Directory { root: from });
202
                    for d in oxygenhub::backups::restore_into_hub(&app, dest, run).await? {
203
                        println!("restored {d}");
204
                    }
205
                    Ok(())
206
                }
207
            }
d23464c0 Andy McCall ↶ 208
        }
19bb31ff Andy McCall ↶ 209
        Cmd::Admin { cmd: AdminCmd::User { cmd } } => {
210
            let config = Config::load(&cli.data)?;
211
            let app = oxygenhub::build(config).await?;
212
            user_admin(&app, cmd).await
213
        }
d23464c0 Andy McCall ↶ 214
        Cmd::Check => {
215
            let config = Config::load(&cli.data)?;
216
            let app = oxygenhub::build(config).await?;
217
            check(&app).await
218
        }
8a3334b7 Andy McCall ↶ 219
        Cmd::Gc => {
220
            let config = Config::load(&cli.data)?;
221
            let app = oxygenhub::build(config).await?;
222
            let done = oxygenhub::maintenance::gc_all(&app).await;
223
            for d in &done {
224
                println!("{d}");
225
            }
226
            if done.iter().any(|d| d.failed) {
227
                anyhow::bail!("a repository could not be garbage collected");
228
            }
229
            Ok(())
230
        }
d23464c0 Andy McCall ↶ 231
    }
232
}
233
234
async fn init(data: &PathBuf, base_url: Option<String>, self_signed: bool) -> anyhow::Result<()> {
235
    std::fs::create_dir_all(data)?;
236
    let toml_path = data.join("hub.toml");
237
    if !toml_path.exists() {
238
        let base = base_url.clone().unwrap_or_else(|| "http://127.0.0.1:7300".into());
8d2caf12 Andy McCall ↶ 239
        let mut text = String::from("# OxygenHub configuration (docs/spec.md section 17.1)\n\n[site]\nname = \"OxygenHub\"\n\n[server]\nbind = \"127.0.0.1:7300\"\n");
d23464c0 Andy McCall ↶ 240
        if self_signed {
241
            text.push_str("tls_bind = \"0.0.0.0:7343\"\n");
242
        }
243
        text.push_str(&format!("base_url = \"{base}\"\n\n[tls]\nself_signed = {self_signed}\n\n[auth]\n# Bearer tokens allowed to push until accounts exist.\npush_tokens = []\n"));
244
        std::fs::write(&toml_path, text)?;
245
        println!("wrote {}", toml_path.display());
246
    }
247
    let mut config = Config::load(data)?;
248
    if let Some(b) = base_url {
249
        config.server.base_url = b;
250
    }
251
    let app = oxygenhub::build(config).await?;
252
    println!("hub key {}", app.hub_key.key.to_hex());
253
    if (self_signed || app.config.tls.self_signed)
254
        && keys::ensure_self_signed(&app.config.tls_cert(), &app.config.tls_key(), app.config.base_url())?
255
    {
256
        println!("wrote self-signed certificate {}", app.config.tls_cert().display());
257
    }
258
    println!("data directory {} is ready", data.display());
259
    Ok(())
260
}
261
262
async fn repo_admin(app: &oxygenhub::Shared, cmd: RepoCmd) -> anyhow::Result<()> {
263
    match cmd {
264
        RepoCmd::Create { slug, description, visibility, id } => {
265
            let name = RepoName::parse(&slug).ok_or_else(|| anyhow::anyhow!("{slug:?} is not a valid owner/name"))?;
266
            let visibility = Visibility::parse(&visibility).ok_or_else(|| anyhow::anyhow!("visibility is public, internal or private"))?;
267
            let id = match id {
268
                Some(h) => {
269
                    let bytes = hex::decode(h.trim()).ok().and_then(|v| <[u8; 16]>::try_from(v).ok());
270
                    Some(bytes.ok_or_else(|| anyhow::anyhow!("--id takes 32 hex characters"))?)
271
                }
272
                None => None,
273
            };
274
            if db::repos::get(&app.db, &name.owner, &name.name).await?.is_some() {
275
                anyhow::bail!("{name} already exists");
276
            }
277
            app.repos.create(&name, id)?;
278
            db::repos::create(&app.db, &name.owner, &name.name, &description, visibility).await?;
279
            println!("created {name}: oxy clone {}/{name}", app.config.base_url());
280
        }
281
        RepoCmd::List => {
282
            for r in db::repos::list(&app.db).await? {
283
                println!("{:<40} {:<8} {}", r.slug(), r.visibility, r.description);
284
            }
285
        }
286
        RepoCmd::Delete { slug } => {
287
            let name = RepoName::parse(&slug).ok_or_else(|| anyhow::anyhow!("{slug:?} is not a valid owner/name"))?;
288
            let row = db::repos::get(&app.db, &name.owner, &name.name).await?.ok_or_else(|| anyhow::anyhow!("{name} does not exist"))?;
289
            let dest = app.repos.trash(&name)?;
290
            db::repos::delete(&app.db, row.id).await?;
291
            println!("moved {name} to {}", dest.display());
292
        }
293
        RepoCmd::Set { slug, description, default_bookmark, visibility } => {
294
            let name = RepoName::parse(&slug).ok_or_else(|| anyhow::anyhow!("{slug:?} is not a valid owner/name"))?;
295
            let row = db::repos::get(&app.db, &name.owner, &name.name).await?.ok_or_else(|| anyhow::anyhow!("{name} does not exist"))?;
296
            if let Some(d) = description {
297
                db::repos::set_description(&app.db, row.id, &d).await?;
298
            }
299
            if let Some(b) = default_bookmark {
300
                db::repos::set_default_bookmark(&app.db, row.id, &b).await?;
301
            }
302
            if let Some(v) = visibility {
303
                let v = Visibility::parse(&v).ok_or_else(|| anyhow::anyhow!("visibility is public, internal or private"))?;
304
                db::repos::set_visibility(&app.db, row.id, v).await?;
305
            }
306
            println!("updated {name}");
307
        }
308
    }
309
    Ok(())
310
}
311
19bb31ff Andy McCall ↶ 312
async fn user_admin(app: &oxygenhub::Shared, cmd: UserCmd) -> anyhow::Result<()> {
313
    use oxygenhub::auth;
314
    use oxygenhub::db::accounts;
315
    match cmd {
316
        UserCmd::Create { username, email, admin } => {
317
            if !oxygenhub::repos::names::valid_owner(&username) {
318
                anyhow::bail!("{username:?} is not a valid username");
319
            }
320
            if accounts::account_by_name(&app.db, &username).await?.is_some() || accounts::org_by_name(&app.db, &username).await?.is_some()
321
            {
322
                anyhow::bail!("{username} is taken");
323
            }
324
            let password = auth::random_hex(12);
325
            let hash = auth::hash_password(&password)?;
326
            accounts::create_account(&app.db, &username, &email, &hash, &auth::random_hex(16), admin).await?;
327
            accounts::audit(&app.db, None, "admin.user.create", &username, if admin { "administrator" } else { "" }).await?;
328
            println!("created {username}{}; one-time password: {password}", if admin { " (administrator)" } else { "" });
329
            println!("sign in at {}/login and change it under Settings", app.config.base_url());
330
        }
331
        UserCmd::ResetPassword { username } => {
332
            let account = accounts::account_by_name(&app.db, &username).await?.ok_or_else(|| anyhow::anyhow!("no such account"))?;
333
            let password = auth::random_hex(12);
334
            let hash = auth::hash_password(&password)?;
335
            accounts::set_password(&app.db, account.id, &hash, &auth::random_hex(16)).await?;
336
            accounts::audit(&app.db, None, "admin.user.reset_password", &username, "").await?;
337
            println!("new one-time password for {username}: {password}");
338
        }
339
        UserCmd::List => {
340
            for a in accounts::accounts(&app.db).await? {
341
                println!(
6a178055 Andy McCall ↶ 342
                    "{:<24} {:<32} {}{}{}",
19bb31ff Andy McCall ↶ 343
                    a.username,
344
                    a.email,
345
                    if a.is_admin { "admin " } else { "" },
6a178055 Andy McCall ↶ 346
                    if a.banned { "banned " } else { "" },
347
                    if a.disabled { "disabled" } else { "" }
19bb31ff Andy McCall ↶ 348
                );
349
            }
350
        }
351
    }
352
    Ok(())
353
}
354
d23464c0 Andy McCall ↶ 355
async fn check(app: &oxygenhub::Shared) -> anyhow::Result<()> {
356
    let mut problems = 0;
357
    for r in db::repos::list(&app.db).await? {
358
        let name = RepoName::new(&r.owner, &r.name).unwrap();
9ff27da5 Andy McCall ↶ 359
        let report = app.repos.read(&name, |r, cache| oxygenhub::repos::view::verification(r, cache)).await;
d23464c0 Andy McCall ↶ 360
        match report {
361
            Ok(v) => {
362
                println!(
363
                    "{:<40} {} changes, {} signed, {} sealed, {} unsigned, {} invalid",
364
                    name,
365
                    v.total,
366
                    v.signed,
367
                    v.sealed,
368
                    v.unsigned,
369
                    v.invalid.len()
370
                );
371
                problems += v.invalid.len();
372
            }
373
            Err(e) => {
374
                println!("{name:<40} ERROR {e}");
375
                problems += 1;
376
            }
377
        }
378
    }
379
    if problems > 0 {
380
        anyhow::bail!("{problems} problem(s)");
381
    }
382
    Ok(())
383
}