OXYGENHUB
◆ clear-waiting oxygenhub / crates / hub / src / main.rs
12.5 KiB · Rust · last changed Hub memory: two allocator arenas, freed memory returned, cache in /health 27 days ago source blame history raw
d23464c0 Andy McCall ↶ 1
//! `oxygenhub`: `init`, `serve`, `admin`, `check` (hub spec §17.3).
2
3
use clap::{Parser, Subcommand};
4
use hub_api::Visibility;
5
use oxygenhub::config::Config;
6
use oxygenhub::repos::RepoName;
7
use oxygenhub::{db, keys};
8
use std::path::PathBuf;
9
10
#[derive(Parser)]
11
#[command(name = "oxygenhub", version, about = "A forge for Oxygen repositories")]
12
struct Cli {
13
    /// The data directory (hub spec §3.2)
14
    #[arg(long, global = true, env = "OXYHUB_DATA", default_value = "data")]
15
    data: PathBuf,
16
    #[command(subcommand)]
17
    cmd: Cmd,
18
}
19
20
#[derive(Subcommand)]
21
enum Cmd {
22
    /// Create the data directory, keys, database and a default hub.toml
23
    Init {
24
        /// What links and clone URLs start with
25
        #[arg(long)]
26
        base_url: Option<String>,
27
        /// Generate a self-signed TLS certificate (developer edition)
28
        #[arg(long)]
29
        self_signed: bool,
30
    },
31
    /// Run the hub
32
    Serve,
33
    /// Offline administration
34
    Admin {
35
        #[command(subcommand)]
36
        cmd: AdminCmd,
37
    },
38
    /// Verify every repository's objects and signatures
39
    Check,
8a3334b7 Andy McCall ↶ 40
    /// Garbage-collect every repository now, as the nightly job does
41
    Gc,
d23464c0 Andy McCall ↶ 42
}
43
44
#[derive(Subcommand)]
45
enum AdminCmd {
46
    /// Repositories
47
    Repo {
48
        #[command(subcommand)]
49
        cmd: RepoCmd,
50
    },
19bb31ff Andy McCall ↶ 51
    /// Accounts
52
    User {
53
        #[command(subcommand)]
54
        cmd: UserCmd,
55
    },
d23464c0 Andy McCall ↶ 56
}
57
58
#[derive(Subcommand)]
19bb31ff Andy McCall ↶ 59
enum UserCmd {
60
    /// Create an account with a generated password, printed once
61
    Create {
62
        username: String,
63
        #[arg(long, default_value = "")]
64
        email: String,
65
        /// Make it a site administrator
66
        #[arg(long)]
67
        admin: bool,
68
    },
69
    /// Reset an account's password to a new generated one, printed once
70
    ResetPassword { username: String },
71
    /// List accounts
72
    List,
73
}
74
75
#[derive(Subcommand)]
d23464c0 Andy McCall ↶ 76
enum RepoCmd {
77
    /// Create an empty repository OWNER/NAME
78
    Create {
79
        slug: String,
80
        #[arg(long, default_value = "")]
81
        description: String,
82
        #[arg(long, default_value = "public")]
83
        visibility: String,
84
        /// Bind to this repository id now (32 hex characters, from `.oxy/id`);
85
        /// otherwise the first push binds it
86
        #[arg(long)]
87
        id: Option<String>,
88
    },
89
    /// List repositories
90
    List,
91
    /// Move a repository to the trash
92
    Delete { slug: String },
93
    /// Change the description, default bookmark or visibility
94
    Set {
95
        slug: String,
96
        #[arg(long)]
97
        description: Option<String>,
98
        #[arg(long)]
99
        default_bookmark: Option<String>,
100
        #[arg(long)]
101
        visibility: Option<String>,
102
    },
103
}
104
b2107275 Andy McCall ↶ 105
fn main() -> anyhow::Result<()> {
106
    tune_allocator();
107
    let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?;
108
    rt.spawn(return_freed_memory());
109
    rt.block_on(run())
110
}
111
112
/// glibc's allocator gives each thread that allocates an arena of its own
113
/// and keeps what is freed in it: under the hub's worker threads, loaded
114
/// states that the cache had let go still held their pages, and the hub
115
/// grew to 12 GB with its state cache capped at 3 GiB (oxygenhub#117).
116
/// Two arenas, as oxy found for its own threads; MALLOC_ARENA_MAX in the
117
/// environment still has the last word. Called before any thread starts.
118
fn tune_allocator() {
119
    #[cfg(all(target_os = "linux", target_env = "gnu"))]
120
    if std::env::var_os("MALLOC_ARENA_MAX").is_none() {
121
        // SAFETY: mallopt is called once, before any other thread exists.
122
        unsafe {
123
            libc::mallopt(libc::M_ARENA_MAX, 2);
124
        }
125
    }
126
}
127
128
/// Hand freed memory back to the system every minute, so the hub's size
129
/// follows what it holds rather than its peak (oxygenhub#117).
130
async fn return_freed_memory() {
131
    #[cfg(all(target_os = "linux", target_env = "gnu"))]
132
    loop {
133
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
134
        // SAFETY: malloc_trim only releases free pages.
135
        let _ = tokio::task::spawn_blocking(|| unsafe { libc::malloc_trim(0) }).await;
136
    }
137
}
138
139
async fn run() -> anyhow::Result<()> {
d23464c0 Andy McCall ↶ 140
    tracing_subscriber::fmt()
141
        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env().add_directive("oxygenhub=info".parse()?))
142
        .init();
143
    let cli = Cli::parse();
144
    match cli.cmd {
145
        Cmd::Init { base_url, self_signed } => init(&cli.data, base_url, self_signed).await,
146
        Cmd::Serve => {
147
            let config = Config::load(&cli.data)?;
148
            let app = oxygenhub::build(config).await?;
149
            oxygenhub::serve(app).await
150
        }
151
        Cmd::Admin { cmd: AdminCmd::Repo { cmd } } => {
152
            let config = Config::load(&cli.data)?;
153
            let app = oxygenhub::build(config).await?;
154
            repo_admin(&app, cmd).await
155
        }
19bb31ff Andy McCall ↶ 156
        Cmd::Admin { cmd: AdminCmd::User { cmd } } => {
157
            let config = Config::load(&cli.data)?;
158
            let app = oxygenhub::build(config).await?;
159
            user_admin(&app, cmd).await
160
        }
d23464c0 Andy McCall ↶ 161
        Cmd::Check => {
162
            let config = Config::load(&cli.data)?;
163
            let app = oxygenhub::build(config).await?;
164
            check(&app).await
165
        }
8a3334b7 Andy McCall ↶ 166
        Cmd::Gc => {
167
            let config = Config::load(&cli.data)?;
168
            let app = oxygenhub::build(config).await?;
169
            let done = oxygenhub::maintenance::gc_all(&app).await;
170
            for d in &done {
171
                println!("{d}");
172
            }
173
            if done.iter().any(|d| d.failed) {
174
                anyhow::bail!("a repository could not be garbage collected");
175
            }
176
            Ok(())
177
        }
d23464c0 Andy McCall ↶ 178
    }
179
}
180
181
async fn init(data: &PathBuf, base_url: Option<String>, self_signed: bool) -> anyhow::Result<()> {
182
    std::fs::create_dir_all(data)?;
183
    let toml_path = data.join("hub.toml");
184
    if !toml_path.exists() {
185
        let base = base_url.clone().unwrap_or_else(|| "http://127.0.0.1:7300".into());
8d2caf12 Andy McCall ↶ 186
        let mut text = String::from("# OxygenHub configuration (docs/spec.md section 17.1)\n\n[site]\nname = \"OxygenHub\"\n\n[server]\nbind = \"127.0.0.1:7300\"\n");
d23464c0 Andy McCall ↶ 187
        if self_signed {
188
            text.push_str("tls_bind = \"0.0.0.0:7343\"\n");
189
        }
190
        text.push_str(&format!("base_url = \"{base}\"\n\n[tls]\nself_signed = {self_signed}\n\n[auth]\n# Bearer tokens allowed to push until accounts exist.\npush_tokens = []\n"));
191
        std::fs::write(&toml_path, text)?;
192
        println!("wrote {}", toml_path.display());
193
    }
194
    let mut config = Config::load(data)?;
195
    if let Some(b) = base_url {
196
        config.server.base_url = b;
197
    }
198
    let app = oxygenhub::build(config).await?;
199
    println!("hub key {}", app.hub_key.key.to_hex());
200
    if (self_signed || app.config.tls.self_signed)
201
        && keys::ensure_self_signed(&app.config.tls_cert(), &app.config.tls_key(), app.config.base_url())?
202
    {
203
        println!("wrote self-signed certificate {}", app.config.tls_cert().display());
204
    }
205
    println!("data directory {} is ready", data.display());
206
    Ok(())
207
}
208
209
async fn repo_admin(app: &oxygenhub::Shared, cmd: RepoCmd) -> anyhow::Result<()> {
210
    match cmd {
211
        RepoCmd::Create { slug, description, visibility, id } => {
212
            let name = RepoName::parse(&slug).ok_or_else(|| anyhow::anyhow!("{slug:?} is not a valid owner/name"))?;
213
            let visibility = Visibility::parse(&visibility).ok_or_else(|| anyhow::anyhow!("visibility is public, internal or private"))?;
214
            let id = match id {
215
                Some(h) => {
216
                    let bytes = hex::decode(h.trim()).ok().and_then(|v| <[u8; 16]>::try_from(v).ok());
217
                    Some(bytes.ok_or_else(|| anyhow::anyhow!("--id takes 32 hex characters"))?)
218
                }
219
                None => None,
220
            };
221
            if db::repos::get(&app.db, &name.owner, &name.name).await?.is_some() {
222
                anyhow::bail!("{name} already exists");
223
            }
224
            app.repos.create(&name, id)?;
225
            db::repos::create(&app.db, &name.owner, &name.name, &description, visibility).await?;
226
            println!("created {name}: oxy clone {}/{name}", app.config.base_url());
227
        }
228
        RepoCmd::List => {
229
            for r in db::repos::list(&app.db).await? {
230
                println!("{:<40} {:<8} {}", r.slug(), r.visibility, r.description);
231
            }
232
        }
233
        RepoCmd::Delete { slug } => {
234
            let name = RepoName::parse(&slug).ok_or_else(|| anyhow::anyhow!("{slug:?} is not a valid owner/name"))?;
235
            let row = db::repos::get(&app.db, &name.owner, &name.name).await?.ok_or_else(|| anyhow::anyhow!("{name} does not exist"))?;
236
            let dest = app.repos.trash(&name)?;
237
            db::repos::delete(&app.db, row.id).await?;
238
            println!("moved {name} to {}", dest.display());
239
        }
240
        RepoCmd::Set { slug, description, default_bookmark, visibility } => {
241
            let name = RepoName::parse(&slug).ok_or_else(|| anyhow::anyhow!("{slug:?} is not a valid owner/name"))?;
242
            let row = db::repos::get(&app.db, &name.owner, &name.name).await?.ok_or_else(|| anyhow::anyhow!("{name} does not exist"))?;
243
            if let Some(d) = description {
244
                db::repos::set_description(&app.db, row.id, &d).await?;
245
            }
246
            if let Some(b) = default_bookmark {
247
                db::repos::set_default_bookmark(&app.db, row.id, &b).await?;
248
            }
249
            if let Some(v) = visibility {
250
                let v = Visibility::parse(&v).ok_or_else(|| anyhow::anyhow!("visibility is public, internal or private"))?;
251
                db::repos::set_visibility(&app.db, row.id, v).await?;
252
            }
253
            println!("updated {name}");
254
        }
255
    }
256
    Ok(())
257
}
258
19bb31ff Andy McCall ↶ 259
async fn user_admin(app: &oxygenhub::Shared, cmd: UserCmd) -> anyhow::Result<()> {
260
    use oxygenhub::auth;
261
    use oxygenhub::db::accounts;
262
    match cmd {
263
        UserCmd::Create { username, email, admin } => {
264
            if !oxygenhub::repos::names::valid_owner(&username) {
265
                anyhow::bail!("{username:?} is not a valid username");
266
            }
267
            if accounts::account_by_name(&app.db, &username).await?.is_some() || accounts::org_by_name(&app.db, &username).await?.is_some()
268
            {
269
                anyhow::bail!("{username} is taken");
270
            }
271
            let password = auth::random_hex(12);
272
            let hash = auth::hash_password(&password)?;
273
            accounts::create_account(&app.db, &username, &email, &hash, &auth::random_hex(16), admin).await?;
274
            accounts::audit(&app.db, None, "admin.user.create", &username, if admin { "administrator" } else { "" }).await?;
275
            println!("created {username}{}; one-time password: {password}", if admin { " (administrator)" } else { "" });
276
            println!("sign in at {}/login and change it under Settings", app.config.base_url());
277
        }
278
        UserCmd::ResetPassword { username } => {
279
            let account = accounts::account_by_name(&app.db, &username).await?.ok_or_else(|| anyhow::anyhow!("no such account"))?;
280
            let password = auth::random_hex(12);
281
            let hash = auth::hash_password(&password)?;
282
            accounts::set_password(&app.db, account.id, &hash, &auth::random_hex(16)).await?;
283
            accounts::audit(&app.db, None, "admin.user.reset_password", &username, "").await?;
284
            println!("new one-time password for {username}: {password}");
285
        }
286
        UserCmd::List => {
287
            for a in accounts::accounts(&app.db).await? {
288
                println!(
6a178055 Andy McCall ↶ 289
                    "{:<24} {:<32} {}{}{}",
19bb31ff Andy McCall ↶ 290
                    a.username,
291
                    a.email,
292
                    if a.is_admin { "admin " } else { "" },
6a178055 Andy McCall ↶ 293
                    if a.banned { "banned " } else { "" },
294
                    if a.disabled { "disabled" } else { "" }
19bb31ff Andy McCall ↶ 295
                );
296
            }
297
        }
298
    }
299
    Ok(())
300
}
301
d23464c0 Andy McCall ↶ 302
async fn check(app: &oxygenhub::Shared) -> anyhow::Result<()> {
303
    let mut problems = 0;
304
    for r in db::repos::list(&app.db).await? {
305
        let name = RepoName::new(&r.owner, &r.name).unwrap();
e1078e99 Andy McCall ↶ 306
        let report = app.repos.read(&name, oxygenhub::repos::view::verification).await;
d23464c0 Andy McCall ↶ 307
        match report {
308
            Ok(v) => {
309
                println!(
310
                    "{:<40} {} changes, {} signed, {} sealed, {} unsigned, {} invalid",
311
                    name,
312
                    v.total,
313
                    v.signed,
314
                    v.sealed,
315
                    v.unsigned,
316
                    v.invalid.len()
317
                );
318
                problems += v.invalid.len();
319
            }
320
            Err(e) => {
321
                println!("{name:<40} ERROR {e}");
322
                problems += 1;
323
            }
324
        }
325
    }
326
    if problems > 0 {
327
        anyhow::bail!("{problems} problem(s)");
328
    }
329
    Ok(())
330
}