OXYGENHUB
# b210727578e8 oxygenhub / crates / hub / src / main.rs
12.0 KiB · Rust · last changed Hub memory: two allocator arenas, freed memory returned, cache in /health 28 days ago source blame history raw
1
//! `oxygenhub`: `init`, `serve`, `admin`, `check` (hub spec §17.3).
2
3
use clap::{Parser, Subcommand};
4
use hub_api::Visibility;
5
use oxygenhub::config::Config;
6
use oxygenhub::repos::RepoName;
7
use oxygenhub::{db, keys};
8
use std::path::PathBuf;
9
10
#[derive(Parser)]
11
#[command(name = "oxygenhub", version, about = "A forge for Oxygen repositories")]
12
struct Cli {
13
    /// The data directory (hub spec §3.2)
14
    #[arg(long, global = true, env = "OXYHUB_DATA", default_value = "data")]
15
    data: PathBuf,
16
    #[command(subcommand)]
17
    cmd: Cmd,
18
}
19
20
#[derive(Subcommand)]
21
enum Cmd {
22
    /// Create the data directory, keys, database and a default hub.toml
23
    Init {
24
        /// What links and clone URLs start with
25
        #[arg(long)]
26
        base_url: Option<String>,
27
        /// Generate a self-signed TLS certificate (developer edition)
28
        #[arg(long)]
29
        self_signed: bool,
30
    },
31
    /// Run the hub
32
    Serve,
33
    /// Offline administration
34
    Admin {
35
        #[command(subcommand)]
36
        cmd: AdminCmd,
37
    },
38
    /// Verify every repository's objects and signatures
39
    Check,
40
}
41
42
#[derive(Subcommand)]
43
enum AdminCmd {
44
    /// Repositories
45
    Repo {
46
        #[command(subcommand)]
47
        cmd: RepoCmd,
48
    },
49
    /// Accounts
50
    User {
51
        #[command(subcommand)]
52
        cmd: UserCmd,
53
    },
54
}
55
56
#[derive(Subcommand)]
57
enum UserCmd {
58
    /// Create an account with a generated password, printed once
59
    Create {
60
        username: String,
61
        #[arg(long, default_value = "")]
62
        email: String,
63
        /// Make it a site administrator
64
        #[arg(long)]
65
        admin: bool,
66
    },
67
    /// Reset an account's password to a new generated one, printed once
68
    ResetPassword { username: String },
69
    /// List accounts
70
    List,
71
}
72
73
#[derive(Subcommand)]
74
enum RepoCmd {
75
    /// Create an empty repository OWNER/NAME
76
    Create {
77
        slug: String,
78
        #[arg(long, default_value = "")]
79
        description: String,
80
        #[arg(long, default_value = "public")]
81
        visibility: String,
82
        /// Bind to this repository id now (32 hex characters, from `.oxy/id`);
83
        /// otherwise the first push binds it
84
        #[arg(long)]
85
        id: Option<String>,
86
    },
87
    /// List repositories
88
    List,
89
    /// Move a repository to the trash
90
    Delete { slug: String },
91
    /// Change the description, default bookmark or visibility
92
    Set {
93
        slug: String,
94
        #[arg(long)]
95
        description: Option<String>,
96
        #[arg(long)]
97
        default_bookmark: Option<String>,
98
        #[arg(long)]
99
        visibility: Option<String>,
100
    },
101
}
102
103
fn main() -> anyhow::Result<()> {
104
    tune_allocator();
105
    let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?;
106
    rt.spawn(return_freed_memory());
107
    rt.block_on(run())
108
}
109
110
/// glibc's allocator gives each thread that allocates an arena of its own
111
/// and keeps what is freed in it: under the hub's worker threads, loaded
112
/// states that the cache had let go still held their pages, and the hub
113
/// grew to 12 GB with its state cache capped at 3 GiB (oxygenhub#117).
114
/// Two arenas, as oxy found for its own threads; MALLOC_ARENA_MAX in the
115
/// environment still has the last word. Called before any thread starts.
116
fn tune_allocator() {
117
    #[cfg(all(target_os = "linux", target_env = "gnu"))]
118
    if std::env::var_os("MALLOC_ARENA_MAX").is_none() {
119
        // SAFETY: mallopt is called once, before any other thread exists.
120
        unsafe {
121
            libc::mallopt(libc::M_ARENA_MAX, 2);
122
        }
123
    }
124
}
125
126
/// Hand freed memory back to the system every minute, so the hub's size
127
/// follows what it holds rather than its peak (oxygenhub#117).
128
async fn return_freed_memory() {
129
    #[cfg(all(target_os = "linux", target_env = "gnu"))]
130
    loop {
131
        tokio::time::sleep(std::time::Duration::from_secs(60)).await;
132
        // SAFETY: malloc_trim only releases free pages.
133
        let _ = tokio::task::spawn_blocking(|| unsafe { libc::malloc_trim(0) }).await;
134
    }
135
}
136
137
async fn run() -> anyhow::Result<()> {
138
    tracing_subscriber::fmt()
139
        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env().add_directive("oxygenhub=info".parse()?))
140
        .init();
141
    let cli = Cli::parse();
142
    match cli.cmd {
143
        Cmd::Init { base_url, self_signed } => init(&cli.data, base_url, self_signed).await,
144
        Cmd::Serve => {
145
            let config = Config::load(&cli.data)?;
146
            let app = oxygenhub::build(config).await?;
147
            oxygenhub::serve(app).await
148
        }
149
        Cmd::Admin { cmd: AdminCmd::Repo { cmd } } => {
150
            let config = Config::load(&cli.data)?;
151
            let app = oxygenhub::build(config).await?;
152
            repo_admin(&app, cmd).await
153
        }
154
        Cmd::Admin { cmd: AdminCmd::User { cmd } } => {
155
            let config = Config::load(&cli.data)?;
156
            let app = oxygenhub::build(config).await?;
157
            user_admin(&app, cmd).await
158
        }
159
        Cmd::Check => {
160
            let config = Config::load(&cli.data)?;
161
            let app = oxygenhub::build(config).await?;
162
            check(&app).await
163
        }
164
    }
165
}
166
167
async fn init(data: &PathBuf, base_url: Option<String>, self_signed: bool) -> anyhow::Result<()> {
168
    std::fs::create_dir_all(data)?;
169
    let toml_path = data.join("hub.toml");
170
    if !toml_path.exists() {
171
        let base = base_url.clone().unwrap_or_else(|| "http://127.0.0.1:7300".into());
172
        let mut text = String::from("# OxygenHub configuration (docs/spec.md section 17.1)\n\n[site]\nname = \"OxygenHub\"\n\n[server]\nbind = \"127.0.0.1:7300\"\n");
173
        if self_signed {
174
            text.push_str("tls_bind = \"0.0.0.0:7343\"\n");
175
        }
176
        text.push_str(&format!("base_url = \"{base}\"\n\n[tls]\nself_signed = {self_signed}\n\n[auth]\n# Bearer tokens allowed to push until accounts exist.\npush_tokens = []\n"));
177
        std::fs::write(&toml_path, text)?;
178
        println!("wrote {}", toml_path.display());
179
    }
180
    let mut config = Config::load(data)?;
181
    if let Some(b) = base_url {
182
        config.server.base_url = b;
183
    }
184
    let app = oxygenhub::build(config).await?;
185
    println!("hub key {}", app.hub_key.key.to_hex());
186
    if (self_signed || app.config.tls.self_signed)
187
        && keys::ensure_self_signed(&app.config.tls_cert(), &app.config.tls_key(), app.config.base_url())?
188
    {
189
        println!("wrote self-signed certificate {}", app.config.tls_cert().display());
190
    }
191
    println!("data directory {} is ready", data.display());
192
    Ok(())
193
}
194
195
async fn repo_admin(app: &oxygenhub::Shared, cmd: RepoCmd) -> anyhow::Result<()> {
196
    match cmd {
197
        RepoCmd::Create { slug, description, visibility, id } => {
198
            let name = RepoName::parse(&slug).ok_or_else(|| anyhow::anyhow!("{slug:?} is not a valid owner/name"))?;
199
            let visibility = Visibility::parse(&visibility).ok_or_else(|| anyhow::anyhow!("visibility is public, internal or private"))?;
200
            let id = match id {
201
                Some(h) => {
202
                    let bytes = hex::decode(h.trim()).ok().and_then(|v| <[u8; 16]>::try_from(v).ok());
203
                    Some(bytes.ok_or_else(|| anyhow::anyhow!("--id takes 32 hex characters"))?)
204
                }
205
                None => None,
206
            };
207
            if db::repos::get(&app.db, &name.owner, &name.name).await?.is_some() {
208
                anyhow::bail!("{name} already exists");
209
            }
210
            app.repos.create(&name, id)?;
211
            db::repos::create(&app.db, &name.owner, &name.name, &description, visibility).await?;
212
            println!("created {name}: oxy clone {}/{name}", app.config.base_url());
213
        }
214
        RepoCmd::List => {
215
            for r in db::repos::list(&app.db).await? {
216
                println!("{:<40} {:<8} {}", r.slug(), r.visibility, r.description);
217
            }
218
        }
219
        RepoCmd::Delete { slug } => {
220
            let name = RepoName::parse(&slug).ok_or_else(|| anyhow::anyhow!("{slug:?} is not a valid owner/name"))?;
221
            let row = db::repos::get(&app.db, &name.owner, &name.name).await?.ok_or_else(|| anyhow::anyhow!("{name} does not exist"))?;
222
            let dest = app.repos.trash(&name)?;
223
            db::repos::delete(&app.db, row.id).await?;
224
            println!("moved {name} to {}", dest.display());
225
        }
226
        RepoCmd::Set { slug, description, default_bookmark, visibility } => {
227
            let name = RepoName::parse(&slug).ok_or_else(|| anyhow::anyhow!("{slug:?} is not a valid owner/name"))?;
228
            let row = db::repos::get(&app.db, &name.owner, &name.name).await?.ok_or_else(|| anyhow::anyhow!("{name} does not exist"))?;
229
            if let Some(d) = description {
230
                db::repos::set_description(&app.db, row.id, &d).await?;
231
            }
232
            if let Some(b) = default_bookmark {
233
                db::repos::set_default_bookmark(&app.db, row.id, &b).await?;
234
            }
235
            if let Some(v) = visibility {
236
                let v = Visibility::parse(&v).ok_or_else(|| anyhow::anyhow!("visibility is public, internal or private"))?;
237
                db::repos::set_visibility(&app.db, row.id, v).await?;
238
            }
239
            println!("updated {name}");
240
        }
241
    }
242
    Ok(())
243
}
244
245
async fn user_admin(app: &oxygenhub::Shared, cmd: UserCmd) -> anyhow::Result<()> {
246
    use oxygenhub::auth;
247
    use oxygenhub::db::accounts;
248
    match cmd {
249
        UserCmd::Create { username, email, admin } => {
250
            if !oxygenhub::repos::names::valid_owner(&username) {
251
                anyhow::bail!("{username:?} is not a valid username");
252
            }
253
            if accounts::account_by_name(&app.db, &username).await?.is_some() || accounts::org_by_name(&app.db, &username).await?.is_some()
254
            {
255
                anyhow::bail!("{username} is taken");
256
            }
257
            let password = auth::random_hex(12);
258
            let hash = auth::hash_password(&password)?;
259
            accounts::create_account(&app.db, &username, &email, &hash, &auth::random_hex(16), admin).await?;
260
            accounts::audit(&app.db, None, "admin.user.create", &username, if admin { "administrator" } else { "" }).await?;
261
            println!("created {username}{}; one-time password: {password}", if admin { " (administrator)" } else { "" });
262
            println!("sign in at {}/login and change it under Settings", app.config.base_url());
263
        }
264
        UserCmd::ResetPassword { username } => {
265
            let account = accounts::account_by_name(&app.db, &username).await?.ok_or_else(|| anyhow::anyhow!("no such account"))?;
266
            let password = auth::random_hex(12);
267
            let hash = auth::hash_password(&password)?;
268
            accounts::set_password(&app.db, account.id, &hash, &auth::random_hex(16)).await?;
269
            accounts::audit(&app.db, None, "admin.user.reset_password", &username, "").await?;
270
            println!("new one-time password for {username}: {password}");
271
        }
272
        UserCmd::List => {
273
            for a in accounts::accounts(&app.db).await? {
274
                println!(
275
                    "{:<24} {:<32} {}{}{}",
276
                    a.username,
277
                    a.email,
278
                    if a.is_admin { "admin " } else { "" },
279
                    if a.banned { "banned " } else { "" },
280
                    if a.disabled { "disabled" } else { "" }
281
                );
282
            }
283
        }
284
    }
285
    Ok(())
286
}
287
288
async fn check(app: &oxygenhub::Shared) -> anyhow::Result<()> {
289
    let mut problems = 0;
290
    for r in db::repos::list(&app.db).await? {
291
        let name = RepoName::new(&r.owner, &r.name).unwrap();
292
        let report = app.repos.read(&name, |r, cache| oxygenhub::repos::view::verification(r, cache)).await;
293
        match report {
294
            Ok(v) => {
295
                println!(
296
                    "{:<40} {} changes, {} signed, {} sealed, {} unsigned, {} invalid",
297
                    name,
298
                    v.total,
299
                    v.signed,
300
                    v.sealed,
301
                    v.unsigned,
302
                    v.invalid.len()
303
                );
304
                problems += v.invalid.len();
305
            }
306
            Err(e) => {
307
                println!("{name:<40} ERROR {e}");
308
                problems += 1;
309
            }
310
        }
311
    }
312
    if problems > 0 {
313
        anyhow::bail!("{problems} problem(s)");
314
    }
315
    Ok(())
316
}