d7d18246 Andy McCall 22 days ago ↶
1
2
3
4
5
6
use oxy_core:: repo:: Repo;
7
use oxygenhub:: backups:: { self , Destination, Directory, Setting} ;
8
use oxygenhub:: config:: Config;
9
use oxygenhub:: repos:: RepoName;
10
use oxygenhub:: { db, Shared} ;
11
use std:: path:: Path;
12
use std:: sync:: Arc;
13
14
fn hermetic ( ) {
15
static ONCE : std:: sync:: Once = std:: sync:: Once:: new( ) ;
16
static DIR : std:: sync:: OnceLock< tempfile:: TempDir> = std:: sync:: OnceLock:: new( ) ;
17
ONCE .call_once ( | | {
18
let d = tempfile:: tempdir( ) .unwrap ( ) ;
19
std:: env:: set_var( " OXY_HOME" , d.path ( ) ) ;
20
DIR .set ( d) .ok ( ) ;
21
} ) ;
22
}
23
24
struct Hub {
25
app : Shared,
26
base : String,
27
_dir : tempfile:: TempDir,
28
}
29
30
async fn hub ( backups : & Path) -> Hub {
35fea423 Andy McCall 22 days ago ↶
31
hub_with ( backups, None ) .await
32
}
33
34
async fn hub_with ( backups : & Path, google : Option< & str > ) -> Hub {
d7d18246 Andy McCall 22 days ago ↶
35
hermetic ( ) ;
36
let dir = tempfile:: tempdir( ) .unwrap ( ) ;
37
let mut config = Config:: load( dir.path ( ) ) .unwrap ( ) ;
c8684d44 Andy McCall 19 days ago
38
39
config.registration.setup_code = " test-setup" .into ( ) ;
d7d18246 Andy McCall 22 days ago ↶
40
config.auth.push_tokens = vec! [ " site" .into ( ) ] ;
41
config.backups.directories = vec! [ backups.to_path_buf ( ) ] ;
35fea423 Andy McCall 22 days ago ↶
42
if let Some ( g) = google {
43
config.backups.sts_url = g.to_string ( ) ;
44
config.backups.storage_url = g.to_string ( ) ;
45
}
d7d18246 Andy McCall 22 days ago ↶
46
let listener = tokio:: net:: TcpListener:: bind( " 127.0.0.1:0" ) .await.unwrap ( ) ;
47
let port = listener.local_addr ( ) .unwrap ( ) .port ( ) ;
48
config.server.base_url = format! ( " http://127.0.0.1:{port} " ) ;
49
let app = oxygenhub:: build( config) .await.unwrap ( ) ;
50
let router = oxygenhub:: router( app.clone ( ) ) ;
51
tokio:: spawn( async move { axum:: serve( listener, router) .await.unwrap ( ) } ) ;
52
Hub { app, base: format! ( " http://127.0.0.1:{port} " ) , _dir: dir }
53
}
54
55
fn write ( root : & Path, rel : & str , content : & str ) {
56
let p = root.join ( rel) ;
57
std:: fs:: create_dir_all( p.parent ( ) .unwrap ( ) ) .unwrap ( ) ;
58
std:: fs:: write( p, content) .unwrap ( ) ;
59
}
60
61
async fn create ( app : & Shared, owner : & str , name : & str , visibility : hub_api:: Visibility) {
62
app.repos.create ( & RepoName:: parse( & format! ( " {owner} /{name} " ) ) .unwrap ( ) , None ) .unwrap ( ) ;
63
db:: repos:: create( & app.db, owner, name, " a demo" , visibility) .await.unwrap ( ) ;
64
}
65
66
fn push ( url : & str , work : & Path, files : & [( &str , &str ) ]) {
67
let mut c = Repo:: init( work) .unwrap ( ) ;
68
c.config.set ( " remote.origin.url" , url) .unwrap ( ) ;
69
c.config.set ( " remote.origin.token" , " site" ) .unwrap ( ) ;
70
c.save_config ( ) .unwrap ( ) ;
71
for ( f, t) in files {
72
write ( work, f, t) ;
73
}
74
c.snapshot ( " t" ) .unwrap ( ) ;
75
c.commit ( " Some work" , & [ ] ) .unwrap ( ) ;
76
assert! ( c.push ( " origin" , false ) .unwrap ( ) .iter ( ) .all ( | r | r.ok ) ) ;
77
}
78
79
80
fn files ( dir : & Path) -> Vec< ( String, Vec< u8 > ) > {
81
let mut out = Vec :: new( ) ;
82
let mut stack = vec! [ dir.to_path_buf ( ) ] ;
83
while let Some ( d) = stack.pop ( ) {
84
for e in std:: fs:: read_dir( & d) .unwrap ( ) .flatten ( ) {
85
let p = e.path ( ) ;
86
if p.is_dir ( ) {
87
stack.push ( p) ;
88
} else {
89
out.push ( ( p.strip_prefix ( dir) .unwrap ( ) .to_string_lossy ( ) .into_owned ( ) , std:: fs:: read( & p) .unwrap ( ) ) ) ;
90
}
91
}
92
}
93
out.sort ( ) ;
94
out
95
}
96
97
# [ tokio ::test ( flavor = " multi_thread" ) ]
98
async fn a_backup_restores_the_same_repository_and_sends_only_what_changed ( ) {
99
let store = tempfile:: tempdir( ) .unwrap ( ) ;
100
let h = hub ( store.path ( ) ) .await;
101
db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
102
db:: accounts:: create_account( & h.app.db, " bob" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
103
create ( & h.app, " andy" , " demo" , hub_api:: Visibility:: Private) .await;
104
create ( & h.app, " bob" , " other" , hub_api:: Visibility:: Public) .await;
105
let work = tempfile:: tempdir( ) .unwrap ( ) ;
106
push ( & format! ( " {} /andy/demo" , h.base) , work.path ( ) , & [ ( " README.md" , " hello\n " ) , ( " src/main.rs" , " fn main() {}\n " ) ] ) ;
107
let other = tempfile:: tempdir( ) .unwrap ( ) ;
108
push ( & format! ( " {} /bob/other" , h.base) , other.path ( ) , & [ ( " x.txt" , " bob's\n " ) ] ) ;
109
110
111
let target = store.path ( ) .display ( ) .to_string ( ) ;
112
let mut s = Setting:: off( " andy" , " user" ) ;
113
s.every = " hourly" .into ( ) ;
114
s.target = target.clone ( ) ;
115
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
116
let first = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
117
assert_eq! ( ( first.status.as_str ( ) , first.repos) , ( " ok" , 1 ) , " {}" , first.detail) ;
118
assert! ( first.sent > 0 & & first.bytes > 0 ) ;
119
let dest = Directory { root: store.path ( ) .join ( " andy" ) } ;
120
assert! ( dest.exists ( & format! ( " runs/{} /DONE" , first.at) ) .unwrap ( ) ) ;
121
let records: serde_json:: Value = serde_json:: from_slice( & dest.get ( & format! ( " runs/{} /records/demo.json" , first.at) ) .unwrap ( ) ) .unwrap ( ) ;
122
assert_eq! ( records[ " repositories" ] [ 0 ] [ " visibility" ] , " private" ) ;
123
let text = String :: from_utf8_lossy( & dest.get ( & format! ( " runs/{} /records/demo.json" , first.at) ) .unwrap ( ) ) .into_owned ( ) ;
124
assert! ( ! text.contains ( " bob/other" ) & & ! text.contains ( " \" other\" " ) , " nothing of another owner" ) ;
125
assert! ( ! store.path ( ) .join ( " bob" ) .exists ( ) , " bob set up no backup" ) ;
126
127
128
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
129
let second = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
130
assert_eq! ( ( second.status.as_str ( ) , second.sent) , ( " ok" , 0 ) , " {}" , second.detail) ;
131
132
133
let empty = tempfile:: tempdir( ) .unwrap ( ) ;
134
let fresh = hub ( empty.path ( ) ) .await;
135
let dest: Arc< dyn Destination> = Arc:: new( Directory { root: store.path ( ) .join ( " andy" ) } ) ;
136
let done = backups:: restore_into_hub( & fresh.app, dest, None ) .await.unwrap ( ) ;
137
assert_eq! ( done, vec! [ " andy/demo (repos)" ] ) ;
138
let name = RepoName:: parse( " andy/demo" ) .unwrap ( ) ;
139
assert_eq! ( files ( & fresh.app.repos.dir ( & name) ) , files ( & h.app.repos.dir ( & name) ) , " byte for byte" ) ;
140
let row = db:: repos:: get( & fresh.app.db, " andy" , " demo" ) .await.unwrap ( ) .expect ( " listed on the hub" ) ;
141
assert_eq! ( ( row.visibility.as_str ( ) , row.description.as_str ( ) ) , ( " private" , " a demo" ) ) ;
142
let ( a, b) = ( h.app.repos.open ( & name) .unwrap ( ) , fresh.app.repos.open ( & name) .unwrap ( ) ) ;
143
assert_eq! (
144
a.bookmarks ( ) .unwrap ( ) .iter ( ) .map ( | b | b.heads.clone ( ) ) .collect:: < Vec< _ > > ( ) ,
145
b.bookmarks ( ) .unwrap ( ) .iter ( ) .map ( | b | b.heads.clone ( ) ) .collect:: < Vec< _ > > ( )
146
) ;
147
148
149
let mut bad = Setting:: off( " andy" , " user" ) ;
150
bad.every = " hourly" .into ( ) ;
151
bad.target = " /tmp/somewhere-else" .into ( ) ;
152
backups:: save( & h.app.db, & bad) .await.unwrap ( ) ;
153
let refused = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
154
assert_eq! ( refused.status, " failed" ) ;
155
assert! ( refused.detail.contains ( " not a backup directory this hub allows" ) , " {}" , refused.detail) ;
156
}
157
158
# [ tokio ::test ( flavor = " multi_thread" ) ]
159
async fn backups_fall_due_on_their_schedule ( ) {
160
let store = tempfile:: tempdir( ) .unwrap ( ) ;
161
let h = hub ( store.path ( ) ) .await;
162
let mut s = Setting:: off( " andy" , " user" ) ;
163
s.target = store.path ( ) .display ( ) .to_string ( ) ;
164
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
165
assert! ( backups:: due( & h.app.db) .await.unwrap ( ) .is_empty ( ) , " off is never due" ) ;
166
s.every = " hourly" .into ( ) ;
167
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
168
assert_eq! ( backups:: due( & h.app.db) .await.unwrap ( ) .len ( ) , 1 , " never run: due" ) ;
3b1bc27f Andy McCall 20 days ago
169
sqlx:: query( " INSERT INTO backup_runs (owner, started_ms, status) VALUES ('andy', $1, 'ok')" )
d7d18246 Andy McCall 22 days ago ↶
170
.bind ( db:: now_ms( ) )
171
.execute ( & h.app.db)
172
.await
173
.unwrap ( ) ;
174
assert! ( backups:: due( & h.app.db) .await.unwrap ( ) .is_empty ( ) , " ran just now" ) ;
35fea423 Andy McCall 22 days ago ↶
175
}
176
177
178
179
180
# [ derive ( Default ) ]
181
struct Google {
182
183
objects : std:: collections:: BTreeMap< ( String, String) , Vec< u8 > > ,
184
185
exchanged : Vec< ( String, String) > ,
186
}
187
188
type Shared2 = Arc< std:: sync:: Mutex< Google> > ;
189
190
fn decode ( s : & str ) -> String {
191
let b = s.as_bytes ( ) ;
192
let mut out = Vec :: new( ) ;
193
let mut i = 0 ;
194
while i < b.len ( ) {
195
if b[ i] = = b ' %' & & i + 2 < b.len ( ) {
196
out.push ( u8 :: from_str_radix( & s[ i + 1 .. i + 3 ] , 16 ) .unwrap ( ) ) ;
197
i + = 3 ;
198
} else {
199
out.push ( b[ i] ) ;
200
i + = 1 ;
201
}
202
}
203
String :: from_utf8( out) .unwrap ( )
204
}
205
206
fn query ( uri : & axum:: http:: Uri, key : & str ) -> Option< String> {
207
uri.query ( ) ? .split ( ' &' ) .find_map ( | kv | kv.split_once ( ' =' ) .filter ( | ( k , _) | * k = = key) .map ( | ( _, v ) | decode ( v) ) )
208
}
209
210
211
212
213
async fn fake_google ( ) -> ( String, Shared2) {
214
use axum:: extract:: { Request, State} ;
215
use axum:: http:: StatusCode;
216
use axum:: response:: IntoResponse;
217
let state: Shared2 = Default :: default( ) ;
218
async fn handle ( State( g ) : State< Shared2> , req : Request) -> axum:: response:: Response {
219
let ( parts, body) = req.into_parts ( ) ;
220
let bytes = http_body_util:: BodyExt:: collect( body) .await.unwrap ( ) .to_bytes ( ) ;
221
let path = parts.uri.path ( ) .to_string ( ) ;
222
if path = = " /v1/token" {
223
let v: serde_json:: Value = serde_json:: from_slice( & bytes) .unwrap ( ) ;
224
let jwt = v[ " subjectToken" ] .as_str ( ) .unwrap ( ) ;
225
let payload = jwt.split ( ' .' ) .nth ( 1 ) .unwrap ( ) ;
226
let claims: serde_json:: Value = serde_json:: from_slice( & base64url ( payload) ) .unwrap ( ) ;
227
let ( sub, aud) = ( claims[ " sub" ] .as_str ( ) .unwrap ( ) .to_string ( ) , claims[ " aud" ] .as_str ( ) .unwrap ( ) .to_string ( ) ) ;
228
let provider = v[ " audience" ] .as_str ( ) .unwrap ( ) ;
229
g.lock ( ) .unwrap ( ) .exchanged.push ( ( sub.clone ( ) , aud.clone ( ) ) ) ;
230
if ! sub.starts_with ( " backup:" ) | | aud ! = format! ( " https:{provider} " ) {
231
return (
232
StatusCode:: BAD_REQUEST ,
233
axum:: Json( serde_json:: json! ( { " error" : " invalid_grant" , " error_description" : " not admitted" } ) ) ,
234
)
235
.into_response ( ) ;
236
}
237
return axum:: Json( serde_json:: json! ( { " access_token" : format! ( " tok-{sub} " ) , " expires_in" : 3600 } ) ) .into_response ( ) ;
238
}
239
let auth = parts.headers.get ( " authorization" ) .and_then ( | v | v.to_str ( ) .ok ( ) ) .unwrap_or ( " " ) .to_string ( ) ;
240
let segs: Vec< & str > = path.split ( ' /' ) .collect ( ) ;
241
242
let upload = segs.get ( 1 ) = = Some ( & " upload" ) ;
243
let at = if upload { 5 } else { 4 } ;
244
let bucket = segs.get ( at) .unwrap_or ( & " " ) .to_string ( ) ;
245
let owner = bucket.trim_end_matches ( " -backup" ) ;
246
if auth ! = format! ( " Bearer tok-backup:{owner} " ) {
247
return ( StatusCode:: FORBIDDEN , " caller does not have storage.objects access" ) .into_response ( ) ;
248
}
249
let mut g = g.lock ( ) .unwrap ( ) ;
250
if upload {
0cd11874 Andy McCall 22 days ago ↶
251
252
if ! parts.headers.contains_key ( " content-length" ) {
253
return ( StatusCode:: LENGTH_REQUIRED , " Error 411 (Length Required)" ) .into_response ( ) ;
254
}
35fea423 Andy McCall 22 days ago ↶
255
let name = query ( & parts.uri, " name" ) .unwrap ( ) ;
256
g.objects.insert ( ( bucket, name) , bytes.to_vec ( ) ) ;
257
return axum:: Json( serde_json:: json! ( { } ) ) .into_response ( ) ;
258
}
259
let name = segs.get ( at + 2 ) .map ( | n | decode ( n) ) ;
260
match ( parts.method.as_str ( ) , name) {
261
( " GET" , Some ( n) ) => match g.objects.get ( & ( bucket, n.clone ( ) ) ) {
262
Some ( b) if query ( & parts.uri, " alt" ) .as_deref ( ) = = Some ( " media" ) => b.clone ( ) .into_response ( ) ,
263
Some ( _ ) => axum:: Json( serde_json:: json! ( { " name" : n } ) ) .into_response ( ) ,
264
None => StatusCode:: NOT_FOUND .into_response ( ) ,
265
} ,
266
( " DELETE" , Some ( n) ) => {
267
g.objects.remove ( & ( bucket, n) ) ;
268
StatusCode:: NO_CONTENT .into_response ( )
269
}
270
( " GET" , None ) => {
271
let prefix = query ( & parts.uri, " prefix" ) .unwrap_or_default ( ) ;
272
let delimited = query ( & parts.uri, " delimiter" ) .is_some ( ) ;
273
let ( mut items, mut prefixes) = ( Vec :: new( ) , std:: collections:: BTreeSet:: new( ) ) ;
274
for ( b, n) in g.objects.keys ( ) {
275
if * b ! = bucket | | ! n.starts_with ( & prefix) {
276
continue ;
277
}
278
let rest = & n[ prefix.len ( ) .. ] ;
279
match rest.find ( ' /' ) {
280
Some ( i) if delimited => {
281
prefixes.insert ( format! ( " {prefix} {} " , & rest[ .. = i] ) ) ;
282
}
283
_ => items.push ( serde_json:: json! ( { " name" : n } ) ) ,
284
}
285
}
286
axum:: Json( serde_json:: json! ( { " items" : items, " prefixes" : prefixes } ) ) .into_response ( )
287
}
288
_ => StatusCode:: METHOD_NOT_ALLOWED .into_response ( ) ,
289
}
290
}
291
let app = axum:: Router:: new( ) .fallback ( handle) .with_state ( state.clone ( ) ) ;
292
let listener = tokio:: net:: TcpListener:: bind( " 127.0.0.1:0" ) .await.unwrap ( ) ;
293
let port = listener.local_addr ( ) .unwrap ( ) .port ( ) ;
294
tokio:: spawn( async move { axum:: serve( listener, app) .await.unwrap ( ) } ) ;
295
( format! ( " http://127.0.0.1:{port} " ) , state)
296
}
297
298
fn base64url ( s : & str ) -> Vec< u8 > {
299
let table = b " ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" ;
300
let mut bits = 0 u32 ;
301
let mut n = 0 ;
302
let mut out = Vec :: new( ) ;
303
for c in s.bytes ( ) {
304
let v = table.iter ( ) .position ( | t | * t = = c) .unwrap ( ) as u32 ;
305
bits = ( bits < < 6 ) | v;
306
n + = 6 ;
307
if n > = 8 {
308
n - = 8 ;
309
out.push ( ( bits > > n) as u8 ) ;
310
bits & = ( 1 < < n) - 1 ;
311
}
312
}
313
out
314
}
315
316
const PROVIDER : & str = " //iam.googleapis.com/projects/1/locations/global/workloadIdentityPools/oxygenhub/providers/oxygenhub-io" ;
317
318
fn gcs ( owner : & str , bucket : & str ) -> Setting {
319
let mut s = Setting:: off( owner, " user" ) ;
320
s.every = " hourly" .into ( ) ;
321
s.destination = " gcs" .into ( ) ;
322
s.target = bucket.into ( ) ;
323
s.provider = PROVIDER .into ( ) ;
324
s
325
}
326
327
# [ tokio ::test ( flavor = " multi_thread" ) ]
328
async fn a_gcs_backup_needs_no_key_and_restores_the_same_repository ( ) {
329
let ( google, seen) = fake_google ( ) .await;
330
let store = tempfile:: tempdir( ) .unwrap ( ) ;
331
let h = hub_with ( store.path ( ) , Some ( & google) ) .await;
332
db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
333
db:: accounts:: create_account( & h.app.db, " bob" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
334
create ( & h.app, " andy" , " demo" , hub_api:: Visibility:: Public) .await;
335
let work = tempfile:: tempdir( ) .unwrap ( ) ;
336
push ( & format! ( " {} /andy/demo" , h.base) , work.path ( ) , & [ ( " README.md" , " hello\n " ) ] ) ;
0cd11874 Andy McCall 22 days ago ↶
337
338
339
std:: fs:: write( h.app.repos.dir ( & RepoName:: parse( " andy/demo" ) .unwrap ( ) ) .join ( " .oxy/empty-marker" ) , b " " ) .unwrap ( ) ;
35fea423 Andy McCall 22 days ago ↶
340
341
backups:: save( & h.app.db, & gcs ( " andy" , " andy-backup" ) ) .await.unwrap ( ) ;
342
let first = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
343
assert_eq! ( ( first.status.as_str ( ) , first.repos) , ( " ok" , 1 ) , " {}" , first.detail) ;
344
{
345
let g = seen.lock ( ) .unwrap ( ) ;
346
assert! ( g.objects.keys ( ) .any ( | ( b , n ) | b = = " andy-backup" & & n = = & format! ( " runs/{} /DONE" , first.at) ) ) ;
347
assert! ( g.objects.keys ( ) .all ( | ( b , _) | b = = " andy-backup" ) ) ;
348
let ( sub, aud) = g.exchanged.first ( ) .unwrap ( ) ;
349
assert_eq! (
350
( sub.as_str ( ) , aud.as_str ( ) ) ,
351
( " backup:andy" , format! ( " https:{PROVIDER} " ) .as_str ( ) ) ,
352
" the hub signs in as the owner, for the provider"
353
) ;
354
}
355
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
356
let second = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
357
assert_eq! ( ( second.status.as_str ( ) , second.sent) , ( " ok" , 0 ) , " {}" , second.detail) ;
358
359
let before = seen.lock ( ) .unwrap ( ) .objects.len ( ) ;
360
let dest = backups:: destination( & h.app, & gcs ( " andy" , " andy-backup" ) ) .unwrap ( ) ;
361
tokio:: task:: spawn_blocking( move | | backups:: probe( dest.as_ref ( ) ) ) .await.unwrap ( ) .unwrap ( ) ;
362
assert_eq! ( seen.lock ( ) .unwrap ( ) .objects.len ( ) , before) ;
363
364
365
let empty = tempfile:: tempdir( ) .unwrap ( ) ;
366
let fresh = hub_with ( empty.path ( ) , Some ( & google) ) .await;
367
let dest: Arc< dyn Destination> = backups:: destination( & fresh.app, & gcs ( " andy" , " andy-backup" ) ) .unwrap ( ) .into ( ) ;
368
assert_eq! ( backups:: restore_into_hub( & fresh.app, dest, None ) .await.unwrap ( ) , vec! [ " andy/demo (repos)" ] ) ;
369
let name = RepoName:: parse( " andy/demo" ) .unwrap ( ) ;
370
assert_eq! ( files ( & fresh.app.repos.dir ( & name) ) , files ( & h.app.repos.dir ( & name) ) , " byte for byte" ) ;
371
372
373
374
backups:: save( & h.app.db, & gcs ( " bob" , " andy-backup" ) ) .await.unwrap ( ) ;
375
let refused = backups:: run( & h.app, " bob" , " now" ) .await.unwrap ( ) ;
376
assert_eq! ( refused.status, " failed" ) ;
377
assert! ( refused.detail.contains ( " roles/storage.objectAdmin" ) & & refused.detail.contains ( " backup:bob" ) , " {}" , refused.detail) ;
d7d18246 Andy McCall 22 days ago ↶
378
}
5ef6cf72 Andy McCall 22 days ago ↶
379
380
381
382
383
384
385
# [ tokio ::test ( flavor = " multi_thread" ) ]
386
async fn a_restore_brings_back_issues_proposals_and_releases ( ) {
387
let store = tempfile:: tempdir( ) .unwrap ( ) ;
388
let h = hub ( store.path ( ) ) .await;
389
let andy = db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
390
let bob = db:: accounts:: create_account( & h.app.db, " bob" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
391
create ( & h.app, " andy" , " demo" , hub_api:: Visibility:: Public) .await;
392
let work = tempfile:: tempdir( ) .unwrap ( ) ;
393
push ( & format! ( " {} /andy/demo" , h.base) , work.path ( ) , & [ ( " README.md" , " hello\n " ) ] ) ;
394
let row = db:: repos:: get( & h.app.db, " andy" , " demo" ) .await.unwrap ( ) .unwrap ( ) ;
395
db:: repos:: set_diffusion( & h.app.db, row.id, " green" ) .await.unwrap ( ) ;
396
397
for n in 0 .. 3 {
398
db:: issues:: upsert_label( & h.app.db, row.id, & format! ( " spare{n} " ) , " #000000" , " " ) .await.unwrap ( ) ;
399
}
400
let bug = db:: issues:: upsert_label( & h.app.db, row.id, " bug" , " #d73a4a" , " Something is wrong" ) .await.unwrap ( ) ;
401
let m = db:: issues:: upsert_milestone( & h.app.db, row.id, " 1.0" , " the first" , None ) .await.unwrap ( ) ;
402
let issue = db:: issues:: create(
403
& h.app.db,
404
db:: issues:: NewIssue { repo_id: row.id, title: " Sprites flicker" , body: " on level two" , author_id: Some ( andy.id) , author: " andy" } ,
405
)
406
.await
407
.unwrap ( ) ;
408
db:: issues:: add_comment( & h.app.db, issue.id, Some ( bob.id) , " bob" , " seen it too" ) .await.unwrap ( ) ;
409
db:: issues:: set_issue_label( & h.app.db, issue.id, bug.id, true ) .await.unwrap ( ) ;
410
db:: issues:: set_milestone( & h.app.db, issue.id, Some ( m.id) ) .await.unwrap ( ) ;
411
db:: issues:: set_assignee( & h.app.db, issue.id, andy.id, true ) .await.unwrap ( ) ;
412
db:: issues:: set_assignee( & h.app.db, issue.id, bob.id, true ) .await.unwrap ( ) ;
413
let p = db:: proposals:: create(
414
& h.app.db,
415
db:: proposals:: NewProposal {
416
repo_id: row.id,
417
title: " Double-buffer the sprites" ,
418
body: " fixes the flicker" ,
419
author_id: Some ( bob.id) ,
420
author: " bob" ,
421
draft: false ,
422
target: " main" ,
423
source_kind: " bookmark" ,
424
source_repo_id: Some ( row.id) ,
425
source_bookmark: " fix" ,
426
source_changes: " " ,
427
} ,
428
)
429
.await
430
.unwrap ( ) ;
431
db:: proposals:: add_event( & h.app.db, p.id, " bob" , " opened" , " " ) .await.unwrap ( ) ;
432
db:: proposals:: add_review( & h.app.db, p.id, andy.id, " andy" , " approve" , " nice" ) .await.unwrap ( ) ;
433
let release = db:: releases:: create( & h.app.db, row.id, " v1.0" , " One" , " notes" , false , Some ( ( andy.id, " andy" ) ) ) .await.unwrap ( ) ;
434
let asset = b " a game image" .to_vec ( ) ;
435
let hash = blake3:: hash( & asset) .to_hex ( ) .to_string ( ) ;
436
let path = h.app.config.data.join ( " assets" ) .join ( & hash[ .. 2 ] ) .join ( & hash) ;
437
std:: fs:: create_dir_all( path.parent ( ) .unwrap ( ) ) .unwrap ( ) ;
438
std:: fs:: write( & path, & asset) .unwrap ( ) ;
439
db:: releases:: add_asset( & h.app.db, release.id, " game.prg" , asset.len ( ) as i64 , & hash, " application/octet-stream" ) .await.unwrap ( ) ;
440
db:: hooks:: create_hook( & h.app.db, Some ( row.id) , None , " https://example.com/hook" , " s3cret-value" , " push" ) .await.unwrap ( ) ;
441
442
let mut s = Setting:: off( " andy" , " user" ) ;
443
s.every = " hourly" .into ( ) ;
444
s.target = store.path ( ) .display ( ) .to_string ( ) ;
445
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
446
let run = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
447
assert_eq! ( run.status, " ok" , " {}" , run.detail) ;
448
let dest = Directory { root: store.path ( ) .join ( " andy" ) } ;
449
let text = String :: from_utf8_lossy( & dest.get ( & format! ( " runs/{} /records/demo.json" , run.at) ) .unwrap ( ) ) .into_owned ( ) ;
450
assert! ( ! text.contains ( " s3cret-value" ) , " never a secret" ) ;
451
assert! ( text.contains ( " \" _names\" " ) & & text.contains ( " \" bob\" " ) , " accounts by name" ) ;
452
453
454
let empty = tempfile:: tempdir( ) .unwrap ( ) ;
455
let fresh = hub ( empty.path ( ) ) .await;
456
let andy2 = db:: accounts:: create_account( & fresh.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
457
let dest: Arc< dyn Destination> = Arc:: new( Directory { root: store.path ( ) .join ( " andy" ) } ) ;
458
let done = backups:: restore_into_hub( & fresh.app, dest, None ) .await.unwrap ( ) ;
459
assert_eq! ( done[ 0 ] , " andy/demo (repos)" ) ;
460
let notes = done[ 1 .. ] .join ( " \n " ) ;
461
assert! ( notes.contains ( " webhook https://example.com/hook: enter its secret again" ) , " {notes}" ) ;
462
assert! ( notes.contains ( " issue_assignees: 1 row(s) left out" ) , " bob is not on this hub: {notes}" ) ;
463
let r2 = db:: repos:: get( & fresh.app.db, " andy" , " demo" ) .await.unwrap ( ) .unwrap ( ) ;
464
assert_eq! ( r2.diffusion, " green" , " settings come back" ) ;
465
3b1bc27f Andy McCall 20 days ago
466
let issues: Vec< ( i64 , i64 , String, Option< i64 > ) > = sqlx:: query_as( " SELECT id, number, title, author_id FROM issues WHERE repo_id = $1" )
5ef6cf72 Andy McCall 22 days ago ↶
467
.bind ( r2.id)
468
.fetch_all ( & fresh.app.db)
469
.await
470
.unwrap ( ) ;
471
assert_eq! ( issues.len ( ) , 1 ) ;
472
let ( iid, number, title, author) = issues[ 0 ] .clone ( ) ;
473
assert_eq! ( ( number, title.as_str ( ) , author) , ( issue.number, " Sprites flicker" , Some ( andy2.id) ) ) ;
474
let comments = db:: issues:: comments( & fresh.app.db, iid) .await.unwrap ( ) ;
475
assert_eq! ( comments.len ( ) , 1 ) ;
476
assert_eq! ( ( comments[ 0 ] .author.as_str ( ) , comments[ 0 ] .body.as_str ( ) ) , ( " bob" , " seen it too" ) ) ;
477
let labels: Vec< String> =
3b1bc27f Andy McCall 20 days ago
478
sqlx:: query_scalar( " SELECT l.name FROM issue_labels il JOIN labels l ON l.id = il.label_id WHERE il.issue_id = $1" )
5ef6cf72 Andy McCall 22 days ago ↶
479
.bind ( iid)
480
.fetch_all ( & fresh.app.db)
481
.await
482
.unwrap ( ) ;
483
assert_eq! ( labels, vec! [ " bug" ] , " the label, its id mapped" ) ;
484
let milestone: Option< String> =
3b1bc27f Andy McCall 20 days ago
485
sqlx:: query_scalar( " SELECT m.title FROM issues i JOIN milestones m ON m.id = i.milestone_id WHERE i.id = $1" )
5ef6cf72 Andy McCall 22 days ago ↶
486
.bind ( iid)
487
.fetch_one ( & fresh.app.db)
488
.await
489
.unwrap ( ) ;
490
assert_eq! ( milestone.as_deref ( ) , Some ( " 1.0" ) ) ;
491
assert_eq! ( db:: issues:: assignees( & fresh.app.db, iid) .await.unwrap ( ) , vec! [ " andy" ] ) ;
492
493
let p2 = db:: proposals:: get( & fresh.app.db, r2.id, p.number) .await.unwrap ( ) .expect ( " the proposal" ) ;
494
assert_eq! ( ( p2.title.as_str ( ) , p2.author.as_str ( ) , p2.author_id) , ( " Double-buffer the sprites" , " bob" , None ) ) ;
3b1bc27f Andy McCall 20 days ago
495
let reviews: Vec< ( String, String) > = sqlx:: query_as( " SELECT author, verdict FROM proposal_reviews WHERE proposal_id = $1" )
5ef6cf72 Andy McCall 22 days ago ↶
496
.bind ( p2.id)
497
.fetch_all ( & fresh.app.db)
498
.await
499
.unwrap ( ) ;
500
assert_eq! ( reviews, vec! [ ( " andy" .to_string ( ) , " approve" .to_string ( ) ) ] ) ;
501
assert! ( db:: proposals:: events( & fresh.app.db, p2.id) .await.unwrap ( ) .iter ( ) .any ( | e | e.event = = " opened" ) ) ;
502
503
let rel = db:: releases:: get( & fresh.app.db, r2.id, " v1.0" ) .await.unwrap ( ) .expect ( " the release" ) ;
504
let assets = db:: releases:: assets( & fresh.app.db, rel.id) .await.unwrap ( ) ;
505
assert_eq! ( assets.len ( ) , 1 ) ;
506
assert_eq! ( std:: fs:: read( fresh.app.config.data.join ( " assets" ) .join ( & hash[ .. 2 ] ) .join ( & hash) ) .unwrap ( ) , asset) ;
507
508
let hits = db:: notify:: search( & fresh.app.db, " flicker" , Some ( r2.id) , 10 ) .await.unwrap ( ) ;
509
assert! ( hits.iter ( ) .any ( | h | h.kind = = " issue" & & h.target_id = = iid ) , " {:?}" , hits.iter ( ) .map ( | h | & h.title) .collect:: < Vec< _ > > ( ) ) ;
510
511
let next = db:: issues:: create(
512
& fresh.app.db,
513
db:: issues:: NewIssue { repo_id: r2.id, title: " After" , body: " " , author_id: Some ( andy2.id) , author: " andy" } ,
514
)
515
.await
516
.unwrap ( ) ;
517
assert! ( next.number > p.number, " {} after {}" , next.number, p.number) ;
518
}
d3b048b2 Andy McCall 22 days ago ↶
519
520
521
522
523
524
# [ tokio ::test ( flavor = " multi_thread" ) ]
525
async fn repositories_choose_their_own_backup ( ) {
526
let store = tempfile:: tempdir( ) .unwrap ( ) ;
527
let h = hub ( store.path ( ) ) .await;
528
db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
529
for n in [ " game" , " notes" , " secret" ] {
530
create ( & h.app, " andy" , n, hub_api:: Visibility:: Public) .await;
531
let work = tempfile:: tempdir( ) .unwrap ( ) ;
532
push ( & format! ( " {} /andy/{n} " , h.base) , work.path ( ) , & [ ( " README.md" , n) ] ) ;
533
}
534
let id = | n : & 'static str | {
535
let db = h.app.db.clone ( ) ;
536
async move { db:: repos:: get( & db, " andy" , n) .await.unwrap ( ) .unwrap ( ) .id }
537
} ;
538
db:: repos:: set_backup( & h.app.db, id ( " secret" ) .await, " off" ) .await.unwrap ( ) ;
539
let mut s = Setting:: off( " andy" , " user" ) ;
540
s.every = " daily" .into ( ) ;
541
s.target = store.path ( ) .display ( ) .to_string ( ) ;
542
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
543
let names = | at : String| {
544
let dest = Directory { root: store.path ( ) .join ( " andy" ) } ;
545
let mut v = dest.list ( & format! ( " runs/{at} /records" ) ) .unwrap ( ) ;
546
v.sort ( ) ;
547
v
548
} ;
549
let first = backups:: run( & h.app, " andy" , " schedule" ) .await.unwrap ( ) ;
550
assert_eq! ( ( first.status.as_str ( ) , first.repos) , ( " ok" , 2 ) , " {}" , first.detail) ;
551
assert_eq! ( names ( first.at.clone ( ) ) , vec! [ " game.json" , " notes.json" ] , " the one left out is not in it" ) ;
552
553
assert! ( backups:: due( & h.app.db) .await.unwrap ( ) .is_empty ( ) ) ;
554
db:: repos:: set_backup( & h.app.db, id ( " game" ) .await, " hourly" ) .await.unwrap ( ) ;
555
sqlx:: query( " UPDATE backup_runs SET started_ms = started_ms - 2 * 3600 * 1000" ) .execute ( & h.app.db) .await.unwrap ( ) ;
556
assert_eq! ( backups:: due( & h.app.db) .await.unwrap ( ) .len ( ) , 1 , " hourly for the game" ) ;
557
558
s.every = " off" .into ( ) ;
559
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
560
assert_eq! ( backups:: due( & h.app.db) .await.unwrap ( ) .len ( ) , 1 ) ;
561
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
562
let second = backups:: run( & h.app, " andy" , " schedule" ) .await.unwrap ( ) ;
563
assert_eq! ( names ( second.at.clone ( ) ) , vec! [ " game.json" ] ) ;
564
565
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
566
let third = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
567
assert_eq! ( names ( third.at.clone ( ) ) , vec! [ " game.json" , " notes.json" ] ) ;
568
569
let ( owners, uncovered) = backups:: overview( & h.app.db) .await.unwrap ( ) ;
570
assert_eq! ( ( owners[ 0 ] .repos, uncovered.len ( ) ) , ( 1 , 0 ) ) ;
571
}
7b9f4570 Andy McCall 22 days ago ↶
572
573
574
575
576
577
async fn fake_aws ( ) -> ( String, Shared2) {
578
use axum:: extract:: { Request, State} ;
579
use axum:: http:: StatusCode;
580
use axum:: response:: IntoResponse;
581
let state: Shared2 = Default :: default( ) ;
582
async fn handle ( State( g ) : State< Shared2> , req : Request) -> axum:: response:: Response {
583
let ( parts, body) = req.into_parts ( ) ;
584
let bytes = http_body_util:: BodyExt:: collect( body) .await.unwrap ( ) .to_bytes ( ) ;
585
let path = parts.uri.path ( ) .to_string ( ) ;
586
let header = | h : & str | parts.headers.get ( h) .and_then ( | v | v.to_str ( ) .ok ( ) ) .unwrap_or ( " " ) .to_string ( ) ;
587
if path = = " /" & & parts.method = = " POST" {
588
let form: Vec< ( String, String) > = String :: from_utf8_lossy( & bytes)
589
.split ( ' &' )
590
.filter_map ( | kv | kv.split_once ( ' =' ) .map ( | ( k , v ) | ( k.to_string ( ) , decode ( & v.replace ( ' +' , " " ) ) ) ) )
591
.collect ( ) ;
592
let field = | k : & str | form.iter ( ) .find ( | ( f , _) | f = = k ) .map ( | ( _, v ) | v.clone ( ) ) .unwrap_or_default ( ) ;
593
let jwt = field ( " WebIdentityToken" ) ;
594
let claims: serde_json:: Value = serde_json:: from_slice( & base64url ( jwt.split ( ' .' ) .nth ( 1 ) .unwrap ( ) ) ) .unwrap ( ) ;
595
let ( sub, aud) = ( claims[ " sub" ] .as_str ( ) .unwrap ( ) .to_string ( ) , claims[ " aud" ] .as_str ( ) .unwrap ( ) .to_string ( ) ) ;
596
g.lock ( ) .unwrap ( ) .exchanged.push ( ( sub.clone ( ) , aud.clone ( ) ) ) ;
597
if field ( " Action" ) ! = " AssumeRoleWithWebIdentity" | | ! sub.starts_with ( " backup:" ) | | aud ! = " sts.amazonaws.com" {
598
return ( StatusCode:: FORBIDDEN , " <ErrorResponse><Error><Code>AccessDenied</Code><Message>Not authorized to perform sts:AssumeRoleWithWebIdentity</Message></Error></ErrorResponse>" ) .into_response ( ) ;
599
}
600
let owner = sub.trim_start_matches ( " backup:" ) ;
601
return format! (
602
" <AssumeRoleWithWebIdentityResponse><AssumeRoleWithWebIdentityResult><Credentials>\
603
<AccessKeyId>ASIA{owner} </AccessKeyId><SecretAccessKey>secret/{owner} </SecretAccessKey>\
604
<SessionToken>session-{owner} </SessionToken><Expiration>2030-01-01T00:00:00Z</Expiration>\
605
</Credentials></AssumeRoleWithWebIdentityResult></AssumeRoleWithWebIdentityResponse>"
606
)
607
.into_response ( ) ;
608
}
609
610
let rest = path.strip_prefix ( " /s3/" ) .unwrap_or ( " " ) ;
611
let ( bucket, key) = rest.split_once ( ' /' ) .unwrap_or ( ( rest, " " ) ) ;
612
let ( bucket, key) = ( bucket.to_string ( ) , decode ( key) ) ;
613
let owner = bucket.trim_end_matches ( " -backup" ) .to_string ( ) ;
614
615
let session = header ( " x-amz-security-token" ) ;
616
let who = session.trim_start_matches ( " session-" ) .to_string ( ) ;
617
let query: Vec< ( String, String) > = parts
618
.uri
619
.query ( )
620
.unwrap_or ( " " )
621
.split ( ' &' )
622
.filter ( | kv | ! kv.is_empty ( ) )
623
.map ( | kv | kv.split_once ( ' =' ) .map ( | ( k , v ) | ( decode ( k) , decode ( v) ) ) .unwrap_or ( ( decode ( kv) , String :: new( ) ) ) )
624
.collect ( ) ;
625
let signed = vec! [
626
( " host" .to_string ( ) , header ( " host" ) ) ,
627
( " x-amz-content-sha256" .to_string ( ) , header ( " x-amz-content-sha256" ) ) ,
628
( " x-amz-date" .to_string ( ) , header ( " x-amz-date" ) ) ,
629
( " x-amz-security-token" .to_string ( ) , session.clone ( ) ) ,
630
] ;
631
let id = format! ( " ASIA{who} " ) ;
632
let secret = format! ( " secret/{who} " ) ;
633
let want = oxygenhub:: backups:: sigv4(
634
parts.method.as_str ( ) ,
635
& path,
636
& query,
637
& signed,
638
& header ( " x-amz-content-sha256" ) ,
639
( & id, & secret) ,
640
" eu-west-2" ,
641
" s3" ,
642
& header ( " x-amz-date" ) ,
643
) ;
644
use sha2:: Digest;
645
let body_hash = hex:: encode( sha2:: Sha256:: digest( & bytes) ) ;
646
if header ( " authorization" ) ! = want | | header ( " x-amz-content-sha256" ) ! = body_hash {
647
return (
648
StatusCode:: FORBIDDEN ,
649
" <Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>" ,
650
)
651
.into_response ( ) ;
652
}
653
if who ! = owner {
654
return ( StatusCode:: FORBIDDEN , " <Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>" ) .into_response ( ) ;
655
}
656
let mut g = g.lock ( ) .unwrap ( ) ;
657
match ( parts.method.as_str ( ) , key.is_empty ( ) ) {
0cd11874 Andy McCall 22 days ago ↶
658
( " PUT" , false ) if ! parts.headers.contains_key ( " content-length" ) => {
659
( StatusCode:: LENGTH_REQUIRED , " <Error><Code>MissingContentLength</Code></Error>" ) .into_response ( )
660
}
7b9f4570 Andy McCall 22 days ago ↶
661
( " PUT" , false ) => {
662
g.objects.insert ( ( bucket, key) , bytes.to_vec ( ) ) ;
663
StatusCode:: OK .into_response ( )
664
}
665
( " GET" , false ) | ( " HEAD" , false ) => match g.objects.get ( & ( bucket, key) ) {
666
Some ( b) => b.clone ( ) .into_response ( ) ,
667
None => ( StatusCode:: NOT_FOUND , " <Error><Code>NoSuchKey</Code></Error>" ) .into_response ( ) ,
668
} ,
669
( " DELETE" , false ) => {
670
g.objects.remove ( & ( bucket, key) ) ;
671
StatusCode:: NO_CONTENT .into_response ( )
672
}
673
( " GET" , true ) => {
674
let prefix = query.iter ( ) .find ( | ( k , _) | k = = " prefix" ) .map ( | ( _, v ) | v.clone ( ) ) .unwrap_or_default ( ) ;
675
let delimited = query.iter ( ) .any ( | ( k , _) | k = = " delimiter" ) ;
676
let ( mut keys, mut prefixes) = ( String :: new( ) , std:: collections:: BTreeSet:: new( ) ) ;
677
for ( b, n) in g.objects.keys ( ) {
678
if * b ! = bucket | | ! n.starts_with ( & prefix) {
679
continue ;
680
}
681
match n[ prefix.len ( ) .. ] .find ( ' /' ) {
682
Some ( i) if delimited => {
683
prefixes.insert ( format! ( " {prefix} {} " , & n[ prefix.len ( ) .. prefix.len ( ) + i + 1 ] ) ) ;
684
}
685
_ => keys.push_str ( & format! ( " <Contents><Key>{n} </Key></Contents>" ) ) ,
686
}
687
}
688
let common: String = prefixes.iter ( ) .map ( | p | format! ( " <CommonPrefixes><Prefix>{p} </Prefix></CommonPrefixes>" ) ) .collect ( ) ;
689
format! ( " <ListBucketResult><Prefix>{prefix} </Prefix><IsTruncated>false</IsTruncated>{keys} {common} </ListBucketResult>" )
690
.into_response ( )
691
}
692
_ => StatusCode:: METHOD_NOT_ALLOWED .into_response ( ) ,
693
}
694
}
695
let app = axum:: Router:: new( ) .fallback ( handle) .with_state ( state.clone ( ) ) ;
696
let listener = tokio:: net:: TcpListener:: bind( " 127.0.0.1:0" ) .await.unwrap ( ) ;
697
let port = listener.local_addr ( ) .unwrap ( ) .port ( ) ;
698
tokio:: spawn( async move { axum:: serve( listener, app) .await.unwrap ( ) } ) ;
699
( format! ( " http://127.0.0.1:{port} " ) , state)
700
}
701
702
async fn hub_aws ( backups : & Path, aws : & str ) -> Hub {
703
hermetic ( ) ;
704
let dir = tempfile:: tempdir( ) .unwrap ( ) ;
705
let mut config = Config:: load( dir.path ( ) ) .unwrap ( ) ;
c8684d44 Andy McCall 19 days ago
706
707
config.registration.setup_code = " test-setup" .into ( ) ;
7b9f4570 Andy McCall 22 days ago ↶
708
config.auth.push_tokens = vec! [ " site" .into ( ) ] ;
709
config.backups.directories = vec! [ backups.to_path_buf ( ) ] ;
710
config.backups.aws_sts_url = aws.to_string ( ) ;
711
config.backups.s3_url = format! ( " {aws} /s3/{{ bucket}} " ) ;
712
let listener = tokio:: net:: TcpListener:: bind( " 127.0.0.1:0" ) .await.unwrap ( ) ;
713
let port = listener.local_addr ( ) .unwrap ( ) .port ( ) ;
714
config.server.base_url = format! ( " http://127.0.0.1:{port} " ) ;
715
let app = oxygenhub:: build( config) .await.unwrap ( ) ;
716
let router = oxygenhub:: router( app.clone ( ) ) ;
717
tokio:: spawn( async move { axum:: serve( listener, router) .await.unwrap ( ) } ) ;
718
Hub { app, base: format! ( " http://127.0.0.1:{port} " ) , _dir: dir }
719
}
720
721
const ROLE : & str = " arn:aws:iam::123456789012:role/oxygenhub-backup" ;
722
723
fn s3 ( owner : & str , bucket : & str ) -> Setting {
724
let mut s = Setting:: off( owner, " user" ) ;
725
s.every = " hourly" .into ( ) ;
726
s.destination = " s3" .into ( ) ;
727
s.target = bucket.into ( ) ;
728
s.provider = ROLE .into ( ) ;
729
s.region = " eu-west-2" .into ( ) ;
730
s.prefix = " oxygen hub" .into ( ) ;
731
s
732
}
733
734
735
736
737
738
739
# [ tokio ::test ( flavor = " multi_thread" ) ]
740
async fn an_s3_backup_needs_no_key_and_restores_the_same_repository ( ) {
741
let ( aws, seen) = fake_aws ( ) .await;
742
let store = tempfile:: tempdir( ) .unwrap ( ) ;
743
let h = hub_aws ( store.path ( ) , & aws) .await;
744
db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
745
db:: accounts:: create_account( & h.app.db, " bob" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
746
create ( & h.app, " andy" , " demo" , hub_api:: Visibility:: Public) .await;
747
let work = tempfile:: tempdir( ) .unwrap ( ) ;
748
push ( & format! ( " {} /andy/demo" , h.base) , work.path ( ) , & [ ( " README.md" , " hello\n " ) , ( " src/a b+c.rs" , " fn main() {}\n " ) ] ) ;
0cd11874 Andy McCall 22 days ago ↶
749
std:: fs:: write( h.app.repos.dir ( & RepoName:: parse( " andy/demo" ) .unwrap ( ) ) .join ( " .oxy/empty-marker" ) , b " " ) .unwrap ( ) ;
7b9f4570 Andy McCall 22 days ago ↶
750
751
backups:: save( & h.app.db, & s3 ( " andy" , " andy-backup" ) ) .await.unwrap ( ) ;
752
let first = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
753
assert_eq! ( ( first.status.as_str ( ) , first.repos) , ( " ok" , 1 ) , " {}" , first.detail) ;
754
{
755
let g = seen.lock ( ) .unwrap ( ) ;
756
assert! (
757
g.objects.keys ( ) .any ( | ( b , n ) | b = = " andy-backup" & & n = = & format! ( " oxygen hub/runs/{} /DONE" , first.at) ) ,
758
" under the prefix"
759
) ;
760
assert_eq! ( g.exchanged.first ( ) .map ( | ( s , a ) | ( s.as_str ( ) , a.as_str ( ) ) ) , Some ( ( " backup:andy" , " sts.amazonaws.com" ) ) ) ;
761
}
762
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
763
let second = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
764
assert_eq! ( ( second.status.as_str ( ) , second.sent) , ( " ok" , 0 ) , " {}" , second.detail) ;
765
let before = seen.lock ( ) .unwrap ( ) .objects.len ( ) ;
766
let dest = backups:: destination( & h.app, & s3 ( " andy" , " andy-backup" ) ) .unwrap ( ) ;
767
tokio:: task:: spawn_blocking( move | | backups:: probe( dest.as_ref ( ) ) ) .await.unwrap ( ) .unwrap ( ) ;
768
assert_eq! ( seen.lock ( ) .unwrap ( ) .objects.len ( ) , before, " the probe cleans up" ) ;
769
770
let empty = tempfile:: tempdir( ) .unwrap ( ) ;
771
let fresh = hub_aws ( empty.path ( ) , & aws) .await;
772
let dest: Arc< dyn Destination> = backups:: destination( & fresh.app, & s3 ( " andy" , " andy-backup" ) ) .unwrap ( ) .into ( ) ;
773
assert_eq! ( backups:: restore_into_hub( & fresh.app, dest, None ) .await.unwrap ( ) , vec! [ " andy/demo (repos)" ] ) ;
774
let name = RepoName:: parse( " andy/demo" ) .unwrap ( ) ;
775
assert_eq! ( files ( & fresh.app.repos.dir ( & name) ) , files ( & h.app.repos.dir ( & name) ) , " byte for byte" ) ;
776
777
backups:: save( & h.app.db, & s3 ( " bob" , " andy-backup" ) ) .await.unwrap ( ) ;
778
let refused = backups:: run( & h.app, " bob" , " now" ) .await.unwrap ( ) ;
779
assert_eq! ( refused.status, " failed" ) ;
780
assert! ( refused.detail.contains ( " s3:PutObject" ) & & refused.detail.contains ( ROLE ) , " {}" , refused.detail) ;
781
}
2e9188ca Andy McCall 22 days ago ↶
782
783
784
async fn api (
785
app : & Shared,
786
method : & str ,
787
path : & str ,
788
token : & str ,
789
body : Option< serde_json:: Value> ,
790
) -> ( axum:: http:: StatusCode, serde_json:: Value) {
791
use axum:: body:: Body;
792
use http_body_util:: BodyExt;
793
use tower:: ServiceExt;
794
let mut req =
795
axum:: http:: Request:: builder( ) .method ( method) .uri ( format! ( " /api/v1{path} " ) ) .header ( " authorization" , format! ( " Bearer {token} " ) ) ;
796
let body = match body {
797
Some ( b) => {
798
req = req.header ( " content-type" , " application/json" ) ;
799
Body:: from( b.to_string ( ) )
800
}
801
None => Body:: empty( ) ,
802
} ;
803
let res = oxygenhub:: router( app.clone ( ) ) .oneshot ( req.body ( body) .unwrap ( ) ) .await.unwrap ( ) ;
804
let st = res.status ( ) ;
805
let bytes = res.into_body ( ) .collect ( ) .await.unwrap ( ) .to_bytes ( ) ;
806
( st, serde_json:: from_slice( & bytes) .unwrap_or ( serde_json:: Value:: Null) )
807
}
808
809
810
811
812
# [ tokio ::test ( flavor = " multi_thread" ) ]
813
async fn backups_are_set_up_and_run_through_the_api ( ) {
814
let store = tempfile:: tempdir( ) .unwrap ( ) ;
815
let h = hub ( store.path ( ) ) .await;
816
let andy = db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
817
let bob = db:: accounts:: create_account( & h.app.db, " bob" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
818
db:: accounts:: create_org( & h.app.db, " club" , " Club" , andy.id) .await.unwrap ( ) ;
819
create ( & h.app, " andy" , " demo" , hub_api:: Visibility:: Public) .await;
820
create ( & h.app, " club" , " game" , hub_api:: Visibility:: Private) .await;
821
let token = | id : i64 , scopes : & 'static str | {
822
let db = h.app.db.clone ( ) ;
823
async move {
824
let ( t, hash) = oxygenhub:: auth:: new_token( ) ;
825
db:: accounts:: create_token( & db, id, " t" , & hash, scopes, " " , None ) .await.unwrap ( ) ;
826
t
827
}
828
} ;
829
let ( andy_t , bob_t , weak) = ( token ( andy.id, " repo:admin" ) .await, token ( bob.id, " repo:admin" ) .await, token ( andy.id, " repo:read" ) .await) ;
830
let dir = store.path ( ) .display ( ) .to_string ( ) ;
831
832
let ( st, empty) = api ( & h.app, " GET" , " /user/backups" , & andy_t , None ) .await;
833
assert_eq! ( st, 200 ) ;
834
assert! ( empty[ " setting" ] .is_null ( ) & & empty[ " runs" ] .as_array ( ) .unwrap ( ) .is_empty ( ) , " {empty}" ) ;
835
let ( st, _ ) = api ( & h.app, " POST" , " /user/backups/run" , & andy_t , None ) .await;
836
assert_eq! ( st, 409 , " nothing saved yet" ) ;
837
let ( st, _ ) =
838
api ( & h.app, " PUT" , " /user/backups" , & weak, Some ( serde_json:: json! ( { " every" : " hourly" , " destination" : " dir" , " dir" : dir } ) ) ) .await;
839
assert_eq! ( st, 403 , " repo:admin only" ) ;
840
let ( st, bad) = api (
841
& h.app,
842
" PUT" ,
843
" /user/backups" ,
844
& andy_t ,
845
Some ( serde_json:: json! ( { " every" : " hourly" , " destination" : " gcs" , " bucket" : " andy-backup" , " provider" : " nope" } ) ) ,
846
)
847
.await;
848
assert_eq! ( st, 400 , " checked as the page checks it: {bad}" ) ;
849
let ( st, saved) = api (
850
& h.app,
851
" PUT" ,
852
" /user/backups" ,
853
& andy_t ,
854
Some ( serde_json:: json! ( { " every" : " hourly" , " destination" : " dir" , " dir" : dir, " keep_days" : 30 } ) ) ,
855
)
856
.await;
857
assert_eq! ( st, 200 , " {saved}" ) ;
858
assert_eq! ( ( saved[ " setting" ] [ " every" ] .as_str ( ) , saved[ " place" ] .as_str ( ) ) , ( Some ( " hourly" ) , Some ( format! ( " {dir} /andy" ) .as_str ( ) ) ) ) ;
859
let ( st, probe) = api ( & h.app, " POST" , " /user/backups/test" , & andy_t , None ) .await;
860
assert_eq! ( ( st.as_u16 ( ) , probe[ " ok" ] .as_bool ( ) ) , ( 200 , Some ( true ) ) , " {probe}" ) ;
861
let ( st, started) = api ( & h.app, " POST" , " /user/backups/run" , & andy_t , None ) .await;
862
assert_eq! ( st, 202 , " {started}" ) ;
863
let mut done = serde_json:: Value:: Null;
864
for _ in 0 .. 200 {
865
let ( _ , runs) = api ( & h.app, " GET" , " /user/backups/runs" , & andy_t , None ) .await;
866
if runs[ 0 ] [ " status" ] .as_str ( ) .is_some_and ( | s | s ! = " running" ) {
867
done = runs[ 0 ] .clone ( ) ;
868
break ;
869
}
870
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 50 ) ) .await;
871
}
872
assert_eq! ( ( done[ " status" ] .as_str ( ) , done[ " repos" ] .as_i64 ( ) , done[ " trigger" ] .as_str ( ) ) , ( Some ( " ok" ) , Some ( 1 ) , Some ( " now" ) ) , " {done}" ) ;
bff9106e Andy McCall 22 days ago ↶
873
874
875
sqlx:: query( " INSERT INTO backup_runs (owner, started_ms, finished_ms, status, at, trigger, detail) VALUES ('andy', 1, 2, 'failed', 'x', 'schedule', 'GCS refused')" )
876
.execute ( & h.app.db)
877
.await
878
.unwrap ( ) ;
879
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
880
let again = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
881
assert_eq! ( again.status, " ok" ) ;
3b1bc27f Andy McCall 20 days ago
882
let told = | db : oxygenhub:: db:: Pool| async move { db:: notify:: inbox( & db, andy.id, 20 ) .await.unwrap ( ) } ;
bff9106e Andy McCall 22 days ago ↶
883
assert! ( told ( h.app.db.clone ( ) ) .await.iter ( ) .any ( | n | n.kind = = " backup.recovered" & & n.title.contains ( " works again" ) ) ) ;
884
assert_eq! ( backups:: daily_summary( & h.app, db:: now_ms( ) , true ) .await.unwrap ( ) , 1 ) ;
885
let summary = told ( h.app.db.clone ( ) ) .await.into_iter ( ) .find ( | n | n.kind = = " backup.summary" ) .expect ( " a summary" ) ;
886
assert! ( summary.title.contains ( " Backups of andy today" ) & & summary.title.contains ( " runs" ) , " {}" , summary.title) ;
2e9188ca Andy McCall 22 days ago ↶
887
888
889
let ( st, _ ) =
890
api ( & h.app, " PUT" , " /orgs/club/backups" , & bob_t , Some ( serde_json:: json! ( { " every" : " hourly" , " destination" : " dir" , " dir" : dir } ) ) )
891
.await;
892
assert_eq! ( st, 404 ) ;
893
let ( st, org) = api (
894
& h.app,
895
" PUT" ,
896
" /orgs/club/backups" ,
897
& andy_t ,
898
Some ( serde_json:: json! ( { " every" : " daily" , " destination" : " dir" , " dir" : dir, " keep_days" : 90 } ) ) ,
899
)
900
.await;
901
assert_eq! ( st, 200 , " {org}" ) ;
902
assert_eq! ( ( org[ " setting" ] [ " kind" ] .as_str ( ) , org[ " setting" ] [ " keep_days" ] .as_i64 ( ) ) , ( Some ( " org" ) , Some ( 90 ) ) ) ;
903
let ( st, _ ) = api ( & h.app, " GET" , " /orgs/club/backups" , & bob_t , None ) .await;
904
assert_eq! ( st, 404 ) ;
905
}