d7d18246 Andy McCall 23 days ago ↶
1
2
3
4
5
6
use oxy_core:: repo:: Repo;
7
use oxygenhub:: backups:: { self , Destination, Directory, Setting} ;
8
use oxygenhub:: config:: Config;
9
use oxygenhub:: repos:: RepoName;
10
use oxygenhub:: { db, Shared} ;
11
use std:: path:: Path;
12
use std:: sync:: Arc;
13
14
fn hermetic ( ) {
15
static ONCE : std:: sync:: Once = std:: sync:: Once:: new( ) ;
16
static DIR : std:: sync:: OnceLock< tempfile:: TempDir> = std:: sync:: OnceLock:: new( ) ;
17
ONCE .call_once ( | | {
18
let d = tempfile:: tempdir( ) .unwrap ( ) ;
19
std:: env:: set_var( " OXY_HOME" , d.path ( ) ) ;
20
DIR .set ( d) .ok ( ) ;
21
} ) ;
22
}
23
24
struct Hub {
25
app : Shared,
26
base : String,
27
_dir : tempfile:: TempDir,
28
}
29
30
async fn hub ( backups : & Path) -> Hub {
35fea423 Andy McCall 22 days ago ↶
31
hub_with ( backups, None ) .await
32
}
33
34
async fn hub_with ( backups : & Path, google : Option< & str > ) -> Hub {
d7d18246 Andy McCall 23 days ago ↶
35
hermetic ( ) ;
36
let dir = tempfile:: tempdir( ) .unwrap ( ) ;
37
let mut config = Config:: load( dir.path ( ) ) .unwrap ( ) ;
38
config.auth.push_tokens = vec! [ " site" .into ( ) ] ;
39
config.backups.directories = vec! [ backups.to_path_buf ( ) ] ;
35fea423 Andy McCall 22 days ago ↶
40
if let Some ( g) = google {
41
config.backups.sts_url = g.to_string ( ) ;
42
config.backups.storage_url = g.to_string ( ) ;
43
}
d7d18246 Andy McCall 23 days ago ↶
44
let listener = tokio:: net:: TcpListener:: bind( " 127.0.0.1:0" ) .await.unwrap ( ) ;
45
let port = listener.local_addr ( ) .unwrap ( ) .port ( ) ;
46
config.server.base_url = format! ( " http://127.0.0.1:{port} " ) ;
47
let app = oxygenhub:: build( config) .await.unwrap ( ) ;
48
let router = oxygenhub:: router( app.clone ( ) ) ;
49
tokio:: spawn( async move { axum:: serve( listener, router) .await.unwrap ( ) } ) ;
50
Hub { app, base: format! ( " http://127.0.0.1:{port} " ) , _dir: dir }
51
}
52
53
fn write ( root : & Path, rel : & str , content : & str ) {
54
let p = root.join ( rel) ;
55
std:: fs:: create_dir_all( p.parent ( ) .unwrap ( ) ) .unwrap ( ) ;
56
std:: fs:: write( p, content) .unwrap ( ) ;
57
}
58
59
async fn create ( app : & Shared, owner : & str , name : & str , visibility : hub_api:: Visibility) {
60
app.repos.create ( & RepoName:: parse( & format! ( " {owner} /{name} " ) ) .unwrap ( ) , None ) .unwrap ( ) ;
61
db:: repos:: create( & app.db, owner, name, " a demo" , visibility) .await.unwrap ( ) ;
62
}
63
64
fn push ( url : & str , work : & Path, files : & [( &str , &str ) ]) {
65
let mut c = Repo:: init( work) .unwrap ( ) ;
66
c.config.set ( " remote.origin.url" , url) .unwrap ( ) ;
67
c.config.set ( " remote.origin.token" , " site" ) .unwrap ( ) ;
68
c.save_config ( ) .unwrap ( ) ;
69
for ( f, t) in files {
70
write ( work, f, t) ;
71
}
72
c.snapshot ( " t" ) .unwrap ( ) ;
73
c.commit ( " Some work" , & [ ] ) .unwrap ( ) ;
74
assert! ( c.push ( " origin" , false ) .unwrap ( ) .iter ( ) .all ( | r | r.ok ) ) ;
75
}
76
77
78
fn files ( dir : & Path) -> Vec< ( String, Vec< u8 > ) > {
79
let mut out = Vec :: new( ) ;
80
let mut stack = vec! [ dir.to_path_buf ( ) ] ;
81
while let Some ( d) = stack.pop ( ) {
82
for e in std:: fs:: read_dir( & d) .unwrap ( ) .flatten ( ) {
83
let p = e.path ( ) ;
84
if p.is_dir ( ) {
85
stack.push ( p) ;
86
} else {
87
out.push ( ( p.strip_prefix ( dir) .unwrap ( ) .to_string_lossy ( ) .into_owned ( ) , std:: fs:: read( & p) .unwrap ( ) ) ) ;
88
}
89
}
90
}
91
out.sort ( ) ;
92
out
93
}
94
95
# [ tokio ::test ( flavor = " multi_thread" ) ]
96
async fn a_backup_restores_the_same_repository_and_sends_only_what_changed ( ) {
97
let store = tempfile:: tempdir( ) .unwrap ( ) ;
98
let h = hub ( store.path ( ) ) .await;
99
db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
100
db:: accounts:: create_account( & h.app.db, " bob" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
101
create ( & h.app, " andy" , " demo" , hub_api:: Visibility:: Private) .await;
102
create ( & h.app, " bob" , " other" , hub_api:: Visibility:: Public) .await;
103
let work = tempfile:: tempdir( ) .unwrap ( ) ;
104
push ( & format! ( " {} /andy/demo" , h.base) , work.path ( ) , & [ ( " README.md" , " hello\n " ) , ( " src/main.rs" , " fn main() {}\n " ) ] ) ;
105
let other = tempfile:: tempdir( ) .unwrap ( ) ;
106
push ( & format! ( " {} /bob/other" , h.base) , other.path ( ) , & [ ( " x.txt" , " bob's\n " ) ] ) ;
107
108
109
let target = store.path ( ) .display ( ) .to_string ( ) ;
110
let mut s = Setting:: off( " andy" , " user" ) ;
111
s.every = " hourly" .into ( ) ;
112
s.target = target.clone ( ) ;
113
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
114
let first = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
115
assert_eq! ( ( first.status.as_str ( ) , first.repos) , ( " ok" , 1 ) , " {}" , first.detail) ;
116
assert! ( first.sent > 0 & & first.bytes > 0 ) ;
117
let dest = Directory { root: store.path ( ) .join ( " andy" ) } ;
118
assert! ( dest.exists ( & format! ( " runs/{} /DONE" , first.at) ) .unwrap ( ) ) ;
119
let records: serde_json:: Value = serde_json:: from_slice( & dest.get ( & format! ( " runs/{} /records/demo.json" , first.at) ) .unwrap ( ) ) .unwrap ( ) ;
120
assert_eq! ( records[ " repositories" ] [ 0 ] [ " visibility" ] , " private" ) ;
121
let text = String :: from_utf8_lossy( & dest.get ( & format! ( " runs/{} /records/demo.json" , first.at) ) .unwrap ( ) ) .into_owned ( ) ;
122
assert! ( ! text.contains ( " bob/other" ) & & ! text.contains ( " \" other\" " ) , " nothing of another owner" ) ;
123
assert! ( ! store.path ( ) .join ( " bob" ) .exists ( ) , " bob set up no backup" ) ;
124
125
126
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
127
let second = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
128
assert_eq! ( ( second.status.as_str ( ) , second.sent) , ( " ok" , 0 ) , " {}" , second.detail) ;
129
130
131
let empty = tempfile:: tempdir( ) .unwrap ( ) ;
132
let fresh = hub ( empty.path ( ) ) .await;
133
let dest: Arc< dyn Destination> = Arc:: new( Directory { root: store.path ( ) .join ( " andy" ) } ) ;
134
let done = backups:: restore_into_hub( & fresh.app, dest, None ) .await.unwrap ( ) ;
135
assert_eq! ( done, vec! [ " andy/demo (repos)" ] ) ;
136
let name = RepoName:: parse( " andy/demo" ) .unwrap ( ) ;
137
assert_eq! ( files ( & fresh.app.repos.dir ( & name) ) , files ( & h.app.repos.dir ( & name) ) , " byte for byte" ) ;
138
let row = db:: repos:: get( & fresh.app.db, " andy" , " demo" ) .await.unwrap ( ) .expect ( " listed on the hub" ) ;
139
assert_eq! ( ( row.visibility.as_str ( ) , row.description.as_str ( ) ) , ( " private" , " a demo" ) ) ;
140
let ( a, b) = ( h.app.repos.open ( & name) .unwrap ( ) , fresh.app.repos.open ( & name) .unwrap ( ) ) ;
141
assert_eq! (
142
a.bookmarks ( ) .unwrap ( ) .iter ( ) .map ( | b | b.heads.clone ( ) ) .collect:: < Vec< _ > > ( ) ,
143
b.bookmarks ( ) .unwrap ( ) .iter ( ) .map ( | b | b.heads.clone ( ) ) .collect:: < Vec< _ > > ( )
144
) ;
145
146
147
let mut bad = Setting:: off( " andy" , " user" ) ;
148
bad.every = " hourly" .into ( ) ;
149
bad.target = " /tmp/somewhere-else" .into ( ) ;
150
backups:: save( & h.app.db, & bad) .await.unwrap ( ) ;
151
let refused = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
152
assert_eq! ( refused.status, " failed" ) ;
153
assert! ( refused.detail.contains ( " not a backup directory this hub allows" ) , " {}" , refused.detail) ;
154
}
155
156
# [ tokio ::test ( flavor = " multi_thread" ) ]
157
async fn backups_fall_due_on_their_schedule ( ) {
158
let store = tempfile:: tempdir( ) .unwrap ( ) ;
159
let h = hub ( store.path ( ) ) .await;
160
let mut s = Setting:: off( " andy" , " user" ) ;
161
s.target = store.path ( ) .display ( ) .to_string ( ) ;
162
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
163
assert! ( backups:: due( & h.app.db) .await.unwrap ( ) .is_empty ( ) , " off is never due" ) ;
164
s.every = " hourly" .into ( ) ;
165
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
166
assert_eq! ( backups:: due( & h.app.db) .await.unwrap ( ) .len ( ) , 1 , " never run: due" ) ;
167
sqlx:: query( " INSERT INTO backup_runs (owner, started_ms, status) VALUES ('andy', ?, 'ok')" )
168
.bind ( db:: now_ms( ) )
169
.execute ( & h.app.db)
170
.await
171
.unwrap ( ) ;
172
assert! ( backups:: due( & h.app.db) .await.unwrap ( ) .is_empty ( ) , " ran just now" ) ;
35fea423 Andy McCall 22 days ago ↶
173
}
174
175
176
177
178
# [ derive ( Default ) ]
179
struct Google {
180
181
objects : std:: collections:: BTreeMap< ( String, String) , Vec< u8 > > ,
182
183
exchanged : Vec< ( String, String) > ,
184
}
185
186
type Shared2 = Arc< std:: sync:: Mutex< Google> > ;
187
188
fn decode ( s : & str ) -> String {
189
let b = s.as_bytes ( ) ;
190
let mut out = Vec :: new( ) ;
191
let mut i = 0 ;
192
while i < b.len ( ) {
193
if b[ i] = = b ' %' & & i + 2 < b.len ( ) {
194
out.push ( u8 :: from_str_radix( & s[ i + 1 .. i + 3 ] , 16 ) .unwrap ( ) ) ;
195
i + = 3 ;
196
} else {
197
out.push ( b[ i] ) ;
198
i + = 1 ;
199
}
200
}
201
String :: from_utf8( out) .unwrap ( )
202
}
203
204
fn query ( uri : & axum:: http:: Uri, key : & str ) -> Option< String> {
205
uri.query ( ) ? .split ( ' &' ) .find_map ( | kv | kv.split_once ( ' =' ) .filter ( | ( k , _) | * k = = key) .map ( | ( _, v ) | decode ( v) ) )
206
}
207
208
209
210
211
async fn fake_google ( ) -> ( String, Shared2) {
212
use axum:: extract:: { Request, State} ;
213
use axum:: http:: StatusCode;
214
use axum:: response:: IntoResponse;
215
let state: Shared2 = Default :: default( ) ;
216
async fn handle ( State( g ) : State< Shared2> , req : Request) -> axum:: response:: Response {
217
let ( parts, body) = req.into_parts ( ) ;
218
let bytes = http_body_util:: BodyExt:: collect( body) .await.unwrap ( ) .to_bytes ( ) ;
219
let path = parts.uri.path ( ) .to_string ( ) ;
220
if path = = " /v1/token" {
221
let v: serde_json:: Value = serde_json:: from_slice( & bytes) .unwrap ( ) ;
222
let jwt = v[ " subjectToken" ] .as_str ( ) .unwrap ( ) ;
223
let payload = jwt.split ( ' .' ) .nth ( 1 ) .unwrap ( ) ;
224
let claims: serde_json:: Value = serde_json:: from_slice( & base64url ( payload) ) .unwrap ( ) ;
225
let ( sub, aud) = ( claims[ " sub" ] .as_str ( ) .unwrap ( ) .to_string ( ) , claims[ " aud" ] .as_str ( ) .unwrap ( ) .to_string ( ) ) ;
226
let provider = v[ " audience" ] .as_str ( ) .unwrap ( ) ;
227
g.lock ( ) .unwrap ( ) .exchanged.push ( ( sub.clone ( ) , aud.clone ( ) ) ) ;
228
if ! sub.starts_with ( " backup:" ) | | aud ! = format! ( " https:{provider} " ) {
229
return (
230
StatusCode:: BAD_REQUEST ,
231
axum:: Json( serde_json:: json! ( { " error" : " invalid_grant" , " error_description" : " not admitted" } ) ) ,
232
)
233
.into_response ( ) ;
234
}
235
return axum:: Json( serde_json:: json! ( { " access_token" : format! ( " tok-{sub} " ) , " expires_in" : 3600 } ) ) .into_response ( ) ;
236
}
237
let auth = parts.headers.get ( " authorization" ) .and_then ( | v | v.to_str ( ) .ok ( ) ) .unwrap_or ( " " ) .to_string ( ) ;
238
let segs: Vec< & str > = path.split ( ' /' ) .collect ( ) ;
239
240
let upload = segs.get ( 1 ) = = Some ( & " upload" ) ;
241
let at = if upload { 5 } else { 4 } ;
242
let bucket = segs.get ( at) .unwrap_or ( & " " ) .to_string ( ) ;
243
let owner = bucket.trim_end_matches ( " -backup" ) ;
244
if auth ! = format! ( " Bearer tok-backup:{owner} " ) {
245
return ( StatusCode:: FORBIDDEN , " caller does not have storage.objects access" ) .into_response ( ) ;
246
}
247
let mut g = g.lock ( ) .unwrap ( ) ;
248
if upload {
0cd11874 Andy McCall 22 days ago ↶
249
250
if ! parts.headers.contains_key ( " content-length" ) {
251
return ( StatusCode:: LENGTH_REQUIRED , " Error 411 (Length Required)" ) .into_response ( ) ;
252
}
35fea423 Andy McCall 22 days ago ↶
253
let name = query ( & parts.uri, " name" ) .unwrap ( ) ;
254
g.objects.insert ( ( bucket, name) , bytes.to_vec ( ) ) ;
255
return axum:: Json( serde_json:: json! ( { } ) ) .into_response ( ) ;
256
}
257
let name = segs.get ( at + 2 ) .map ( | n | decode ( n) ) ;
258
match ( parts.method.as_str ( ) , name) {
259
( " GET" , Some ( n) ) => match g.objects.get ( & ( bucket, n.clone ( ) ) ) {
260
Some ( b) if query ( & parts.uri, " alt" ) .as_deref ( ) = = Some ( " media" ) => b.clone ( ) .into_response ( ) ,
261
Some ( _ ) => axum:: Json( serde_json:: json! ( { " name" : n } ) ) .into_response ( ) ,
262
None => StatusCode:: NOT_FOUND .into_response ( ) ,
263
} ,
264
( " DELETE" , Some ( n) ) => {
265
g.objects.remove ( & ( bucket, n) ) ;
266
StatusCode:: NO_CONTENT .into_response ( )
267
}
268
( " GET" , None ) => {
269
let prefix = query ( & parts.uri, " prefix" ) .unwrap_or_default ( ) ;
270
let delimited = query ( & parts.uri, " delimiter" ) .is_some ( ) ;
271
let ( mut items, mut prefixes) = ( Vec :: new( ) , std:: collections:: BTreeSet:: new( ) ) ;
272
for ( b, n) in g.objects.keys ( ) {
273
if * b ! = bucket | | ! n.starts_with ( & prefix) {
274
continue ;
275
}
276
let rest = & n[ prefix.len ( ) .. ] ;
277
match rest.find ( ' /' ) {
278
Some ( i) if delimited => {
279
prefixes.insert ( format! ( " {prefix} {} " , & rest[ .. = i] ) ) ;
280
}
281
_ => items.push ( serde_json:: json! ( { " name" : n } ) ) ,
282
}
283
}
284
axum:: Json( serde_json:: json! ( { " items" : items, " prefixes" : prefixes } ) ) .into_response ( )
285
}
286
_ => StatusCode:: METHOD_NOT_ALLOWED .into_response ( ) ,
287
}
288
}
289
let app = axum:: Router:: new( ) .fallback ( handle) .with_state ( state.clone ( ) ) ;
290
let listener = tokio:: net:: TcpListener:: bind( " 127.0.0.1:0" ) .await.unwrap ( ) ;
291
let port = listener.local_addr ( ) .unwrap ( ) .port ( ) ;
292
tokio:: spawn( async move { axum:: serve( listener, app) .await.unwrap ( ) } ) ;
293
( format! ( " http://127.0.0.1:{port} " ) , state)
294
}
295
296
fn base64url ( s : & str ) -> Vec< u8 > {
297
let table = b " ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" ;
298
let mut bits = 0 u32 ;
299
let mut n = 0 ;
300
let mut out = Vec :: new( ) ;
301
for c in s.bytes ( ) {
302
let v = table.iter ( ) .position ( | t | * t = = c) .unwrap ( ) as u32 ;
303
bits = ( bits < < 6 ) | v;
304
n + = 6 ;
305
if n > = 8 {
306
n - = 8 ;
307
out.push ( ( bits > > n) as u8 ) ;
308
bits & = ( 1 < < n) - 1 ;
309
}
310
}
311
out
312
}
313
314
const PROVIDER : & str = " //iam.googleapis.com/projects/1/locations/global/workloadIdentityPools/oxygenhub/providers/oxygenhub-io" ;
315
316
fn gcs ( owner : & str , bucket : & str ) -> Setting {
317
let mut s = Setting:: off( owner, " user" ) ;
318
s.every = " hourly" .into ( ) ;
319
s.destination = " gcs" .into ( ) ;
320
s.target = bucket.into ( ) ;
321
s.provider = PROVIDER .into ( ) ;
322
s
323
}
324
325
# [ tokio ::test ( flavor = " multi_thread" ) ]
326
async fn a_gcs_backup_needs_no_key_and_restores_the_same_repository ( ) {
327
let ( google, seen) = fake_google ( ) .await;
328
let store = tempfile:: tempdir( ) .unwrap ( ) ;
329
let h = hub_with ( store.path ( ) , Some ( & google) ) .await;
330
db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
331
db:: accounts:: create_account( & h.app.db, " bob" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
332
create ( & h.app, " andy" , " demo" , hub_api:: Visibility:: Public) .await;
333
let work = tempfile:: tempdir( ) .unwrap ( ) ;
334
push ( & format! ( " {} /andy/demo" , h.base) , work.path ( ) , & [ ( " README.md" , " hello\n " ) ] ) ;
0cd11874 Andy McCall 22 days ago ↶
335
336
337
std:: fs:: write( h.app.repos.dir ( & RepoName:: parse( " andy/demo" ) .unwrap ( ) ) .join ( " .oxy/empty-marker" ) , b " " ) .unwrap ( ) ;
35fea423 Andy McCall 22 days ago ↶
338
339
backups:: save( & h.app.db, & gcs ( " andy" , " andy-backup" ) ) .await.unwrap ( ) ;
340
let first = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
341
assert_eq! ( ( first.status.as_str ( ) , first.repos) , ( " ok" , 1 ) , " {}" , first.detail) ;
342
{
343
let g = seen.lock ( ) .unwrap ( ) ;
344
assert! ( g.objects.keys ( ) .any ( | ( b , n ) | b = = " andy-backup" & & n = = & format! ( " runs/{} /DONE" , first.at) ) ) ;
345
assert! ( g.objects.keys ( ) .all ( | ( b , _) | b = = " andy-backup" ) ) ;
346
let ( sub, aud) = g.exchanged.first ( ) .unwrap ( ) ;
347
assert_eq! (
348
( sub.as_str ( ) , aud.as_str ( ) ) ,
349
( " backup:andy" , format! ( " https:{PROVIDER} " ) .as_str ( ) ) ,
350
" the hub signs in as the owner, for the provider"
351
) ;
352
}
353
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
354
let second = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
355
assert_eq! ( ( second.status.as_str ( ) , second.sent) , ( " ok" , 0 ) , " {}" , second.detail) ;
356
357
let before = seen.lock ( ) .unwrap ( ) .objects.len ( ) ;
358
let dest = backups:: destination( & h.app, & gcs ( " andy" , " andy-backup" ) ) .unwrap ( ) ;
359
tokio:: task:: spawn_blocking( move | | backups:: probe( dest.as_ref ( ) ) ) .await.unwrap ( ) .unwrap ( ) ;
360
assert_eq! ( seen.lock ( ) .unwrap ( ) .objects.len ( ) , before) ;
361
362
363
let empty = tempfile:: tempdir( ) .unwrap ( ) ;
364
let fresh = hub_with ( empty.path ( ) , Some ( & google) ) .await;
365
let dest: Arc< dyn Destination> = backups:: destination( & fresh.app, & gcs ( " andy" , " andy-backup" ) ) .unwrap ( ) .into ( ) ;
366
assert_eq! ( backups:: restore_into_hub( & fresh.app, dest, None ) .await.unwrap ( ) , vec! [ " andy/demo (repos)" ] ) ;
367
let name = RepoName:: parse( " andy/demo" ) .unwrap ( ) ;
368
assert_eq! ( files ( & fresh.app.repos.dir ( & name) ) , files ( & h.app.repos.dir ( & name) ) , " byte for byte" ) ;
369
370
371
372
backups:: save( & h.app.db, & gcs ( " bob" , " andy-backup" ) ) .await.unwrap ( ) ;
373
let refused = backups:: run( & h.app, " bob" , " now" ) .await.unwrap ( ) ;
374
assert_eq! ( refused.status, " failed" ) ;
375
assert! ( refused.detail.contains ( " roles/storage.objectAdmin" ) & & refused.detail.contains ( " backup:bob" ) , " {}" , refused.detail) ;
d7d18246 Andy McCall 23 days ago ↶
376
}
5ef6cf72 Andy McCall 22 days ago ↶
377
378
379
380
381
382
383
# [ tokio ::test ( flavor = " multi_thread" ) ]
384
async fn a_restore_brings_back_issues_proposals_and_releases ( ) {
385
let store = tempfile:: tempdir( ) .unwrap ( ) ;
386
let h = hub ( store.path ( ) ) .await;
387
let andy = db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
388
let bob = db:: accounts:: create_account( & h.app.db, " bob" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
389
create ( & h.app, " andy" , " demo" , hub_api:: Visibility:: Public) .await;
390
let work = tempfile:: tempdir( ) .unwrap ( ) ;
391
push ( & format! ( " {} /andy/demo" , h.base) , work.path ( ) , & [ ( " README.md" , " hello\n " ) ] ) ;
392
let row = db:: repos:: get( & h.app.db, " andy" , " demo" ) .await.unwrap ( ) .unwrap ( ) ;
393
db:: repos:: set_diffusion( & h.app.db, row.id, " green" ) .await.unwrap ( ) ;
394
395
for n in 0 .. 3 {
396
db:: issues:: upsert_label( & h.app.db, row.id, & format! ( " spare{n} " ) , " #000000" , " " ) .await.unwrap ( ) ;
397
}
398
let bug = db:: issues:: upsert_label( & h.app.db, row.id, " bug" , " #d73a4a" , " Something is wrong" ) .await.unwrap ( ) ;
399
let m = db:: issues:: upsert_milestone( & h.app.db, row.id, " 1.0" , " the first" , None ) .await.unwrap ( ) ;
400
let issue = db:: issues:: create(
401
& h.app.db,
402
db:: issues:: NewIssue { repo_id: row.id, title: " Sprites flicker" , body: " on level two" , author_id: Some ( andy.id) , author: " andy" } ,
403
)
404
.await
405
.unwrap ( ) ;
406
db:: issues:: add_comment( & h.app.db, issue.id, Some ( bob.id) , " bob" , " seen it too" ) .await.unwrap ( ) ;
407
db:: issues:: set_issue_label( & h.app.db, issue.id, bug.id, true ) .await.unwrap ( ) ;
408
db:: issues:: set_milestone( & h.app.db, issue.id, Some ( m.id) ) .await.unwrap ( ) ;
409
db:: issues:: set_assignee( & h.app.db, issue.id, andy.id, true ) .await.unwrap ( ) ;
410
db:: issues:: set_assignee( & h.app.db, issue.id, bob.id, true ) .await.unwrap ( ) ;
411
let p = db:: proposals:: create(
412
& h.app.db,
413
db:: proposals:: NewProposal {
414
repo_id: row.id,
415
title: " Double-buffer the sprites" ,
416
body: " fixes the flicker" ,
417
author_id: Some ( bob.id) ,
418
author: " bob" ,
419
draft: false ,
420
target: " main" ,
421
source_kind: " bookmark" ,
422
source_repo_id: Some ( row.id) ,
423
source_bookmark: " fix" ,
424
source_changes: " " ,
425
} ,
426
)
427
.await
428
.unwrap ( ) ;
429
db:: proposals:: add_event( & h.app.db, p.id, " bob" , " opened" , " " ) .await.unwrap ( ) ;
430
db:: proposals:: add_review( & h.app.db, p.id, andy.id, " andy" , " approve" , " nice" ) .await.unwrap ( ) ;
431
let release = db:: releases:: create( & h.app.db, row.id, " v1.0" , " One" , " notes" , false , Some ( ( andy.id, " andy" ) ) ) .await.unwrap ( ) ;
432
let asset = b " a game image" .to_vec ( ) ;
433
let hash = blake3:: hash( & asset) .to_hex ( ) .to_string ( ) ;
434
let path = h.app.config.data.join ( " assets" ) .join ( & hash[ .. 2 ] ) .join ( & hash) ;
435
std:: fs:: create_dir_all( path.parent ( ) .unwrap ( ) ) .unwrap ( ) ;
436
std:: fs:: write( & path, & asset) .unwrap ( ) ;
437
db:: releases:: add_asset( & h.app.db, release.id, " game.prg" , asset.len ( ) as i64 , & hash, " application/octet-stream" ) .await.unwrap ( ) ;
438
db:: hooks:: create_hook( & h.app.db, Some ( row.id) , None , " https://example.com/hook" , " s3cret-value" , " push" ) .await.unwrap ( ) ;
439
440
let mut s = Setting:: off( " andy" , " user" ) ;
441
s.every = " hourly" .into ( ) ;
442
s.target = store.path ( ) .display ( ) .to_string ( ) ;
443
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
444
let run = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
445
assert_eq! ( run.status, " ok" , " {}" , run.detail) ;
446
let dest = Directory { root: store.path ( ) .join ( " andy" ) } ;
447
let text = String :: from_utf8_lossy( & dest.get ( & format! ( " runs/{} /records/demo.json" , run.at) ) .unwrap ( ) ) .into_owned ( ) ;
448
assert! ( ! text.contains ( " s3cret-value" ) , " never a secret" ) ;
449
assert! ( text.contains ( " \" _names\" " ) & & text.contains ( " \" bob\" " ) , " accounts by name" ) ;
450
451
452
let empty = tempfile:: tempdir( ) .unwrap ( ) ;
453
let fresh = hub ( empty.path ( ) ) .await;
454
let andy2 = db:: accounts:: create_account( & fresh.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
455
let dest: Arc< dyn Destination> = Arc:: new( Directory { root: store.path ( ) .join ( " andy" ) } ) ;
456
let done = backups:: restore_into_hub( & fresh.app, dest, None ) .await.unwrap ( ) ;
457
assert_eq! ( done[ 0 ] , " andy/demo (repos)" ) ;
458
let notes = done[ 1 .. ] .join ( " \n " ) ;
459
assert! ( notes.contains ( " webhook https://example.com/hook: enter its secret again" ) , " {notes}" ) ;
460
assert! ( notes.contains ( " issue_assignees: 1 row(s) left out" ) , " bob is not on this hub: {notes}" ) ;
461
let r2 = db:: repos:: get( & fresh.app.db, " andy" , " demo" ) .await.unwrap ( ) .unwrap ( ) ;
462
assert_eq! ( r2.diffusion, " green" , " settings come back" ) ;
463
464
let issues: Vec< ( i64 , i64 , String, Option< i64 > ) > = sqlx:: query_as( " SELECT id, number, title, author_id FROM issues WHERE repo_id = ?" )
465
.bind ( r2.id)
466
.fetch_all ( & fresh.app.db)
467
.await
468
.unwrap ( ) ;
469
assert_eq! ( issues.len ( ) , 1 ) ;
470
let ( iid, number, title, author) = issues[ 0 ] .clone ( ) ;
471
assert_eq! ( ( number, title.as_str ( ) , author) , ( issue.number, " Sprites flicker" , Some ( andy2.id) ) ) ;
472
let comments = db:: issues:: comments( & fresh.app.db, iid) .await.unwrap ( ) ;
473
assert_eq! ( comments.len ( ) , 1 ) ;
474
assert_eq! ( ( comments[ 0 ] .author.as_str ( ) , comments[ 0 ] .body.as_str ( ) ) , ( " bob" , " seen it too" ) ) ;
475
let labels: Vec< String> =
476
sqlx:: query_scalar( " SELECT l.name FROM issue_labels il JOIN labels l ON l.id = il.label_id WHERE il.issue_id = ?" )
477
.bind ( iid)
478
.fetch_all ( & fresh.app.db)
479
.await
480
.unwrap ( ) ;
481
assert_eq! ( labels, vec! [ " bug" ] , " the label, its id mapped" ) ;
482
let milestone: Option< String> =
483
sqlx:: query_scalar( " SELECT m.title FROM issues i JOIN milestones m ON m.id = i.milestone_id WHERE i.id = ?" )
484
.bind ( iid)
485
.fetch_one ( & fresh.app.db)
486
.await
487
.unwrap ( ) ;
488
assert_eq! ( milestone.as_deref ( ) , Some ( " 1.0" ) ) ;
489
assert_eq! ( db:: issues:: assignees( & fresh.app.db, iid) .await.unwrap ( ) , vec! [ " andy" ] ) ;
490
491
let p2 = db:: proposals:: get( & fresh.app.db, r2.id, p.number) .await.unwrap ( ) .expect ( " the proposal" ) ;
492
assert_eq! ( ( p2.title.as_str ( ) , p2.author.as_str ( ) , p2.author_id) , ( " Double-buffer the sprites" , " bob" , None ) ) ;
493
let reviews: Vec< ( String, String) > = sqlx:: query_as( " SELECT author, verdict FROM proposal_reviews WHERE proposal_id = ?" )
494
.bind ( p2.id)
495
.fetch_all ( & fresh.app.db)
496
.await
497
.unwrap ( ) ;
498
assert_eq! ( reviews, vec! [ ( " andy" .to_string ( ) , " approve" .to_string ( ) ) ] ) ;
499
assert! ( db:: proposals:: events( & fresh.app.db, p2.id) .await.unwrap ( ) .iter ( ) .any ( | e | e.event = = " opened" ) ) ;
500
501
let rel = db:: releases:: get( & fresh.app.db, r2.id, " v1.0" ) .await.unwrap ( ) .expect ( " the release" ) ;
502
let assets = db:: releases:: assets( & fresh.app.db, rel.id) .await.unwrap ( ) ;
503
assert_eq! ( assets.len ( ) , 1 ) ;
504
assert_eq! ( std:: fs:: read( fresh.app.config.data.join ( " assets" ) .join ( & hash[ .. 2 ] ) .join ( & hash) ) .unwrap ( ) , asset) ;
505
506
let hits = db:: notify:: search( & fresh.app.db, " flicker" , Some ( r2.id) , 10 ) .await.unwrap ( ) ;
507
assert! ( hits.iter ( ) .any ( | h | h.kind = = " issue" & & h.target_id = = iid ) , " {:?}" , hits.iter ( ) .map ( | h | & h.title) .collect:: < Vec< _ > > ( ) ) ;
508
509
let next = db:: issues:: create(
510
& fresh.app.db,
511
db:: issues:: NewIssue { repo_id: r2.id, title: " After" , body: " " , author_id: Some ( andy2.id) , author: " andy" } ,
512
)
513
.await
514
.unwrap ( ) ;
515
assert! ( next.number > p.number, " {} after {}" , next.number, p.number) ;
516
}
d3b048b2 Andy McCall 22 days ago ↶
517
518
519
520
521
522
# [ tokio ::test ( flavor = " multi_thread" ) ]
523
async fn repositories_choose_their_own_backup ( ) {
524
let store = tempfile:: tempdir( ) .unwrap ( ) ;
525
let h = hub ( store.path ( ) ) .await;
526
db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
527
for n in [ " game" , " notes" , " secret" ] {
528
create ( & h.app, " andy" , n, hub_api:: Visibility:: Public) .await;
529
let work = tempfile:: tempdir( ) .unwrap ( ) ;
530
push ( & format! ( " {} /andy/{n} " , h.base) , work.path ( ) , & [ ( " README.md" , n) ] ) ;
531
}
532
let id = | n : & 'static str | {
533
let db = h.app.db.clone ( ) ;
534
async move { db:: repos:: get( & db, " andy" , n) .await.unwrap ( ) .unwrap ( ) .id }
535
} ;
536
db:: repos:: set_backup( & h.app.db, id ( " secret" ) .await, " off" ) .await.unwrap ( ) ;
537
let mut s = Setting:: off( " andy" , " user" ) ;
538
s.every = " daily" .into ( ) ;
539
s.target = store.path ( ) .display ( ) .to_string ( ) ;
540
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
541
let names = | at : String| {
542
let dest = Directory { root: store.path ( ) .join ( " andy" ) } ;
543
let mut v = dest.list ( & format! ( " runs/{at} /records" ) ) .unwrap ( ) ;
544
v.sort ( ) ;
545
v
546
} ;
547
let first = backups:: run( & h.app, " andy" , " schedule" ) .await.unwrap ( ) ;
548
assert_eq! ( ( first.status.as_str ( ) , first.repos) , ( " ok" , 2 ) , " {}" , first.detail) ;
549
assert_eq! ( names ( first.at.clone ( ) ) , vec! [ " game.json" , " notes.json" ] , " the one left out is not in it" ) ;
550
551
assert! ( backups:: due( & h.app.db) .await.unwrap ( ) .is_empty ( ) ) ;
552
db:: repos:: set_backup( & h.app.db, id ( " game" ) .await, " hourly" ) .await.unwrap ( ) ;
553
sqlx:: query( " UPDATE backup_runs SET started_ms = started_ms - 2 * 3600 * 1000" ) .execute ( & h.app.db) .await.unwrap ( ) ;
554
assert_eq! ( backups:: due( & h.app.db) .await.unwrap ( ) .len ( ) , 1 , " hourly for the game" ) ;
555
556
s.every = " off" .into ( ) ;
557
backups:: save( & h.app.db, & s) .await.unwrap ( ) ;
558
assert_eq! ( backups:: due( & h.app.db) .await.unwrap ( ) .len ( ) , 1 ) ;
559
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
560
let second = backups:: run( & h.app, " andy" , " schedule" ) .await.unwrap ( ) ;
561
assert_eq! ( names ( second.at.clone ( ) ) , vec! [ " game.json" ] ) ;
562
563
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
564
let third = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
565
assert_eq! ( names ( third.at.clone ( ) ) , vec! [ " game.json" , " notes.json" ] ) ;
566
567
let ( owners, uncovered) = backups:: overview( & h.app.db) .await.unwrap ( ) ;
568
assert_eq! ( ( owners[ 0 ] .repos, uncovered.len ( ) ) , ( 1 , 0 ) ) ;
569
}
7b9f4570 Andy McCall 22 days ago ↶
570
571
572
573
574
575
async fn fake_aws ( ) -> ( String, Shared2) {
576
use axum:: extract:: { Request, State} ;
577
use axum:: http:: StatusCode;
578
use axum:: response:: IntoResponse;
579
let state: Shared2 = Default :: default( ) ;
580
async fn handle ( State( g ) : State< Shared2> , req : Request) -> axum:: response:: Response {
581
let ( parts, body) = req.into_parts ( ) ;
582
let bytes = http_body_util:: BodyExt:: collect( body) .await.unwrap ( ) .to_bytes ( ) ;
583
let path = parts.uri.path ( ) .to_string ( ) ;
584
let header = | h : & str | parts.headers.get ( h) .and_then ( | v | v.to_str ( ) .ok ( ) ) .unwrap_or ( " " ) .to_string ( ) ;
585
if path = = " /" & & parts.method = = " POST" {
586
let form: Vec< ( String, String) > = String :: from_utf8_lossy( & bytes)
587
.split ( ' &' )
588
.filter_map ( | kv | kv.split_once ( ' =' ) .map ( | ( k , v ) | ( k.to_string ( ) , decode ( & v.replace ( ' +' , " " ) ) ) ) )
589
.collect ( ) ;
590
let field = | k : & str | form.iter ( ) .find ( | ( f , _) | f = = k ) .map ( | ( _, v ) | v.clone ( ) ) .unwrap_or_default ( ) ;
591
let jwt = field ( " WebIdentityToken" ) ;
592
let claims: serde_json:: Value = serde_json:: from_slice( & base64url ( jwt.split ( ' .' ) .nth ( 1 ) .unwrap ( ) ) ) .unwrap ( ) ;
593
let ( sub, aud) = ( claims[ " sub" ] .as_str ( ) .unwrap ( ) .to_string ( ) , claims[ " aud" ] .as_str ( ) .unwrap ( ) .to_string ( ) ) ;
594
g.lock ( ) .unwrap ( ) .exchanged.push ( ( sub.clone ( ) , aud.clone ( ) ) ) ;
595
if field ( " Action" ) ! = " AssumeRoleWithWebIdentity" | | ! sub.starts_with ( " backup:" ) | | aud ! = " sts.amazonaws.com" {
596
return ( StatusCode:: FORBIDDEN , " <ErrorResponse><Error><Code>AccessDenied</Code><Message>Not authorized to perform sts:AssumeRoleWithWebIdentity</Message></Error></ErrorResponse>" ) .into_response ( ) ;
597
}
598
let owner = sub.trim_start_matches ( " backup:" ) ;
599
return format! (
600
" <AssumeRoleWithWebIdentityResponse><AssumeRoleWithWebIdentityResult><Credentials>\
601
<AccessKeyId>ASIA{owner} </AccessKeyId><SecretAccessKey>secret/{owner} </SecretAccessKey>\
602
<SessionToken>session-{owner} </SessionToken><Expiration>2030-01-01T00:00:00Z</Expiration>\
603
</Credentials></AssumeRoleWithWebIdentityResult></AssumeRoleWithWebIdentityResponse>"
604
)
605
.into_response ( ) ;
606
}
607
608
let rest = path.strip_prefix ( " /s3/" ) .unwrap_or ( " " ) ;
609
let ( bucket, key) = rest.split_once ( ' /' ) .unwrap_or ( ( rest, " " ) ) ;
610
let ( bucket, key) = ( bucket.to_string ( ) , decode ( key) ) ;
611
let owner = bucket.trim_end_matches ( " -backup" ) .to_string ( ) ;
612
613
let session = header ( " x-amz-security-token" ) ;
614
let who = session.trim_start_matches ( " session-" ) .to_string ( ) ;
615
let query: Vec< ( String, String) > = parts
616
.uri
617
.query ( )
618
.unwrap_or ( " " )
619
.split ( ' &' )
620
.filter ( | kv | ! kv.is_empty ( ) )
621
.map ( | kv | kv.split_once ( ' =' ) .map ( | ( k , v ) | ( decode ( k) , decode ( v) ) ) .unwrap_or ( ( decode ( kv) , String :: new( ) ) ) )
622
.collect ( ) ;
623
let signed = vec! [
624
( " host" .to_string ( ) , header ( " host" ) ) ,
625
( " x-amz-content-sha256" .to_string ( ) , header ( " x-amz-content-sha256" ) ) ,
626
( " x-amz-date" .to_string ( ) , header ( " x-amz-date" ) ) ,
627
( " x-amz-security-token" .to_string ( ) , session.clone ( ) ) ,
628
] ;
629
let id = format! ( " ASIA{who} " ) ;
630
let secret = format! ( " secret/{who} " ) ;
631
let want = oxygenhub:: backups:: sigv4(
632
parts.method.as_str ( ) ,
633
& path,
634
& query,
635
& signed,
636
& header ( " x-amz-content-sha256" ) ,
637
( & id, & secret) ,
638
" eu-west-2" ,
639
" s3" ,
640
& header ( " x-amz-date" ) ,
641
) ;
642
use sha2:: Digest;
643
let body_hash = hex:: encode( sha2:: Sha256:: digest( & bytes) ) ;
644
if header ( " authorization" ) ! = want | | header ( " x-amz-content-sha256" ) ! = body_hash {
645
return (
646
StatusCode:: FORBIDDEN ,
647
" <Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>" ,
648
)
649
.into_response ( ) ;
650
}
651
if who ! = owner {
652
return ( StatusCode:: FORBIDDEN , " <Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>" ) .into_response ( ) ;
653
}
654
let mut g = g.lock ( ) .unwrap ( ) ;
655
match ( parts.method.as_str ( ) , key.is_empty ( ) ) {
0cd11874 Andy McCall 22 days ago ↶
656
( " PUT" , false ) if ! parts.headers.contains_key ( " content-length" ) => {
657
( StatusCode:: LENGTH_REQUIRED , " <Error><Code>MissingContentLength</Code></Error>" ) .into_response ( )
658
}
7b9f4570 Andy McCall 22 days ago ↶
659
( " PUT" , false ) => {
660
g.objects.insert ( ( bucket, key) , bytes.to_vec ( ) ) ;
661
StatusCode:: OK .into_response ( )
662
}
663
( " GET" , false ) | ( " HEAD" , false ) => match g.objects.get ( & ( bucket, key) ) {
664
Some ( b) => b.clone ( ) .into_response ( ) ,
665
None => ( StatusCode:: NOT_FOUND , " <Error><Code>NoSuchKey</Code></Error>" ) .into_response ( ) ,
666
} ,
667
( " DELETE" , false ) => {
668
g.objects.remove ( & ( bucket, key) ) ;
669
StatusCode:: NO_CONTENT .into_response ( )
670
}
671
( " GET" , true ) => {
672
let prefix = query.iter ( ) .find ( | ( k , _) | k = = " prefix" ) .map ( | ( _, v ) | v.clone ( ) ) .unwrap_or_default ( ) ;
673
let delimited = query.iter ( ) .any ( | ( k , _) | k = = " delimiter" ) ;
674
let ( mut keys, mut prefixes) = ( String :: new( ) , std:: collections:: BTreeSet:: new( ) ) ;
675
for ( b, n) in g.objects.keys ( ) {
676
if * b ! = bucket | | ! n.starts_with ( & prefix) {
677
continue ;
678
}
679
match n[ prefix.len ( ) .. ] .find ( ' /' ) {
680
Some ( i) if delimited => {
681
prefixes.insert ( format! ( " {prefix} {} " , & n[ prefix.len ( ) .. prefix.len ( ) + i + 1 ] ) ) ;
682
}
683
_ => keys.push_str ( & format! ( " <Contents><Key>{n} </Key></Contents>" ) ) ,
684
}
685
}
686
let common: String = prefixes.iter ( ) .map ( | p | format! ( " <CommonPrefixes><Prefix>{p} </Prefix></CommonPrefixes>" ) ) .collect ( ) ;
687
format! ( " <ListBucketResult><Prefix>{prefix} </Prefix><IsTruncated>false</IsTruncated>{keys} {common} </ListBucketResult>" )
688
.into_response ( )
689
}
690
_ => StatusCode:: METHOD_NOT_ALLOWED .into_response ( ) ,
691
}
692
}
693
let app = axum:: Router:: new( ) .fallback ( handle) .with_state ( state.clone ( ) ) ;
694
let listener = tokio:: net:: TcpListener:: bind( " 127.0.0.1:0" ) .await.unwrap ( ) ;
695
let port = listener.local_addr ( ) .unwrap ( ) .port ( ) ;
696
tokio:: spawn( async move { axum:: serve( listener, app) .await.unwrap ( ) } ) ;
697
( format! ( " http://127.0.0.1:{port} " ) , state)
698
}
699
700
async fn hub_aws ( backups : & Path, aws : & str ) -> Hub {
701
hermetic ( ) ;
702
let dir = tempfile:: tempdir( ) .unwrap ( ) ;
703
let mut config = Config:: load( dir.path ( ) ) .unwrap ( ) ;
704
config.auth.push_tokens = vec! [ " site" .into ( ) ] ;
705
config.backups.directories = vec! [ backups.to_path_buf ( ) ] ;
706
config.backups.aws_sts_url = aws.to_string ( ) ;
707
config.backups.s3_url = format! ( " {aws} /s3/{{ bucket}} " ) ;
708
let listener = tokio:: net:: TcpListener:: bind( " 127.0.0.1:0" ) .await.unwrap ( ) ;
709
let port = listener.local_addr ( ) .unwrap ( ) .port ( ) ;
710
config.server.base_url = format! ( " http://127.0.0.1:{port} " ) ;
711
let app = oxygenhub:: build( config) .await.unwrap ( ) ;
712
let router = oxygenhub:: router( app.clone ( ) ) ;
713
tokio:: spawn( async move { axum:: serve( listener, router) .await.unwrap ( ) } ) ;
714
Hub { app, base: format! ( " http://127.0.0.1:{port} " ) , _dir: dir }
715
}
716
717
const ROLE : & str = " arn:aws:iam::123456789012:role/oxygenhub-backup" ;
718
719
fn s3 ( owner : & str , bucket : & str ) -> Setting {
720
let mut s = Setting:: off( owner, " user" ) ;
721
s.every = " hourly" .into ( ) ;
722
s.destination = " s3" .into ( ) ;
723
s.target = bucket.into ( ) ;
724
s.provider = ROLE .into ( ) ;
725
s.region = " eu-west-2" .into ( ) ;
726
s.prefix = " oxygen hub" .into ( ) ;
727
s
728
}
729
730
731
732
733
734
735
# [ tokio ::test ( flavor = " multi_thread" ) ]
736
async fn an_s3_backup_needs_no_key_and_restores_the_same_repository ( ) {
737
let ( aws, seen) = fake_aws ( ) .await;
738
let store = tempfile:: tempdir( ) .unwrap ( ) ;
739
let h = hub_aws ( store.path ( ) , & aws) .await;
740
db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
741
db:: accounts:: create_account( & h.app.db, " bob" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
742
create ( & h.app, " andy" , " demo" , hub_api:: Visibility:: Public) .await;
743
let work = tempfile:: tempdir( ) .unwrap ( ) ;
744
push ( & format! ( " {} /andy/demo" , h.base) , work.path ( ) , & [ ( " README.md" , " hello\n " ) , ( " src/a b+c.rs" , " fn main() {}\n " ) ] ) ;
0cd11874 Andy McCall 22 days ago ↶
745
std:: fs:: write( h.app.repos.dir ( & RepoName:: parse( " andy/demo" ) .unwrap ( ) ) .join ( " .oxy/empty-marker" ) , b " " ) .unwrap ( ) ;
7b9f4570 Andy McCall 22 days ago ↶
746
747
backups:: save( & h.app.db, & s3 ( " andy" , " andy-backup" ) ) .await.unwrap ( ) ;
748
let first = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
749
assert_eq! ( ( first.status.as_str ( ) , first.repos) , ( " ok" , 1 ) , " {}" , first.detail) ;
750
{
751
let g = seen.lock ( ) .unwrap ( ) ;
752
assert! (
753
g.objects.keys ( ) .any ( | ( b , n ) | b = = " andy-backup" & & n = = & format! ( " oxygen hub/runs/{} /DONE" , first.at) ) ,
754
" under the prefix"
755
) ;
756
assert_eq! ( g.exchanged.first ( ) .map ( | ( s , a ) | ( s.as_str ( ) , a.as_str ( ) ) ) , Some ( ( " backup:andy" , " sts.amazonaws.com" ) ) ) ;
757
}
758
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
759
let second = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
760
assert_eq! ( ( second.status.as_str ( ) , second.sent) , ( " ok" , 0 ) , " {}" , second.detail) ;
761
let before = seen.lock ( ) .unwrap ( ) .objects.len ( ) ;
762
let dest = backups:: destination( & h.app, & s3 ( " andy" , " andy-backup" ) ) .unwrap ( ) ;
763
tokio:: task:: spawn_blocking( move | | backups:: probe( dest.as_ref ( ) ) ) .await.unwrap ( ) .unwrap ( ) ;
764
assert_eq! ( seen.lock ( ) .unwrap ( ) .objects.len ( ) , before, " the probe cleans up" ) ;
765
766
let empty = tempfile:: tempdir( ) .unwrap ( ) ;
767
let fresh = hub_aws ( empty.path ( ) , & aws) .await;
768
let dest: Arc< dyn Destination> = backups:: destination( & fresh.app, & s3 ( " andy" , " andy-backup" ) ) .unwrap ( ) .into ( ) ;
769
assert_eq! ( backups:: restore_into_hub( & fresh.app, dest, None ) .await.unwrap ( ) , vec! [ " andy/demo (repos)" ] ) ;
770
let name = RepoName:: parse( " andy/demo" ) .unwrap ( ) ;
771
assert_eq! ( files ( & fresh.app.repos.dir ( & name) ) , files ( & h.app.repos.dir ( & name) ) , " byte for byte" ) ;
772
773
backups:: save( & h.app.db, & s3 ( " bob" , " andy-backup" ) ) .await.unwrap ( ) ;
774
let refused = backups:: run( & h.app, " bob" , " now" ) .await.unwrap ( ) ;
775
assert_eq! ( refused.status, " failed" ) ;
776
assert! ( refused.detail.contains ( " s3:PutObject" ) & & refused.detail.contains ( ROLE ) , " {}" , refused.detail) ;
777
}
2e9188ca Andy McCall 22 days ago ↶
778
779
780
async fn api (
781
app : & Shared,
782
method : & str ,
783
path : & str ,
784
token : & str ,
785
body : Option< serde_json:: Value> ,
786
) -> ( axum:: http:: StatusCode, serde_json:: Value) {
787
use axum:: body:: Body;
788
use http_body_util:: BodyExt;
789
use tower:: ServiceExt;
790
let mut req =
791
axum:: http:: Request:: builder( ) .method ( method) .uri ( format! ( " /api/v1{path} " ) ) .header ( " authorization" , format! ( " Bearer {token} " ) ) ;
792
let body = match body {
793
Some ( b) => {
794
req = req.header ( " content-type" , " application/json" ) ;
795
Body:: from( b.to_string ( ) )
796
}
797
None => Body:: empty( ) ,
798
} ;
799
let res = oxygenhub:: router( app.clone ( ) ) .oneshot ( req.body ( body) .unwrap ( ) ) .await.unwrap ( ) ;
800
let st = res.status ( ) ;
801
let bytes = res.into_body ( ) .collect ( ) .await.unwrap ( ) .to_bytes ( ) ;
802
( st, serde_json:: from_slice( & bytes) .unwrap_or ( serde_json:: Value:: Null) )
803
}
804
805
806
807
808
# [ tokio ::test ( flavor = " multi_thread" ) ]
809
async fn backups_are_set_up_and_run_through_the_api ( ) {
810
let store = tempfile:: tempdir( ) .unwrap ( ) ;
811
let h = hub ( store.path ( ) ) .await;
812
let andy = db:: accounts:: create_account( & h.app.db, " andy" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
813
let bob = db:: accounts:: create_account( & h.app.db, " bob" , " [email protected] " , " x" , " salt" , false ) .await.unwrap ( ) ;
814
db:: accounts:: create_org( & h.app.db, " club" , " Club" , andy.id) .await.unwrap ( ) ;
815
create ( & h.app, " andy" , " demo" , hub_api:: Visibility:: Public) .await;
816
create ( & h.app, " club" , " game" , hub_api:: Visibility:: Private) .await;
817
let token = | id : i64 , scopes : & 'static str | {
818
let db = h.app.db.clone ( ) ;
819
async move {
820
let ( t, hash) = oxygenhub:: auth:: new_token( ) ;
821
db:: accounts:: create_token( & db, id, " t" , & hash, scopes, " " , None ) .await.unwrap ( ) ;
822
t
823
}
824
} ;
825
let ( andy_t , bob_t , weak) = ( token ( andy.id, " repo:admin" ) .await, token ( bob.id, " repo:admin" ) .await, token ( andy.id, " repo:read" ) .await) ;
826
let dir = store.path ( ) .display ( ) .to_string ( ) ;
827
828
let ( st, empty) = api ( & h.app, " GET" , " /user/backups" , & andy_t , None ) .await;
829
assert_eq! ( st, 200 ) ;
830
assert! ( empty[ " setting" ] .is_null ( ) & & empty[ " runs" ] .as_array ( ) .unwrap ( ) .is_empty ( ) , " {empty}" ) ;
831
let ( st, _ ) = api ( & h.app, " POST" , " /user/backups/run" , & andy_t , None ) .await;
832
assert_eq! ( st, 409 , " nothing saved yet" ) ;
833
let ( st, _ ) =
834
api ( & h.app, " PUT" , " /user/backups" , & weak, Some ( serde_json:: json! ( { " every" : " hourly" , " destination" : " dir" , " dir" : dir } ) ) ) .await;
835
assert_eq! ( st, 403 , " repo:admin only" ) ;
836
let ( st, bad) = api (
837
& h.app,
838
" PUT" ,
839
" /user/backups" ,
840
& andy_t ,
841
Some ( serde_json:: json! ( { " every" : " hourly" , " destination" : " gcs" , " bucket" : " andy-backup" , " provider" : " nope" } ) ) ,
842
)
843
.await;
844
assert_eq! ( st, 400 , " checked as the page checks it: {bad}" ) ;
845
let ( st, saved) = api (
846
& h.app,
847
" PUT" ,
848
" /user/backups" ,
849
& andy_t ,
850
Some ( serde_json:: json! ( { " every" : " hourly" , " destination" : " dir" , " dir" : dir, " keep_days" : 30 } ) ) ,
851
)
852
.await;
853
assert_eq! ( st, 200 , " {saved}" ) ;
854
assert_eq! ( ( saved[ " setting" ] [ " every" ] .as_str ( ) , saved[ " place" ] .as_str ( ) ) , ( Some ( " hourly" ) , Some ( format! ( " {dir} /andy" ) .as_str ( ) ) ) ) ;
855
let ( st, probe) = api ( & h.app, " POST" , " /user/backups/test" , & andy_t , None ) .await;
856
assert_eq! ( ( st.as_u16 ( ) , probe[ " ok" ] .as_bool ( ) ) , ( 200 , Some ( true ) ) , " {probe}" ) ;
857
let ( st, started) = api ( & h.app, " POST" , " /user/backups/run" , & andy_t , None ) .await;
858
assert_eq! ( st, 202 , " {started}" ) ;
859
let mut done = serde_json:: Value:: Null;
860
for _ in 0 .. 200 {
861
let ( _ , runs) = api ( & h.app, " GET" , " /user/backups/runs" , & andy_t , None ) .await;
862
if runs[ 0 ] [ " status" ] .as_str ( ) .is_some_and ( | s | s ! = " running" ) {
863
done = runs[ 0 ] .clone ( ) ;
864
break ;
865
}
866
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 50 ) ) .await;
867
}
868
assert_eq! ( ( done[ " status" ] .as_str ( ) , done[ " repos" ] .as_i64 ( ) , done[ " trigger" ] .as_str ( ) ) , ( Some ( " ok" ) , Some ( 1 ) , Some ( " now" ) ) , " {done}" ) ;
bff9106e Andy McCall 22 days ago ↶
869
870
871
sqlx:: query( " INSERT INTO backup_runs (owner, started_ms, finished_ms, status, at, trigger, detail) VALUES ('andy', 1, 2, 'failed', 'x', 'schedule', 'GCS refused')" )
872
.execute ( & h.app.db)
873
.await
874
.unwrap ( ) ;
875
tokio:: time:: sleep( std:: time:: Duration:: from_millis( 1100 ) ) .await;
876
let again = backups:: run( & h.app, " andy" , " now" ) .await.unwrap ( ) ;
877
assert_eq! ( again.status, " ok" ) ;
878
let told = | db : sqlx:: SqlitePool| async move { db:: notify:: inbox( & db, andy.id, 20 ) .await.unwrap ( ) } ;
879
assert! ( told ( h.app.db.clone ( ) ) .await.iter ( ) .any ( | n | n.kind = = " backup.recovered" & & n.title.contains ( " works again" ) ) ) ;
880
assert_eq! ( backups:: daily_summary( & h.app, db:: now_ms( ) , true ) .await.unwrap ( ) , 1 ) ;
881
let summary = told ( h.app.db.clone ( ) ) .await.into_iter ( ) .find ( | n | n.kind = = " backup.summary" ) .expect ( " a summary" ) ;
882
assert! ( summary.title.contains ( " Backups of andy today" ) & & summary.title.contains ( " runs" ) , " {}" , summary.title) ;
2e9188ca Andy McCall 22 days ago ↶
883
884
885
let ( st, _ ) =
886
api ( & h.app, " PUT" , " /orgs/club/backups" , & bob_t , Some ( serde_json:: json! ( { " every" : " hourly" , " destination" : " dir" , " dir" : dir } ) ) )
887
.await;
888
assert_eq! ( st, 404 ) ;
889
let ( st, org) = api (
890
& h.app,
891
" PUT" ,
892
" /orgs/club/backups" ,
893
& andy_t ,
894
Some ( serde_json:: json! ( { " every" : " daily" , " destination" : " dir" , " dir" : dir, " keep_days" : 90 } ) ) ,
895
)
896
.await;
897
assert_eq! ( st, 200 , " {org}" ) ;
898
assert_eq! ( ( org[ " setting" ] [ " kind" ] .as_str ( ) , org[ " setting" ] [ " keep_days" ] .as_i64 ( ) ) , ( Some ( " org" ) , Some ( 90 ) ) ) ;
899
let ( st, _ ) = api ( & h.app, " GET" , " /orgs/club/backups" , & bob_t , None ) .await;
900
assert_eq! ( st, 404 ) ;
901
}