Backups: send the length with every upload, an empty file's too 0cd118748131
The first real runs on oxygenhub.io ended partial: every repository was refused with 'GCS write: 411 Length Required'. A store holds empty files, and an empty upload went without a Content-Length header, which Google (and S3) require. Every GCS and S3 upload now says its length. The fake GCS and S3 refuse an upload without one, as the real ones do, and both keyless tests put an empty file in the store; without the fix the GCS test fails with the same 411. closes #180
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-10-14 10:02 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
0cd118748131b9e7e98a2c9b7b50df8ebcbda337097e6b2d61110408cc4d9b33- Depends on
- 35fea4237cc9 7b9f4570daf5 d7d182467382
- Depended on by
- nothing yet
- In bookmarks
- api-unknown-fields cancel-landed-runs diffusion-pin-bookmark draft-ready main oh-comment-body-file proposal-list-fast
- In tags
- v0.5.0 v0.5.1 v0.5.2 v0.6.0 v0.6.1 v0.7.0 v0.8.0 v0.9.0 v0.9.1 v0.9.11 v0.9.15 v0.9.16 v0.9.19 v0.9.2 v0.9.20 v0.9.21 v0.9.22 v0.9.23 v0.9.24 v0.9.3 v0.9.4 v0.9.5 v0.9.6 v0.9.7 v0.9.8 v0.9.9
- Edits
- 9 atoms, 0 tree operations, 2 files
Browse the tree at this change
modified crates/hub/src/backups.rs +18 −2
| … | 609 unchanged lines |
||
| 610 | 610 | q.sort(); |
|
| 611 | 611 | let qs = q.iter().map(|(k, v)| format!("{k}={v}")).collect::<Vec<_>>().join("&"); |
|
| 612 | 612 | let url = format!("{}://{host}{uri}{}{qs}", self.base.scheme(), if qs.is_empty() { "" } else { "?" }); |
|
| 613 | + | let method_has_body = method == reqwest::Method::PUT; |
|
| 613 | 614 | self.rt.block_on(async { |
|
| 614 | 615 | let mut req = self |
|
| 615 | 616 | .client |
|
| … | 3 unchanged lines |
||
| 619 | 620 | .header("x-amz-date", &date) |
|
| 620 | 621 | .header("x-amz-security-token", &c.token); |
|
| 621 | 622 | if !body.is_empty() { |
|
| 622 | - | req = req.header("content-type", "application/octet-stream").body(body); |
|
| 623 | + | req = req.header("content-type", "application/octet-stream"); |
|
| 623 | 624 | } |
|
| 625 | + | // Always the length, an empty body's too: S3 and GCS refuse |
|
| 626 | + | // an upload without one (411 Length Required). |
|
| 627 | + | if method_has_body { |
|
| 628 | + | req = req.header("content-length", body.len().to_string()).body(body); |
|
| 629 | + | } |
|
| 624 | 630 | let res = req.send().await?; |
|
| 625 | 631 | let status = res.status(); |
|
| 626 | 632 | anyhow::Ok((status, res.bytes().await?.to_vec())) |
|
| … | 106 unchanged lines |
||
| 733 | 739 | u.query_pairs_mut().append_pair("uploadType", "media").append_pair("name", &self.name(key)); |
|
| 734 | 740 | let body = bytes.to_vec(); |
|
| 735 | 741 | self.rt.block_on(async { |
|
| 736 | - | let res = self.client.post(u).bearer_auth(&token).header("content-type", "application/octet-stream").body(body).send().await?; |
|
| 742 | + | // The length always, an empty object's too: Google refuses an |
|
| 743 | + | // upload without one (411 Length Required). |
|
| 744 | + | let res = self |
|
| 745 | + | .client |
|
| 746 | + | .post(u) |
|
| 747 | + | .bearer_auth(&token) |
|
| 748 | + | .header("content-type", "application/octet-stream") |
|
| 749 | + | .header("content-length", body.len().to_string()) |
|
| 750 | + | .body(body) |
|
| 751 | + | .send() |
|
| 752 | + | .await?; |
|
| 737 | 753 | let status = res.status(); |
|
| 738 | 754 | if !status.is_success() { |
|
| 739 | 755 | return Err(self.refused("write", status, &res.text().await.unwrap_or_default())); |
|
| … | 1013 unchanged lines |
modified crates/hub/tests/backups.rs +11 −0
| … | 245 unchanged lines |
||
| 246 | 246 | } |
|
| 247 | 247 | let mut g = g.lock().unwrap(); |
|
| 248 | 248 | if upload { |
|
| 249 | + | // As Google does: an upload must say its length, an empty one too. |
|
| 250 | + | if !parts.headers.contains_key("content-length") { |
|
| 251 | + | return (StatusCode::LENGTH_REQUIRED, "Error 411 (Length Required)").into_response(); |
|
| 252 | + | } |
|
| 249 | 253 | let name = query(&parts.uri, "name").unwrap(); |
|
| 250 | 254 | g.objects.insert((bucket, name), bytes.to_vec()); |
|
| 251 | 255 | return axum::Json(serde_json::json!({})).into_response(); |
|
| … | 76 unchanged lines |
||
| 328 | 332 | create(&h.app, "andy", "demo", hub_api::Visibility::Public).await; |
|
| 329 | 333 | let work = tempfile::tempdir().unwrap(); |
|
| 330 | 334 | push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n")]); |
|
| 335 | + | // A store holds empty files too (as oxygenhub.io's did, 2026-10-14): |
|
| 336 | + | // they go up with a length of 0. |
|
| 337 | + | std::fs::write(h.app.repos.dir(&RepoName::parse("andy/demo").unwrap()).join(".oxy/empty-marker"), b"").unwrap(); |
|
| 331 | 338 | ||
| 332 | 339 | backups::save(&h.app.db, &gcs("andy", "andy-backup")).await.unwrap(); |
|
| 333 | 340 | let first = backups::run(&h.app, "andy", "now").await.unwrap(); |
|
| … | 312 unchanged lines |
||
| 646 | 653 | } |
|
| 647 | 654 | let mut g = g.lock().unwrap(); |
|
| 648 | 655 | match (parts.method.as_str(), key.is_empty()) { |
|
| 656 | + | ("PUT", false) if !parts.headers.contains_key("content-length") => { |
|
| 657 | + | (StatusCode::LENGTH_REQUIRED, "<Error><Code>MissingContentLength</Code></Error>").into_response() |
|
| 658 | + | } |
|
| 649 | 659 | ("PUT", false) => { |
|
| 650 | 660 | g.objects.insert((bucket, key), bytes.to_vec()); |
|
| 651 | 661 | StatusCode::OK.into_response() |
|
| … | 80 unchanged lines |
||
| 732 | 742 | create(&h.app, "andy", "demo", hub_api::Visibility::Public).await; |
|
| 733 | 743 | let work = tempfile::tempdir().unwrap(); |
|
| 734 | 744 | push(&format!("{}/andy/demo", h.base), work.path(), &[("README.md", "hello\n"), ("src/a b+c.rs", "fn main() {}\n")]); |
|
| 745 | + | std::fs::write(h.app.repos.dir(&RepoName::parse("andy/demo").unwrap()).join(".oxy/empty-marker"), b"").unwrap(); |
|
| 735 | 746 | ||
| 736 | 747 | backups::save(&h.app.db, &s3("andy", "andy-backup")).await.unwrap(); |
|
| 737 | 748 | let first = backups::run(&h.app, "andy", "now").await.unwrap(); |
|
| … | 29 unchanged lines |