| 1 |
|
| 2 |
|
| 3 |
|
| 4 |
|
| 5 |
|
| 6 |
|
| 7 |
|
| 8 |
|
| 9 |
|
| 10 |
|
| 11 |
|
| 12 |
|
| 13 |
|
| 14 |
use oxy_core::repo::Repo; |
| 15 |
use oxy_core::strata::{conformance, Store}; |
| 16 |
use oxy_strata::CloudStore; |
| 17 |
use std::path::Path; |
| 18 |
|
| 19 |
fn configured(scheme: &str) -> Option<String> { |
| 20 |
let (var, extra, key) = match scheme { |
| 21 |
"s3" => ("OXY_TEST_S3", "OXY_TEST_S3_PROFILE", "profile"), |
| 22 |
"gs" => ("OXY_TEST_GCS", "OXY_TEST_GCS_CREDENTIALS", "credentials"), |
| 23 |
"az" => ("OXY_TEST_AZURE", "OXY_TEST_AZURE_CREDENTIALS", ""), |
| 24 |
_ => unreachable!(), |
| 25 |
}; |
| 26 |
let url = std::env::var(var).ok().filter(|u| !u.is_empty())?; |
| 27 |
let extra_value = std::env::var(extra).ok().filter(|v| !v.is_empty()); |
| 28 |
if scheme == "az" { |
| 29 |
if let Some(file) = &extra_value { |
| 30 |
for line in std::fs::read_to_string(file).unwrap_or_default().lines() { |
| 31 |
if let Some((k, v)) = line.split_once('=') { |
| 32 |
std::env::set_var(k.trim(), v.trim()); |
| 33 |
} |
| 34 |
} |
| 35 |
} |
| 36 |
return Some(url); |
| 37 |
} |
| 38 |
Some(match extra_value { |
| 39 |
Some(v) if !url.contains(&format!("{key}=")) => { |
| 40 |
let sep = if url.contains('?') { '&' } else { '?' }; |
| 41 |
format!("{url}{sep}{key}={v}") |
| 42 |
} |
| 43 |
_ => url, |
| 44 |
}) |
| 45 |
} |
| 46 |
|
| 47 |
fn each_store() -> Vec<(String, CloudStore)> { |
| 48 |
oxy_strata::register(); |
| 49 |
["s3", "gs", "az"] |
| 50 |
.into_iter() |
| 51 |
.filter_map(|s| { |
| 52 |
let Some(url) = configured(s) else { |
| 53 |
eprintln!("{s}: not configured, skipped"); |
| 54 |
return None; |
| 55 |
}; |
| 56 |
let store = CloudStore::open(&url).unwrap_or_else(|e| panic!("{url}: {e}")); |
| 57 |
Some((url, store)) |
| 58 |
}) |
| 59 |
.collect() |
| 60 |
} |
| 61 |
|
| 62 |
fn remove_all(store: &dyn Store, prefix: &str) { |
| 63 |
for e in store.list(prefix).unwrap() { |
| 64 |
store.delete(&e.key).unwrap(); |
| 65 |
} |
| 66 |
} |
| 67 |
|
| 68 |
|
| 69 |
|
| 70 |
fn remove_stale(store: &dyn Store) { |
| 71 |
let mine = format!("-{}", std::process::id()); |
| 72 |
for e in store.list("").unwrap() { |
| 73 |
let first = e.key.split('/').next().unwrap_or(""); |
| 74 |
if (first.starts_with("probe-") || first.starts_with("repo-")) && !first.ends_with(&mine) { |
| 75 |
store.delete(&e.key).unwrap(); |
| 76 |
} |
| 77 |
} |
| 78 |
} |
| 79 |
|
| 80 |
#[test] |
| 81 |
fn every_configured_store_conforms() { |
| 82 |
for (url, store) in each_store() { |
| 83 |
remove_stale(&store); |
| 84 |
let prefix = format!("probe-{}", std::process::id()); |
| 85 |
let passed = conformance(&store, &prefix).unwrap_or_else(|e| { |
| 86 |
remove_all(&store, &prefix); |
| 87 |
panic!("{url}: {e}") |
| 88 |
}); |
| 89 |
eprintln!("{url}: {} checks passed", passed.len()); |
| 90 |
assert_eq!(passed.len(), 8); |
| 91 |
} |
| 92 |
} |
| 93 |
|
| 94 |
|
| 95 |
|
| 96 |
#[test] |
| 97 |
fn a_repository_round_trips_through_every_configured_store() { |
| 98 |
for (url, base_store) in each_store() { |
| 99 |
remove_stale(&base_store); |
| 100 |
let dir = tempfile::tempdir().unwrap(); |
| 101 |
let root = dir.path(); |
| 102 |
std::fs::create_dir_all(root.join("home")).unwrap(); |
| 103 |
std::env::set_var("OXY_HOME", root.join("home")); |
| 104 |
let (base, query) = match url.split_once('?') { |
| 105 |
Some((b, q)) => (b.to_string(), format!("?{q}")), |
| 106 |
None => (url.clone(), String::new()), |
| 107 |
}; |
| 108 |
let remote_url = format!("{base}/repo-{}{query}", std::process::id()); |
| 109 |
let plain = format!("{base}/repo-{}", std::process::id()); |
| 110 |
let store = CloudStore::open(&remote_url).unwrap(); |
| 111 |
|
| 112 |
let mut a = Repo::init(&root.join("a")).unwrap(); |
| 113 |
a.config.set("remote.cloud.url", &remote_url).unwrap(); |
| 114 |
a.save_config().unwrap(); |
| 115 |
write(root, "a/f.txt", b"hello from a\n"); |
| 116 |
let h1 = a.commit("first", &[]).unwrap(); |
| 117 |
let r = a.push("cloud", false).unwrap(); |
| 118 |
assert!(r.iter().all(|x| x.ok), "{plain}: {r:?}"); |
| 119 |
assert!(store.has("oxygen.toml").unwrap()); |
| 120 |
assert_eq!(store.list("bundles/").unwrap().len(), 1, "{plain}"); |
| 121 |
|
| 122 |
let b = Repo::clone(&remote_url, &root.join("b")) |
| 123 |
.unwrap_or_else(|e| panic!("{plain}: clone: {e}")); |
| 124 |
assert_eq!( |
| 125 |
std::fs::read(root.join("b/f.txt")).unwrap(), |
| 126 |
b"hello from a\n" |
| 127 |
); |
| 128 |
assert_eq!(b.bookmark_heads().unwrap(), vec![h1]); |
| 129 |
write(root, "b/f.txt", b"hello from a\nand b\n"); |
| 130 |
b.commit("second", &[]).unwrap(); |
| 131 |
assert!(b.push("origin", false).unwrap()[0].ok); |
| 132 |
let pr = a.pull("cloud").unwrap(); |
| 133 |
assert!(pr.changed, "{plain}"); |
| 134 |
assert_eq!( |
| 135 |
std::fs::read(root.join("a/f.txt")).unwrap(), |
| 136 |
b"hello from a\nand b\n" |
| 137 |
); |
| 138 |
let total: u64 = store.list("").unwrap().iter().map(|e| e.size).sum(); |
| 139 |
eprintln!("{plain}: round trip ok, {total} bytes in the store before cleanup"); |
| 140 |
remove_all(&store, ""); |
| 141 |
assert!(store.list("").unwrap().is_empty(), "{plain}: cleanup"); |
| 142 |
} |
| 143 |
} |
| 144 |
|
| 145 |
|
| 146 |
|
| 147 |
|
| 148 |
#[test] |
| 149 |
fn a_repository_runs_on_every_configured_store() { |
| 150 |
for (url, base_store) in each_store() { |
| 151 |
remove_stale(&base_store); |
| 152 |
let dir = tempfile::tempdir().unwrap(); |
| 153 |
let root = dir.path(); |
| 154 |
std::fs::create_dir_all(root.join("home")).unwrap(); |
| 155 |
std::env::set_var("OXY_HOME", root.join("home")); |
| 156 |
let (base, query) = match url.split_once('?') { |
| 157 |
Some((b, q)) => (b.to_string(), format!("?{q}")), |
| 158 |
None => (url.clone(), String::new()), |
| 159 |
}; |
| 160 |
let store_url = format!("{base}/repo-{}{query}", std::process::id()); |
| 161 |
let plain = format!("{base}/repo-{}", std::process::id()); |
| 162 |
let store = CloudStore::open(&store_url).unwrap(); |
| 163 |
let t = std::time::Instant::now(); |
| 164 |
let repo = Repo::init_new(&root.join("w"), false, Some(&store_url)) |
| 165 |
.unwrap_or_else(|e| panic!("{plain}: init: {e}")); |
| 166 |
let init_ms = t.elapsed().as_millis(); |
| 167 |
let t = std::time::Instant::now(); |
| 168 |
let mut last = None; |
| 169 |
for i in 0..5 { |
| 170 |
write( |
| 171 |
root, |
| 172 |
"w/f.txt", |
| 173 |
format!("line {i}\n").repeat(i + 1).as_bytes(), |
| 174 |
); |
| 175 |
last = Some(repo.commit(&format!("commit {i}"), &[]).unwrap()); |
| 176 |
} |
| 177 |
let commit_ms = t.elapsed().as_millis() / 5; |
| 178 |
drop(repo); |
| 179 |
std::fs::remove_dir_all(root.join("w/.oxy/cache")).unwrap(); |
| 180 |
let t = std::time::Instant::now(); |
| 181 |
let repo = Repo::open(&root.join("w")).unwrap(); |
| 182 |
let heads = repo.bookmark_heads().unwrap(); |
| 183 |
let state = repo.load_state(&heads).unwrap().materialize(); |
| 184 |
let cold_ms = t.elapsed().as_millis(); |
| 185 |
assert_eq!(heads, vec![last.unwrap()], "{plain}"); |
| 186 |
assert_eq!(state.files.len(), 1); |
| 187 |
assert!( |
| 188 |
store.read_ref("ops/head").unwrap().is_some(), |
| 189 |
"{plain}: the head is in the store" |
| 190 |
); |
| 191 |
let keys = store.list("").unwrap(); |
| 192 |
let total: u64 = keys.iter().map(|e| e.size).sum(); |
| 193 |
eprintln!( |
| 194 |
"{plain}: init {init_ms} ms, commit {commit_ms} ms each, cold open and read {cold_ms} ms; {} keys, {total} bytes", |
| 195 |
keys.len() |
| 196 |
); |
| 197 |
remove_all(&store, ""); |
| 198 |
assert!(store.list("").unwrap().is_empty(), "{plain}: cleanup"); |
| 199 |
} |
| 200 |
} |
| 201 |
|
| 202 |
fn write(root: &Path, rel: &str, content: &[u8]) { |
| 203 |
let p = root.join(rel); |
| 204 |
std::fs::create_dir_all(p.parent().unwrap()).unwrap(); |
| 205 |
std::fs::write(p, content).unwrap(); |
| 206 |
} |