feat(cache): directories keep their times too; FLOW_HEADS only without OXY_BUILD (refs #52) de1d7926185f
A build script's rerun-if-changed on a directory looks at the directory's time, and a fresh checkout gave every directory a new one, so the hub's migrations/ reran its build script and recompiled the whole crate on every cached build. A directory whose listing is unchanged now gets its old time back; one that gained or lost an entry keeps its new time. flow's build script watches FLOW_HEADS only when OXY_BUILD does not name the build.
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-10-13 16:07 UTC
- Signature
- ✔ signed by Andy McCall
- Hash
de1d7926185f52f196469d6034a567cd6380014b0a1099c8d1e1826226a06a8c- Depends on
- 2d025bbf9842 82cf6e91d398 a083cc802892 e0918b887850 ea56c56eaf90 fc10d9e80383
- Depended on by
- nothing yet
- In bookmarks
- main
- In tags
- v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.5.0 v0.5.1 v0.5.2 v0.5.3
- Edits
- 11 atoms, 0 tree operations, 4 files
Browse the tree at this change
modified crates/flow/build.rs +5 −1
| … | 2 unchanged lines |
||
| 3 | 3 | //! building, else "local build". |
|
| 4 | 4 | fn main() { |
|
| 5 | 5 | println!("cargo:rerun-if-env-changed=OXY_BUILD"); |
|
| 6 | - | println!("cargo:rerun-if-env-changed=FLOW_HEADS"); |
|
| 6 | + | // The heads name the build only when OXY_BUILD does not: a CI job that |
|
| 7 | + | // sets it stays fresh from one state to the next (oxygenflow#52). |
|
| 8 | + | if !std::env::var("OXY_BUILD").is_ok_and(|b| !b.trim().is_empty()) { |
|
| 9 | + | println!("cargo:rerun-if-env-changed=FLOW_HEADS"); |
|
| 10 | + | } |
|
| 7 | 11 | let build = std::env::var("OXY_BUILD") |
|
| 8 | 12 | .ok() |
|
| 9 | 13 | .map(|b| b.trim().to_string()) |
|
| … | 11 unchanged lines |
modified crates/flow/src/runner.rs +7 −1
| … | 836 unchanged lines |
||
| 837 | 837 | } else { |
|
| 838 | 838 | let (sources, restored) = workspace::restore_times(&scratch.join("tree"), &files, built_from.as_ref()); |
|
| 839 | 839 | if built_from.is_some() { |
|
| 840 | - | note(&mut sink, format!("cache: {restored} of {} files unchanged since the cached build, their times kept", sources.len())); |
|
| 840 | + | note( |
|
| 841 | + | &mut sink, |
|
| 842 | + | format!( |
|
| 843 | + | "cache: {restored} of {} files and directories unchanged since the cached build, their times kept", |
|
| 844 | + | sources.len() |
|
| 845 | + | ), |
|
| 846 | + | ); |
|
| 841 | 847 | } |
|
| 842 | 848 | sources |
|
| 843 | 849 | }; |
|
| … | 577 unchanged lines |
modified crates/flow/src/workspace.rs +80 −2
| … | 145 unchanged lines |
||
| 146 | 146 | }; |
|
| 147 | 147 | out.insert(path.clone(), (hex, t)); |
|
| 148 | 148 | } |
|
| 149 | + | // Directories too, after their files (#52): a build script's |
|
| 150 | + | // `rerun-if-changed` on a directory looks at its time as well. A |
|
| 151 | + | // directory whose listing is unchanged gets its old time back; one that |
|
| 152 | + | // gained or lost an entry keeps its checkout's, so the change shows. |
|
| 153 | + | restore_dir_times(dir, Path::new(""), before, &mut out, &mut restored); |
|
| 149 | 154 | (out, restored) |
|
| 150 | 155 | } |
|
| 151 | 156 | ||
| 157 | + | /// The key a directory's entry has in the sources: its path and a slash. |
|
| 158 | + | fn dir_key(rel: &Path) -> String { |
|
| 159 | + | format!("{}/", rel.display()) |
|
| 160 | + | } |
|
| 161 | + | ||
| 162 | + | fn restore_dir_times( |
|
| 163 | + | root: &Path, |
|
| 164 | + | rel: &Path, |
|
| 165 | + | before: Option<&crate::cache::Sources>, |
|
| 166 | + | out: &mut crate::cache::Sources, |
|
| 167 | + | restored: &mut usize, |
|
| 168 | + | ) { |
|
| 169 | + | let here = root.join(rel); |
|
| 170 | + | let Ok(entries) = std::fs::read_dir(&here) else { return }; |
|
| 171 | + | let mut names: Vec<(String, bool)> = Vec::new(); |
|
| 172 | + | for e in entries.flatten() { |
|
| 173 | + | let is_dir = e.file_type().is_ok_and(|t| t.is_dir()); |
|
| 174 | + | names.push((e.file_name().to_string_lossy().into_owned(), is_dir)); |
|
| 175 | + | } |
|
| 176 | + | names.sort(); |
|
| 177 | + | for (name, is_dir) in &names { |
|
| 178 | + | if *is_dir { |
|
| 179 | + | restore_dir_times(root, &rel.join(name), before, out, restored); |
|
| 180 | + | } |
|
| 181 | + | } |
|
| 182 | + | let mut h = blake3::Hasher::new(); |
|
| 183 | + | for (name, is_dir) in &names { |
|
| 184 | + | h.update(name.as_bytes()); |
|
| 185 | + | h.update(if *is_dir { b"/\n".as_slice() } else { b"\n".as_slice() }); |
|
| 186 | + | } |
|
| 187 | + | let hex = h.finalize().to_hex().to_string(); |
|
| 188 | + | let key = dir_key(rel); |
|
| 189 | + | let kept = before.and_then(|b| b.get(&key)).filter(|(h, _)| *h == hex).map(|(_, t)| *t); |
|
| 190 | + | let t = match kept { |
|
| 191 | + | Some(t) => { |
|
| 192 | + | let at = std::time::UNIX_EPOCH + std::time::Duration::from_secs(t.max(0) as u64); |
|
| 193 | + | if std::fs::File::open(&here).and_then(|f| f.set_modified(at)).is_err() { |
|
| 194 | + | return; |
|
| 195 | + | } |
|
| 196 | + | *restored += 1; |
|
| 197 | + | t |
|
| 198 | + | } |
|
| 199 | + | None => std::fs::metadata(&here) |
|
| 200 | + | .ok() |
|
| 201 | + | .and_then(|m| m.modified().ok()) |
|
| 202 | + | .and_then(|m| m.duration_since(std::time::UNIX_EPOCH).ok()) |
|
| 203 | + | .map(|d| d.as_secs() as i64) |
|
| 204 | + | .unwrap_or(0), |
|
| 205 | + | }; |
|
| 206 | + | out.insert(key, (hex, t)); |
|
| 207 | + | } |
|
| 208 | + | ||
| 152 | 209 | /// An empty ext4 filesystem of `mib` MiB at `out`, owned by this user like |
|
| 153 | 210 | /// the workspace. mkfs.ext4 creates the file sparse, so only what the job |
|
| 154 | 211 | /// writes takes space on the runner. |
|
| … | 73 unchanged lines |
||
| 228 | 285 | assert_eq!(sources["changed.rs"].0, hex::encode(h("b2")), "kept with its new content"); |
|
| 229 | 286 | assert!(sources.contains_key("new.rs") && !sources.contains_key("link.rs"), "files only"); |
|
| 230 | 287 | ||
| 231 | - | // Without a cached build nothing moves, and the sources are listed. |
|
| 288 | + | // Without a cached build nothing moves, and the sources are listed: |
|
| 289 | + | // three files and the directory holding them. |
|
| 232 | 290 | let (again, restored) = restore_times(dir.path(), &files, None); |
|
| 233 | - | assert_eq!((restored, again.len()), (0, 3)); |
|
| 291 | + | assert_eq!((restored, again.len()), (0, 4)); |
|
| 292 | + | assert!(again.contains_key("/")); |
|
| 293 | + | ||
| 294 | + | // A directory whose listing is unchanged gets its time back; one |
|
| 295 | + | // that gained an entry keeps its new time, so a build script's |
|
| 296 | + | // `rerun-if-changed` on it sees the change. |
|
| 297 | + | std::fs::create_dir(dir.path().join("migrations")).unwrap(); |
|
| 298 | + | std::fs::write(dir.path().join("migrations/0001.sql"), "x").unwrap(); |
|
| 299 | + | let mut files = files; |
|
| 300 | + | files.push(("migrations/0001.sql".into(), h("x"))); |
|
| 301 | + | let (mut kept, _) = restore_times(dir.path(), &files, None); |
|
| 302 | + | kept.get_mut("migrations/").unwrap().1 = 2_000_000; |
|
| 303 | + | kept.get_mut("migrations/0001.sql").unwrap().1 = 2_000_000; |
|
| 304 | + | let (_, restored) = restore_times(dir.path(), &files, Some(&kept)); |
|
| 305 | + | assert_eq!(secs(&dir.path().join("migrations")), 2_000_000, "unchanged listing: its old time"); |
|
| 306 | + | assert!(restored >= 2); |
|
| 307 | + | std::fs::write(dir.path().join("migrations/0002.sql"), "y").unwrap(); |
|
| 308 | + | files.push(("migrations/0002.sql".into(), h("y"))); |
|
| 309 | + | let (now_sources, _) = restore_times(dir.path(), &files, Some(&kept)); |
|
| 310 | + | assert!(secs(&dir.path().join("migrations")) > 2_000_000, "a new entry: it looks changed"); |
|
| 311 | + | assert_ne!(now_sources["migrations/"].0, kept["migrations/"].0); |
|
| 234 | 312 | } |
|
| 235 | 313 | } |
modified docs/spec.md +8 −1
| … | 309 unchanged lines |
||
| 310 | 310 | the time of its checkout, newer than anything on the disk. A build tool |
|
| 311 | 311 | that judges freshness by time, such as cargo, then rebuilds exactly the |
|
| 312 | 312 | crates whose sources changed, where a fresh checkout would make every |
|
| 313 | - | path crate look changed. The log says how many files kept their times. |
|
| 313 | + | path crate look changed. Directories are kept the same way, by their |
|
| 314 | + | listing (the names in them): an unchanged one gets its old time back, |
|
| 315 | + | one that gained or lost an entry keeps its new time, so a build |
|
| 316 | + | script's `rerun-if-changed` on a directory sees exactly what changed. |
|
| 317 | + | The log says how many files and directories kept their times. A build |
|
| 318 | + | script that reads a per-state value (the Oxygen crates' `FLOW_HEADS`) |
|
| 319 | + | reruns on every state; those crates read it only when `OXY_BUILD` is |
|
| 320 | + | unset, so a pipeline that sets `OXY_BUILD` keeps them fresh. |
|
| 314 | 321 | - **Jobs go where their cache is** (oxygenflow#51). Each claim lists the |
|
| 315 | 322 | caches the runner holds a disk of (`caches`, each |
|
| 316 | 323 | `flow_core::api::cache_slot(repo, job)`), and the hub ranks a runner |
|
| … | 324 unchanged lines |