OXYGENHUB

feat(cache): directories keep their times too; FLOW_HEADS only without OXY_BUILD (refs #52) de1d7926185f

A build script's rerun-if-changed on a directory looks at the directory's
time, and a fresh checkout gave every directory a new one, so the hub's
migrations/ reran its build script and recompiled the whole crate on every
cached build. A directory whose listing is unchanged now gets its old time
back; one that gained or lost an entry keeps its new time. flow's build
script watches FLOW_HEADS only when OXY_BUILD does not name the build.
Author
Andy McCall @andymccall <[email protected]>
When
· 2026-10-13 16:07 UTC
Signature
✔ signed by Andy McCall
Hash
de1d7926185f52f196469d6034a567cd6380014b0a1099c8d1e1826226a06a8c
Depends on
2d025bbf9842 82cf6e91d398 a083cc802892 e0918b887850 ea56c56eaf90 fc10d9e80383
Depended on by
nothing yet
In bookmarks
main
In tags
v0.2.0 v0.3.0 v0.3.1 v0.4.0 v0.5.0 v0.5.1 v0.5.2 v0.5.3
Edits
11 atoms, 0 tree operations, 4 files

Browse the tree at this change

modified crates/flow/build.rs +5 −1
…
2 unchanged lines
3 3
//! building, else "local build".
4 4
fn main() {
5 5
    println!("cargo:rerun-if-env-changed=OXY_BUILD");
6 -
    println!("cargo:rerun-if-env-changed=FLOW_HEADS");
6 +
    // The heads name the build only when OXY_BUILD does not: a CI job that
7 +
    // sets it stays fresh from one state to the next (oxygenflow#52).
8 +
    if !std::env::var("OXY_BUILD").is_ok_and(|b| !b.trim().is_empty()) {
9 +
        println!("cargo:rerun-if-env-changed=FLOW_HEADS");
10 +
    }
7 11
    let build = std::env::var("OXY_BUILD")
8 12
        .ok()
9 13
        .map(|b| b.trim().to_string())
…
11 unchanged lines
modified crates/flow/src/runner.rs +7 −1
…
836 unchanged lines
837 837
        } else {
838 838
            let (sources, restored) = workspace::restore_times(&scratch.join("tree"), &files, built_from.as_ref());
839 839
            if built_from.is_some() {
840 -
                note(&mut sink, format!("cache: {restored} of {} files unchanged since the cached build, their times kept", sources.len()));
840 +
                note(
841 +
                    &mut sink,
842 +
                    format!(
843 +
                        "cache: {restored} of {} files and directories unchanged since the cached build, their times kept",
844 +
                        sources.len()
845 +
                    ),
846 +
                );
841 847
            }
842 848
            sources
843 849
        };
…
577 unchanged lines
modified crates/flow/src/workspace.rs +80 −2
…
145 unchanged lines
146 146
        };
147 147
        out.insert(path.clone(), (hex, t));
148 148
    }
149 +
    // Directories too, after their files (#52): a build script's
150 +
    // `rerun-if-changed` on a directory looks at its time as well. A
151 +
    // directory whose listing is unchanged gets its old time back; one that
152 +
    // gained or lost an entry keeps its checkout's, so the change shows.
153 +
    restore_dir_times(dir, Path::new(""), before, &mut out, &mut restored);
149 154
    (out, restored)
150 155
}
151 156
157 +
/// The key a directory's entry has in the sources: its path and a slash.
158 +
fn dir_key(rel: &Path) -> String {
159 +
    format!("{}/", rel.display())
160 +
}
161 +
162 +
fn restore_dir_times(
163 +
    root: &Path,
164 +
    rel: &Path,
165 +
    before: Option<&crate::cache::Sources>,
166 +
    out: &mut crate::cache::Sources,
167 +
    restored: &mut usize,
168 +
) {
169 +
    let here = root.join(rel);
170 +
    let Ok(entries) = std::fs::read_dir(&here) else { return };
171 +
    let mut names: Vec<(String, bool)> = Vec::new();
172 +
    for e in entries.flatten() {
173 +
        let is_dir = e.file_type().is_ok_and(|t| t.is_dir());
174 +
        names.push((e.file_name().to_string_lossy().into_owned(), is_dir));
175 +
    }
176 +
    names.sort();
177 +
    for (name, is_dir) in &names {
178 +
        if *is_dir {
179 +
            restore_dir_times(root, &rel.join(name), before, out, restored);
180 +
        }
181 +
    }
182 +
    let mut h = blake3::Hasher::new();
183 +
    for (name, is_dir) in &names {
184 +
        h.update(name.as_bytes());
185 +
        h.update(if *is_dir { b"/\n".as_slice() } else { b"\n".as_slice() });
186 +
    }
187 +
    let hex = h.finalize().to_hex().to_string();
188 +
    let key = dir_key(rel);
189 +
    let kept = before.and_then(|b| b.get(&key)).filter(|(h, _)| *h == hex).map(|(_, t)| *t);
190 +
    let t = match kept {
191 +
        Some(t) => {
192 +
            let at = std::time::UNIX_EPOCH + std::time::Duration::from_secs(t.max(0) as u64);
193 +
            if std::fs::File::open(&here).and_then(|f| f.set_modified(at)).is_err() {
194 +
                return;
195 +
            }
196 +
            *restored += 1;
197 +
            t
198 +
        }
199 +
        None => std::fs::metadata(&here)
200 +
            .ok()
201 +
            .and_then(|m| m.modified().ok())
202 +
            .and_then(|m| m.duration_since(std::time::UNIX_EPOCH).ok())
203 +
            .map(|d| d.as_secs() as i64)
204 +
            .unwrap_or(0),
205 +
    };
206 +
    out.insert(key, (hex, t));
207 +
}
208 +
152 209
/// An empty ext4 filesystem of `mib` MiB at `out`, owned by this user like
153 210
/// the workspace. mkfs.ext4 creates the file sparse, so only what the job
154 211
/// writes takes space on the runner.
…
73 unchanged lines
228 285
        assert_eq!(sources["changed.rs"].0, hex::encode(h("b2")), "kept with its new content");
229 286
        assert!(sources.contains_key("new.rs") && !sources.contains_key("link.rs"), "files only");
230 287
231 -
        // Without a cached build nothing moves, and the sources are listed.
288 +
        // Without a cached build nothing moves, and the sources are listed:
289 +
        // three files and the directory holding them.
232 290
        let (again, restored) = restore_times(dir.path(), &files, None);
233 -
        assert_eq!((restored, again.len()), (0, 3));
291 +
        assert_eq!((restored, again.len()), (0, 4));
292 +
        assert!(again.contains_key("/"));
293 +
294 +
        // A directory whose listing is unchanged gets its time back; one
295 +
        // that gained an entry keeps its new time, so a build script's
296 +
        // `rerun-if-changed` on it sees the change.
297 +
        std::fs::create_dir(dir.path().join("migrations")).unwrap();
298 +
        std::fs::write(dir.path().join("migrations/0001.sql"), "x").unwrap();
299 +
        let mut files = files;
300 +
        files.push(("migrations/0001.sql".into(), h("x")));
301 +
        let (mut kept, _) = restore_times(dir.path(), &files, None);
302 +
        kept.get_mut("migrations/").unwrap().1 = 2_000_000;
303 +
        kept.get_mut("migrations/0001.sql").unwrap().1 = 2_000_000;
304 +
        let (_, restored) = restore_times(dir.path(), &files, Some(&kept));
305 +
        assert_eq!(secs(&dir.path().join("migrations")), 2_000_000, "unchanged listing: its old time");
306 +
        assert!(restored >= 2);
307 +
        std::fs::write(dir.path().join("migrations/0002.sql"), "y").unwrap();
308 +
        files.push(("migrations/0002.sql".into(), h("y")));
309 +
        let (now_sources, _) = restore_times(dir.path(), &files, Some(&kept));
310 +
        assert!(secs(&dir.path().join("migrations")) > 2_000_000, "a new entry: it looks changed");
311 +
        assert_ne!(now_sources["migrations/"].0, kept["migrations/"].0);
234 312
    }
235 313
}
modified docs/spec.md +8 −1
…
309 unchanged lines
310 310
  the time of its checkout, newer than anything on the disk. A build tool
311 311
  that judges freshness by time, such as cargo, then rebuilds exactly the
312 312
  crates whose sources changed, where a fresh checkout would make every
313 -
  path crate look changed. The log says how many files kept their times.
313 +
  path crate look changed. Directories are kept the same way, by their
314 +
  listing (the names in them): an unchanged one gets its old time back,
315 +
  one that gained or lost an entry keeps its new time, so a build
316 +
  script's `rerun-if-changed` on a directory sees exactly what changed.
317 +
  The log says how many files and directories kept their times. A build
318 +
  script that reads a per-state value (the Oxygen crates' `FLOW_HEADS`)
319 +
  reruns on every state; those crates read it only when `OXY_BUILD` is
320 +
  unset, so a pipeline that sets `OXY_BUILD` keeps them fresh.
314 321
- **Jobs go where their cache is** (oxygenflow#51). Each claim lists the
315 322
  caches the runner holds a disk of (`caches`, each
316 323
  `flow_core::api::cache_slot(repo, job)`), and the hub ranks a runner
…
324 unchanged lines