Cloud stores remember a missing key for 30 s, until this process writes it 8b53b935dd03

After a push to metalgear2 at 07:23, its client's negotiate asked the hub
about ~190 objects the hub lacked, every 8 s; each was a bucket request
that found nothing (45 ms), never cached, so negotiate took 9 s again
(4 270 misses in three minutes). A shared cloud client now takes a key
it found missing in the last 30 s as missing without asking, and forgets
it the moment it writes the key (put_new, write_ref), so its own writes
are never hidden; another writer is seen within 30 s. At most 100 000
kept. Test: remembered, forgotten on write, expired.

Refs andymccall/oxygenhub#228

Author: Andy McCall @andymccall <[email protected]>
via: agent Claude Code <[email protected]> · delegate for changes, until 2026-11-13 19:00Z · key ea5f78160088
When: 19 days ago · 2026-10-17 06:48 UTC
Signature: ✔ signed by Claude Code
Hash: 8b53b935dd0366dba212be45bf9e10486f84aaf660f39c6b88c47cca89e6a8b2
Depends on: 1e0fbb47ab90 8d80356371c5 c9369ad8b1b1
Depended on by: 767ac6cd0545 87c948720343
In bookmarks: carried-not-removed doctor-trust identical-sides keyless-tags main merge-remote-bookmark paths-for-resolutions-and-amend push-notes show-stat subrepo-pin-state
In tags: v0.10.0 v0.11.0 v0.4.4 v0.4.5 v0.4.6 v0.5.0 v0.5.1 v0.6.0 v0.7.0 v0.8.0 v0.9.0
Edits: 8 atoms, 0 tree operations, 1 file

Browse the tree at this change

show whitespace changes

modified crates/oxy-strata/src/lib.rs +62 −0

		…	396 unchanged lines
397	397		/// Always there until the store is dropped: two workers, so the
398	398		/// requests of a shared store run side by side.
399	399		runtime: Option<tokio::runtime::Runtime>,
	400	+	/// Keys `has` found missing lately (see [`Misses`]).
	401	+	misses: Misses,
400	402		}
401	403
	404	+	/// How long a key found missing is taken as missing without asking again.
	405	+	const MISS_FOR: std::time::Duration = std::time::Duration::from_secs(30);
	406	+
	407	+	/// Keys found missing in the last [`MISS_FOR`], so a server answering the
	408	+	/// same question again and again does not send a request each time
	409	+	/// (oxygenhub#228: a client's negotiate asked about ~190 objects the hub
	410	+	/// lacked every 8 s, 45 ms a request). This process forgets a key the
	411	+	/// moment it writes it, so its own writes are never hidden; another writer
	412	+	/// to the same keys is seen within `MISS_FOR`.
	413	+	#[derive(Default)]
	414	+	struct Misses(Mutex<std::collections::HashMap<String, std::time::Instant>>);
	415	+
	416	+	impl Misses {
	417	+	/// At most this many kept; past it the oldest half is let go.
	418	+	const MAX: usize = 100_000;
	419	+
	420	+	fn known(&self, key: &str) -> bool {
	421	+	let map = self.0.lock().unwrap_or_else(\|p\| p.into_inner());
	422	+	map.get(key).is_some_and(\|t\| t.elapsed() < MISS_FOR)
	423	+	}
	424	+
	425	+	fn missed(&self, key: &str) {
	426	+	let mut map = self.0.lock().unwrap_or_else(\|p\| p.into_inner());
	427	+	if map.len() >= Self::MAX {
	428	+	map.retain(\|_, t\| t.elapsed() < MISS_FOR / 2);
	429	+	if map.len() >= Self::MAX {
	430	+	map.clear();
	431	+	}
	432	+	}
	433	+	map.insert(key.to_string(), std::time::Instant::now());
	434	+	}
	435	+
	436	+	fn written(&self, key: &str) {
	437	+	self.0.lock().unwrap_or_else(\|p\| p.into_inner()).remove(key);
	438	+	}
	439	+	}
	440	+
402	441		impl Drop for CloudStore {
403	442		/// Without waiting, so a store can be dropped inside a runtime too.
404	443		fn drop(&mut self) {
		…	86 unchanged lines
491	530		name: parsed.plain(),
492	531		hint,
493	532		runtime: Some(runtime),
	533	+	misses: Misses::default(),
494	534		})
495	535		}
496	536
		…	104 unchanged lines
601	641
602	642		fn has(&self, key: &str) -> Result<bool> {
603	643		let path = self.path(key)?;
	644	+	if self.misses.known(key) {
	645	+	return Ok(false);
	646	+	}
604	647		match self.block(self.inner.head(&path)) {
605	648		Ok(_) => {
606	649		counted(Op::Has);
		…	1 unchanged lines
608	651		}
609	652		Err(object_store::Error::NotFound { .. }) => {
610	653		counted(Op::HasMissing);
	654	+	self.misses.missed(key);
611	655		Ok(false)
612	656		}
613	657		Err(e) => Err(self.map(e, key)),
		…	40 unchanged lines
654	698
655	699		fn put_new(&self, key: &str, bytes: &[u8]) -> Result<()> {
656	700		let path = self.path(key)?;
	701	+	self.misses.written(key);
657	702		let opts = PutOptions {
658	703		mode: PutMode::Create,
659	704		..Default::default()
		…	75 unchanged lines
735	780
736	781		fn write_ref(&self, key: &str, bytes: &[u8], expected: Option<&Version>) -> Result<Version> {
737	782		let path = self.path(key)?;
	783	+	self.misses.written(key);
738	784		let mode = match expected {
739	785		None => PutMode::Create,
740	786		Some(v) => PutMode::Update(Self::update_version(v)),
		…	20 unchanged lines
761	807
762	808		#[cfg(test)]
763	809		mod tests {
	810	+
	811	+	/// A miss is remembered until the key is written, or for MISS_FOR.
	812	+	#[test]
	813	+	fn a_miss_is_remembered_until_written() {
	814	+	let m = super::Misses::default();
	815	+	assert!(!m.known("loose/ab/cd"));
	816	+	m.missed("loose/ab/cd");
	817	+	assert!(m.known("loose/ab/cd") && !m.known("loose/ab/ce"));
	818	+	m.written("loose/ab/cd");
	819	+	assert!(!m.known("loose/ab/cd"), "a write forgets the miss");
	820	+	m.missed("old");
	821	+	m.0.lock()
	822	+	.unwrap()
	823	+	.insert("old".into(), std::time::Instant::now() - super::MISS_FOR);
	824	+	assert!(!m.known("old"), "a miss expires");
	825	+	}
764	826
765	827		/// Every kind of request is reported, once and by its own name.
766	828		#[test]
		…	121 unchanged lines

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

Cloud stores remember a missing key for 30 s, until this process writes it 8b53b935dd03

Keyboard