touched.rs 97.5%

115 of 118 lines run, 34 of 36 functions · run #305 on main · run missed

//! The nodes each change touches, kept beside the store (oxygen#91).
//!
//! Finding the changes that touched a file (`oxy log --path`, the
//! `touches()` revset) otherwise reads and decodes every change of the
//! history, its lines included. A change is immutable, so its entry never
//! goes stale: one missing is read from the store once and added. Like
//! Git's changed-path filters it is derived data, and deleting
//! `.oxy/index/touched` loses nothing but time.

use super::*;
use crate::fasthash::FastMap;
use crate::ids::NodeId;

/// The cache, loaded; [`Touched::save`] writes it back when it grew.
pub(crate) struct Touched {
    nodes: FastMap<ChangeHash, Box<[NodeId]>>,
    /// Working-copy changes, which a snapshot replaces: kept for this
    /// command only, so the cache does not fill with dead entries.
    transient: FastMap<ChangeHash, Box<[NodeId]>>,
    added: usize,
    path: PathBuf,
}

impl Touched {
    /// The nodes `h` touches, from the cache or else from the store; `None`
    /// when the change is not stored (beyond a shallow boundary).
    pub(crate) fn of(&mut self, repo: &Repo, h: &ChangeHash) -> Option<&[NodeId]> {
        if !self.nodes.contains_key(h) && !self.transient.contains_key(h) {
            // The skeleton names every node; no text is read.
            let (c, _) = repo.store.get_change_lazy(h).ok()?;
            self.keep(*h, &c);
        }
        self.nodes
            .get(h)
            .or_else(|| self.transient.get(h))
            .map(|n| &n[..])
    }

    fn keep(&mut self, h: ChangeHash, c: &Change) {
        if c.kind == ChangeKind::WorkingCopy {
            self.transient.insert(h, nodes_of(c));
        } else {
            self.nodes.insert(h, nodes_of(c));
            self.added += 1;
        }
    }

    /// Read every one of `hashes` the cache lacks.
    pub(crate) fn fill(&mut self, repo: &Repo, hashes: &[ChangeHash]) {
        use rayon::prelude::*;
        let missing: Vec<ChangeHash> = hashes
            .iter()
            .filter(|h| !self.nodes.contains_key(*h) && !self.transient.contains_key(*h))
            .copied()
            .collect();
        // Read a batch at a time, then decode skeletons (no text) on every
        // core. A change the store lacks is left out, as `of` leaves it.
        for batch in missing.chunks(8192) {
            let read: Vec<(ChangeHash, Change)> = repo
                .store
                .get_many(batch)
                .into_par_iter()
                .zip(batch.par_iter())
                .filter_map(|(got, h)| match got {
                    Ok((ObjectType::Change, bytes)) => {
                        Some((*h, Change::decode_skeleton(&bytes, *h).ok()?.0))
                    }
                    _ => None,
                })
                .collect();
            for (h, c) in &read {
                self.keep(*h, c);
            }
        }
    }

    /// Whether `h` touches any of `nodes` (false when it is not stored).
    pub(crate) fn touches(&mut self, repo: &Repo, h: &ChangeHash, nodes: &HashSet<NodeId>) -> bool {
        self.of(repo, h)
            .is_some_and(|n| n.iter().any(|n| nodes.contains(n)))
    }

    /// Write the cache back if it grew; quietly nothing when it cannot be.
    pub(crate) fn save(self) {
        if self.added == 0 {
            return;
        }
        let _t = crate::timing::phase("write: touched nodes");
        let mut w = crate::flat::Writer::default();
        w.count(self.nodes.len());
        for (h, nodes) in &self.nodes {
            w.object(h);
            w.count(nodes.len());
            for n in nodes.iter() {
                w.fixed(&n.0);
            }
        }
        // Checked whole on reading: a damaged entry would hide a change.
        let sum = *blake3::hash(&w.0).as_bytes();
        w.fixed(&sum);
        let _ = fs::create_dir_all(self.path.parent().unwrap_or(&self.path));
        let _ = write_cache(&self.path, &w.0);
    }
}

/// The nodes a change's tree operations and edits name, each once.
fn nodes_of(c: &Change) -> Box<[NodeId]> {
    let mut nodes: Vec<NodeId> = c
        .tree_ops
        .iter()
        .map(|o| o.node())
        .chain(c.edits.iter().map(|e| e.node()))
        .collect();
    nodes.sort();
    nodes.dedup();
    nodes.into_boxed_slice()
}

impl Repo {
    /// The touched-nodes cache: empty when there is none yet or it does
    /// not read back whole.
    pub(crate) fn touched(&self) -> Touched {
        let _t = crate::timing::phase("load: touched nodes");
        let path = self.oxy.join("index").join("touched");
        let nodes = fs::read(&path)
            .ok()
            .and_then(|b| read_touched(&b))
            .unwrap_or_default();
        Touched {
            nodes,
            transient: FastMap::default(),
            added: 0,
            path,
        }
    }
}

fn read_touched(bytes: &[u8]) -> Option<FastMap<ChangeHash, Box<[NodeId]>>> {
    let (body, sum) = bytes.split_at_checked(bytes.len().checked_sub(32)?)?;
    if blake3::hash(body).as_bytes() != sum {
        return None;
    }
    let mut r = crate::flat::Reader::new(body);
    let n = r.count().ok()?;
    let mut out = FastMap::default();
    out.reserve(n);
    for _ in 0..n {
        let h = r.object().ok()?;
        let k = r.count().ok()?;
        let mut nodes = Vec::with_capacity(k);
        for _ in 0..k {
            nodes.push(NodeId(r.fixed::<16>().ok()?));
        }
        out.insert(h, nodes.into_boxed_slice());
    }
    r.at_end().then_some(out)
}

/// The changes a cache file holds, for tests.
#[cfg(test)]
pub(crate) fn read_touched_for_test(bytes: &[u8]) -> Option<HashSet<ChangeHash>> {
    read_touched(bytes).map(|m| m.into_keys().collect())
}

1		//! The nodes each change touches, kept beside the store (oxygen#91).
2		//!
3		//! Finding the changes that touched a file (`oxy log --path`, the
4		//! `touches()` revset) otherwise reads and decodes every change of the
5		//! history, its lines included. A change is immutable, so its entry never
6		//! goes stale: one missing is read from the store once and added. Like
7		//! Git's changed-path filters it is derived data, and deleting
8		//! `.oxy/index/touched` loses nothing but time.
9
10		use super::*;
11		use crate::fasthash::FastMap;
12		use crate::ids::NodeId;
13
14		/// The cache, loaded; [`Touched::save`] writes it back when it grew.
15		pub(crate) struct Touched {
16		nodes: FastMap<ChangeHash, Box<[NodeId]>>,
17		/// Working-copy changes, which a snapshot replaces: kept for this
18		/// command only, so the cache does not fill with dead entries.
19		transient: FastMap<ChangeHash, Box<[NodeId]>>,
20		added: usize,
21		path: PathBuf,
22		}
23
24		impl Touched {
25		/// The nodes `h` touches, from the cache or else from the store; `None`
26		/// when the change is not stored (beyond a shallow boundary).
27	45×	pub(crate) fn of(&mut self, repo: &Repo, h: &ChangeHash) -> Option<&[NodeId]> {
28	45×	if !self.nodes.contains_key(h) && !self.transient.contains_key(h) {
29		// The skeleton names every node; no text is read.
30	0×	let (c, _) = repo.store.get_change_lazy(h).ok()?;
31	0×	self.keep(*h, &c);
32	45×	}
33	45×	self.nodes
34	45×	.get(h)
35	45×	.or_else(\|\| self.transient.get(h))
36	45×	.map(\|n\| &n[..])
37	45×	}
38
39	25×	fn keep(&mut self, h: ChangeHash, c: &Change) {
40	25×	if c.kind == ChangeKind::WorkingCopy {
41	7×	self.transient.insert(h, nodes_of(c));
42	18×	} else {
43	18×	self.nodes.insert(h, nodes_of(c));
44	18×	self.added += 1;
45	18×	}
46	25×	}
47
48		/// Read every one of `hashes` the cache lacks.
49	11×	pub(crate) fn fill(&mut self, repo: &Repo, hashes: &[ChangeHash]) {
50		use rayon::prelude::*;
51	11×	let missing: Vec<ChangeHash> = hashes
52	11×	.iter()
53	43×	.filter(\|h\| !self.nodes.contains_key(h) && !self.transient.contains_key(h))
54	11×	.copied()
55	11×	.collect();
56		// Read a batch at a time, then decode skeletons (no text) on every
57		// core. A change the store lacks is left out, as `of` leaves it.
58	11×	for batch in missing.chunks(8192) {
59	9×	let read: Vec<(ChangeHash, Change)> = repo
60	9×	.store
61	9×	.get_many(batch)
62	9×	.into_par_iter()
63	9×	.zip(batch.par_iter())
64	25×	.filter_map(\|(got, h)\| match got {
65	25×	Ok((ObjectType::Change, bytes)) => {
66	25×	Some((h, Change::decode_skeleton(&bytes, h).ok()?.0))
67		}
68	0×	_ => None,
69	25×	})
70	9×	.collect();
71	25×	for (h, c) in &read {
72	25×	self.keep(*h, c);
73	25×	}
74		}
75	11×	}
76
77		/// Whether `h` touches any of `nodes` (false when it is not stored).
78	45×	pub(crate) fn touches(&mut self, repo: &Repo, h: &ChangeHash, nodes: &HashSet<NodeId>) -> bool {
79	45×	self.of(repo, h)
80	72×	.is_some_and(\|n\| n.iter().any(\|n\| nodes.contains(n)))
81	45×	}
82
83		/// Write the cache back if it grew; quietly nothing when it cannot be.
84	12×	pub(crate) fn save(self) {
85	12×	if self.added == 0 {
86	6×	return;
87	6×	}
88	6×	let _t = crate::timing::phase("write: touched nodes");
89	6×	let mut w = crate::flat::Writer::default();
90	6×	w.count(self.nodes.len());
91	18×	for (h, nodes) in &self.nodes {
92	18×	w.object(h);
93	18×	w.count(nodes.len());
94	31×	for n in nodes.iter() {
95	31×	w.fixed(&n.0);
96	31×	}
97		}
98		// Checked whole on reading: a damaged entry would hide a change.
99	6×	let sum = *blake3::hash(&w.0).as_bytes();
100	6×	w.fixed(&sum);
101	6×	let _ = fs::create_dir_all(self.path.parent().unwrap_or(&self.path));
102	6×	let _ = write_cache(&self.path, &w.0);
103	12×	}
104		}
105
106		/// The nodes a change's tree operations and edits name, each once.
107	25×	fn nodes_of(c: &Change) -> Box<[NodeId]> {
108	25×	let mut nodes: Vec<NodeId> = c
109	25×	.tree_ops
110	25×	.iter()
111	25×	.map(\|o\| o.node())
112	33×	.chain(c.edits.iter().map(\|e\| e.node()))
113	25×	.collect();
114	25×	nodes.sort();
115	25×	nodes.dedup();
116	25×	nodes.into_boxed_slice()
117	25×	}
118
119		impl Repo {
120		/// The touched-nodes cache: empty when there is none yet or it does
121		/// not read back whole.
122	12×	pub(crate) fn touched(&self) -> Touched {
123	12×	let _t = crate::timing::phase("load: touched nodes");
124	12×	let path = self.oxy.join("index").join("touched");
125	12×	let nodes = fs::read(&path)
126	12×	.ok()
127	12×	.and_then(\|b\| read_touched(&b))
128	12×	.unwrap_or_default();
129	12×	Touched {
130	12×	nodes,
131	12×	transient: FastMap::default(),
132	12×	added: 0,
133	12×	path,
134	12×	}
135	12×	}
136		}
137
138	8×	fn read_touched(bytes: &[u8]) -> Option<FastMap<ChangeHash, Box<[NodeId]>>> {
139	8×	let (body, sum) = bytes.split_at_checked(bytes.len().checked_sub(32)?)?;
140	8×	if blake3::hash(body).as_bytes() != sum {
141	1×	return None;
142	7×	}
143	7×	let mut r = crate::flat::Reader::new(body);
144	7×	let n = r.count().ok()?;
145	7×	let mut out = FastMap::default();
146	7×	out.reserve(n);
147	7×	for _ in 0..n {
148	25×	let h = r.object().ok()?;
149	25×	let k = r.count().ok()?;
150	25×	let mut nodes = Vec::with_capacity(k);
151	25×	for _ in 0..k {
152	46×	nodes.push(NodeId(r.fixed::<16>().ok()?));
153		}
154	25×	out.insert(h, nodes.into_boxed_slice());
155		}
156	7×	r.at_end().then_some(out)
157	8×	}
158
159		/// The changes a cache file holds, for tests.
160		#[cfg(test)]
161	1×	pub(crate) fn read_touched_for_test(bytes: &[u8]) -> Option<HashSet<ChangeHash>> {
162	1×	read_touched(bytes).map(\|m\| m.into_keys().collect())
163	1×	}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

Coverage / crates/oxy-core/src/repo/touched.rs 97.5%

Keyboard