1 #[cfg(feature = "stack-cache")]
4 use rustc_data_structures::fx::FxHashSet;
6 use crate::stacked_borrows::{AccessKind, Item, Permission, SbTag};
7 use crate::ProvenanceExtra;
9 /// Exactly what cache size we should use is a difficult tradeoff. There will always be some
10 /// workload which has a `SbTag` working set which exceeds the size of the cache, and ends up
11 /// falling back to linear searches of the borrow stack very often.
12 /// The cost of making this value too large is that the loop in `Stack::insert` which ensures the
13 /// entries in the cache stay correct after an insert becomes expensive.
14 #[cfg(feature = "stack-cache")]
15 const CACHE_LEN: usize = 32;
17 /// Extra per-location state.
18 #[derive(Clone, Debug)]
20 /// Used *mostly* as a stack; never empty.
22 /// * Above a `SharedReadOnly` there can only be more `SharedReadOnly`.
23 /// * Except for `Untagged`, no tag occurs in the stack more than once.
25 /// If this is `Some(id)`, then the actual current stack is unknown. This can happen when
26 /// wildcard pointers are used to access this location. What we do know is that `borrows` are at
27 /// the top of the stack, and below it are arbitrarily many items whose `tag` is strictly less
29 /// When the bottom is unknown, `borrows` always has a `SharedReadOnly` or `Unique` at the bottom;
30 /// we never have the unknown-to-known boundary in an SRW group.
31 unknown_bottom: Option<SbTag>,
33 /// A small LRU cache of searches of the borrow stack.
34 #[cfg(feature = "stack-cache")]
36 /// On a read, we need to disable all `Unique` above the granting item. We can avoid most of
37 /// this scan by keeping track of the region of the borrow stack that may contain `Unique`s.
38 #[cfg(feature = "stack-cache")]
39 unique_range: Range<usize>,
43 pub fn retain(&mut self, tags: &FxHashSet<SbTag>) {
44 let mut first_removed = None;
46 // We never consider removing the bottom-most tag. For stacks without an unknown
47 // bottom this preserves the base tag.
48 // Note that the algorithm below is based on considering the tag at read_idx - 1,
49 // so precisely considering the tag at index 0 for removal when we have an unknown
50 // bottom would complicate the implementation. The simplification of not considering
51 // it does not have a significant impact on the degree to which the GC mititages
54 let mut write_idx = read_idx;
55 while read_idx < self.borrows.len() {
56 let left = self.borrows[read_idx - 1];
57 let this = self.borrows[read_idx];
58 let should_keep = match this.perm() {
59 // SharedReadWrite is the simplest case, if it's unreachable we can just remove it.
60 Permission::SharedReadWrite => tags.contains(&this.tag()),
61 // Only retain a Disabled tag if it is terminating a SharedReadWrite block.
62 Permission::Disabled => left.perm() == Permission::SharedReadWrite,
63 // Unique and SharedReadOnly can terminate a SharedReadWrite block, so only remove
64 // them if they are both unreachable and not directly after a SharedReadWrite.
65 Permission::Unique | Permission::SharedReadOnly =>
66 left.perm() == Permission::SharedReadWrite || tags.contains(&this.tag()),
70 if read_idx != write_idx {
71 self.borrows[write_idx] = self.borrows[read_idx];
74 } else if first_removed.is_none() {
75 first_removed = Some(read_idx);
80 self.borrows.truncate(write_idx);
82 #[cfg(not(feature = "stack-cache"))]
83 drop(first_removed); // This is only needed for the stack-cache
85 #[cfg(feature = "stack-cache")]
86 if let Some(first_removed) = first_removed {
87 // Either end of unique_range may have shifted, all we really know is that we can't
88 // have introduced a new Unique.
89 if !self.unique_range.is_empty() {
90 self.unique_range = 0..self.len();
93 // Replace any Items which have been collected with the base item, a known-good value.
94 for i in 0..CACHE_LEN {
95 if self.cache.idx[i] >= first_removed {
96 self.cache.items[i] = self.borrows[0];
97 self.cache.idx[i] = 0;
104 /// A very small cache of searches of a borrow stack, mapping `Item`s to their position in said stack.
106 /// It may seem like maintaining this cache is a waste for small stacks, but
107 /// (a) iterating over small fixed-size arrays is super fast, and (b) empirically this helps *a lot*,
108 /// probably because runtime is dominated by large stacks.
109 #[cfg(feature = "stack-cache")]
110 #[derive(Clone, Debug)]
112 items: [Item; CACHE_LEN], // Hot in find_granting
113 idx: [usize; CACHE_LEN], // Hot in grant
116 #[cfg(feature = "stack-cache")]
118 /// When a tag is used, we call this function to add or refresh it in the cache.
120 /// We use the position in the cache to represent how recently a tag was used; the first position
121 /// is the most recently used tag. So an add shifts every element towards the end, and inserts
122 /// the new element at the start. We lose the last element.
123 /// This strategy is effective at keeping the most-accessed items in the cache, but it costs a
124 /// linear shift across the entire cache when we add a new tag.
125 fn add(&mut self, idx: usize, item: Item) {
126 self.items.copy_within(0..CACHE_LEN - 1, 1);
127 self.items[0] = item;
128 self.idx.copy_within(0..CACHE_LEN - 1, 1);
133 impl PartialEq for Stack {
134 fn eq(&self, other: &Self) -> bool {
135 // All the semantics of Stack are in self.borrows, everything else is caching
136 self.borrows == other.borrows
143 /// Panics if any of the caching mechanisms have broken,
144 /// - The StackCache indices don't refer to the parallel items,
145 /// - There are no Unique items outside of first_unique..last_unique
146 #[cfg(all(feature = "stack-cache", debug_assertions))]
147 fn verify_cache_consistency(&self) {
148 // Only a full cache needs to be valid. Also see the comments in find_granting_cache
149 // and set_unknown_bottom.
150 if self.borrows.len() >= CACHE_LEN {
151 for (tag, stack_idx) in self.cache.items.iter().zip(self.cache.idx.iter()) {
152 assert_eq!(self.borrows[*stack_idx], *tag);
156 // Check that all Unique items fall within unique_range.
157 for (idx, item) in self.borrows.iter().enumerate() {
158 if item.perm() == Permission::Unique {
160 self.unique_range.contains(&idx),
168 // Check that the unique_range is a valid index into the borrow stack.
169 // This asserts that the unique_range's start <= end.
170 let _uniques = &self.borrows[self.unique_range.clone()];
172 // We cannot assert that the unique range is precise.
173 // Both ends may shift around when `Stack::retain` is called. Additionally,
174 // when we pop items within the unique range, setting the end of the range precisely
175 // requires doing a linear search of the borrow stack, which is exactly the kind of
176 // operation that all this caching exists to avoid.
179 /// Find the item granting the given kind of access to the given tag, and return where
180 /// it is on the stack. For wildcard tags, the given index is approximate, but if *no*
181 /// index is given it means the match was *not* in the known part of the stack.
182 /// `Ok(None)` indicates it matched the "unknown" part of the stack.
183 /// `Err` indicates it was not found.
184 pub(super) fn find_granting(
187 tag: ProvenanceExtra,
188 exposed_tags: &FxHashSet<SbTag>,
189 ) -> Result<Option<usize>, ()> {
190 #[cfg(all(feature = "stack-cache", debug_assertions))]
191 self.verify_cache_consistency();
193 let ProvenanceExtra::Concrete(tag) = tag else {
194 // Handle the wildcard case.
195 // Go search the stack for an exposed tag.
199 .enumerate() // we also need to know *where* in the stack
200 .rev() // search top-to-bottom
201 .find_map(|(idx, item)| {
202 // If the item fits and *might* be this wildcard, use it.
203 if item.perm().grants(access) && exposed_tags.contains(&item.tag()) {
210 return Ok(Some(idx));
212 // If we couldn't find it in the stack, check the unknown bottom.
213 return if self.unknown_bottom.is_some() { Ok(None) } else { Err(()) };
216 if let Some(idx) = self.find_granting_tagged(access, tag) {
217 return Ok(Some(idx));
220 // Couldn't find it in the stack; but if there is an unknown bottom it might be there.
221 let found = self.unknown_bottom.is_some_and(|unknown_limit| {
222 tag.0 < unknown_limit.0 // unknown_limit is an upper bound for what can be in the unknown bottom.
224 if found { Ok(None) } else { Err(()) }
227 fn find_granting_tagged(&mut self, access: AccessKind, tag: SbTag) -> Option<usize> {
228 #[cfg(feature = "stack-cache")]
229 if let Some(idx) = self.find_granting_cache(access, tag) {
233 // If we didn't find the tag in the cache, fall back to a linear search of the
234 // whole stack, and add the tag to the cache.
235 for (stack_idx, item) in self.borrows.iter().enumerate().rev() {
236 if tag == item.tag() && item.perm().grants(access) {
237 #[cfg(feature = "stack-cache")]
238 self.cache.add(stack_idx, *item);
239 return Some(stack_idx);
245 #[cfg(feature = "stack-cache")]
246 fn find_granting_cache(&mut self, access: AccessKind, tag: SbTag) -> Option<usize> {
247 // This looks like a common-sense optimization; we're going to do a linear search of the
248 // cache or the borrow stack to scan the shorter of the two. This optimization is miniscule
249 // and this check actually ensures we do not access an invalid cache.
250 // When a stack is created and when items are removed from the top of the borrow stack, we
251 // need some valid value to populate the cache. In both cases, we try to use the bottom
252 // item. But when the stack is cleared in `set_unknown_bottom` there is nothing we could
253 // place in the cache that is correct. But due to the way we populate the cache in
254 // `StackCache::add`, we know that when the borrow stack has grown larger than the cache,
255 // every slot in the cache is valid.
256 if self.borrows.len() <= CACHE_LEN {
259 // Search the cache for the tag we're looking up
260 let cache_idx = self.cache.items.iter().position(|t| t.tag() == tag)?;
261 let stack_idx = self.cache.idx[cache_idx];
262 // If we found the tag, look up its position in the stack to see if it grants
263 // the required permission
264 if self.cache.items[cache_idx].perm().grants(access) {
265 // If it does, and it's not already in the most-recently-used position, re-insert it at
266 // the most-recently-used position. This technically reduces the efficiency of the
267 // cache by duplicating elements, but current benchmarks do not seem to benefit from
268 // avoiding this duplication.
269 // But if the tag is in position 1, avoiding the duplicating add is trivial.
270 // If it does, and it's not already in the most-recently-used position, move it there.
271 // Except if the tag is in position 1, this is equivalent to just a swap, so do that.
273 self.cache.items.swap(0, 1);
274 self.cache.idx.swap(0, 1);
275 } else if cache_idx > 1 {
276 self.cache.add(stack_idx, self.cache.items[cache_idx]);
280 // Tag is in the cache, but it doesn't grant the required permission
285 pub fn insert(&mut self, new_idx: usize, new: Item) {
286 self.borrows.insert(new_idx, new);
288 #[cfg(feature = "stack-cache")]
289 self.insert_cache(new_idx, new);
292 #[cfg(feature = "stack-cache")]
293 fn insert_cache(&mut self, new_idx: usize, new: Item) {
294 // Adjust the possibly-unique range if an insert occurs before or within it
295 if self.unique_range.start >= new_idx {
296 self.unique_range.start += 1;
298 if self.unique_range.end >= new_idx {
299 self.unique_range.end += 1;
301 if new.perm() == Permission::Unique {
302 // If this is the only Unique, set the range to contain just the new item.
303 if self.unique_range.is_empty() {
304 self.unique_range = new_idx..new_idx + 1;
306 // We already have other Unique items, expand the range to include the new item
307 self.unique_range.start = self.unique_range.start.min(new_idx);
308 self.unique_range.end = self.unique_range.end.max(new_idx + 1);
312 // The above insert changes the meaning of every index in the cache >= new_idx, so now
313 // we need to find every one of those indexes and increment it.
314 // But if the insert is at the end (equivalent to a push), we can skip this step because
315 // it didn't change the position of any other items.
316 if new_idx != self.borrows.len() - 1 {
317 for idx in &mut self.cache.idx {
324 // This primes the cache for the next access, which is almost always the just-added tag.
325 self.cache.add(new_idx, new);
327 #[cfg(debug_assertions)]
328 self.verify_cache_consistency();
331 /// Construct a new `Stack` using the passed `Item` as the base tag.
332 pub fn new(item: Item) -> Self {
335 unknown_bottom: None,
336 #[cfg(feature = "stack-cache")]
337 cache: StackCache { idx: [0; CACHE_LEN], items: [item; CACHE_LEN] },
338 #[cfg(feature = "stack-cache")]
339 unique_range: if item.perm() == Permission::Unique { 0..1 } else { 0..0 },
343 pub fn get(&self, idx: usize) -> Option<Item> {
344 self.borrows.get(idx).cloned()
347 #[allow(clippy::len_without_is_empty)] // Stacks are never empty
348 pub fn len(&self) -> usize {
352 pub fn unknown_bottom(&self) -> Option<SbTag> {
356 pub fn set_unknown_bottom(&mut self, tag: SbTag) {
357 // We clear the borrow stack but the lookup cache doesn't support clearing per se. Instead,
358 // there is a check explained in `find_granting_cache` which protects against accessing the
359 // cache when it has been cleared and not yet refilled.
360 self.borrows.clear();
361 self.unknown_bottom = Some(tag);
362 #[cfg(feature = "stack-cache")]
364 self.unique_range = 0..0;
368 /// Find all `Unique` elements in this borrow stack above `granting_idx`, pass a copy of them
369 /// to the `visitor`, then set their `Permission` to `Disabled`.
370 pub fn disable_uniques_starting_at<V: FnMut(Item) -> crate::InterpResult<'tcx>>(
372 disable_start: usize,
374 ) -> crate::InterpResult<'tcx> {
375 #[cfg(feature = "stack-cache")]
376 let unique_range = self.unique_range.clone();
377 #[cfg(not(feature = "stack-cache"))]
378 let unique_range = 0..self.len();
380 if disable_start <= unique_range.end {
381 let lower = unique_range.start.max(disable_start);
382 let upper = unique_range.end;
383 for item in &mut self.borrows[lower..upper] {
384 if item.perm() == Permission::Unique {
385 log::trace!("access: disabling item {:?}", item);
387 item.set_permission(Permission::Disabled);
388 // Also update all copies of this item in the cache.
389 #[cfg(feature = "stack-cache")]
390 for it in &mut self.cache.items {
391 if it.tag() == item.tag() {
392 it.set_permission(Permission::Disabled);
399 #[cfg(feature = "stack-cache")]
400 if disable_start <= self.unique_range.start {
401 // We disabled all Unique items
402 self.unique_range.start = 0;
403 self.unique_range.end = 0;
405 // Truncate the range to only include items up to the index that we started disabling
407 self.unique_range.end = self.unique_range.end.min(disable_start);
410 #[cfg(all(feature = "stack-cache", debug_assertions))]
411 self.verify_cache_consistency();
416 /// Produces an iterator which iterates over `range` in reverse, and when dropped removes that
417 /// range of `Item`s from this `Stack`.
418 pub fn pop_items_after<V: FnMut(Item) -> crate::InterpResult<'tcx>>(
422 ) -> crate::InterpResult<'tcx> {
423 while self.borrows.len() > start {
424 let item = self.borrows.pop().unwrap();
428 #[cfg(feature = "stack-cache")]
429 if !self.borrows.is_empty() {
430 // After we remove from the borrow stack, every aspect of our caching may be invalid, but it is
431 // also possible that the whole cache is still valid. So we call this method to repair what
432 // aspects of the cache are now invalid, instead of resetting the whole thing to a trivially
433 // valid default state.
434 let base_tag = self.borrows[0];
437 // Remove invalid entries from the cache by rotating them to the end of the cache, then
438 // keep track of how many invalid elements there are and overwrite them with the base tag.
439 // The base tag here serves as a harmless default value.
440 for _ in 0..CACHE_LEN - 1 {
441 if self.cache.idx[cursor] >= start {
442 self.cache.idx[cursor..CACHE_LEN - removed].rotate_left(1);
443 self.cache.items[cursor..CACHE_LEN - removed].rotate_left(1);
449 for i in CACHE_LEN - removed - 1..CACHE_LEN {
450 self.cache.idx[i] = 0;
451 self.cache.items[i] = base_tag;
454 if start <= self.unique_range.start {
455 // We removed all the Unique items
456 self.unique_range = 0..0;
458 // Ensure the range doesn't extend past the new top of the stack
459 self.unique_range.end = self.unique_range.end.min(start);
462 self.unique_range = 0..0;
465 #[cfg(all(feature = "stack-cache", debug_assertions))]
466 self.verify_cache_consistency();