1 use crate::stacked_borrows::{AccessKind, Item, Permission, SbTag, SbTagExtra};
2 use rustc_data_structures::fx::FxHashSet;
3 #[cfg(feature = "stack-cache")]
6 /// Exactly what cache size we should use is a difficult tradeoff. There will always be some
7 /// workload which has a `SbTag` working set which exceeds the size of the cache, and ends up
8 /// falling back to linear searches of the borrow stack very often.
9 /// The cost of making this value too large is that the loop in `Stack::insert` which ensures the
10 /// entries in the cache stay correct after an insert becomes expensive.
11 #[cfg(feature = "stack-cache")]
12 const CACHE_LEN: usize = 32;
14 /// Extra per-location state.
15 #[derive(Clone, Debug)]
17 /// Used *mostly* as a stack; never empty.
19 /// * Above a `SharedReadOnly` there can only be more `SharedReadOnly`.
20 /// * Except for `Untagged`, no tag occurs in the stack more than once.
22 /// If this is `Some(id)`, then the actual current stack is unknown. This can happen when
23 /// wildcard pointers are used to access this location. What we do know is that `borrows` are at
24 /// the top of the stack, and below it are arbitrarily many items whose `tag` is strictly less
26 /// When the bottom is unknown, `borrows` always has a `SharedReadOnly` or `Unique` at the bottom;
27 /// we never have the unknown-to-known boundary in an SRW group.
28 unknown_bottom: Option<SbTag>,
30 /// A small LRU cache of searches of the borrow stack. This only caches accesses of `Tagged`,
31 /// because those are unique in the stack.
32 #[cfg(feature = "stack-cache")]
34 /// On a read, we need to disable all `Unique` above the granting item. We can avoid most of
35 /// this scan by keeping track of the region of the borrow stack that may contain `Unique`s.
36 #[cfg(feature = "stack-cache")]
37 unique_range: Range<usize>,
40 /// A very small cache of searches of the borrow stack
41 /// This maps tags to locations in the borrow stack. Any use of this still needs to do a
42 /// probably-cold random access into the borrow stack to figure out what `Permission` an
43 /// `SbTag` grants. We could avoid this by also storing the `Permission` in the cache, but
44 /// most lookups into the cache are immediately followed by access of the full borrow stack anyway.
45 #[cfg(feature = "stack-cache")]
46 #[derive(Clone, Debug)]
48 tags: [SbTag; CACHE_LEN], // Hot in find_granting
49 idx: [usize; CACHE_LEN], // Hot in grant
52 #[cfg(feature = "stack-cache")]
54 /// When a tag is used, we call this function to add or refresh it in the cache.
56 /// We use position in the cache to represent how recently a tag was used; the first position
57 /// is the most recently used tag. So an add shifts every element towards the end, and inserts
58 /// the new element at the start. We lose the last element.
59 /// This strategy is effective at keeping the most-accessed tags in the cache, but it costs a
60 /// linear shift across the entire cache when we add a new tag.
61 fn add(&mut self, idx: usize, tag: SbTag) {
62 self.tags.copy_within(0..CACHE_LEN - 1, 1);
64 self.idx.copy_within(0..CACHE_LEN - 1, 1);
69 impl PartialEq for Stack {
70 fn eq(&self, other: &Self) -> bool {
71 // All the semantics of Stack are in self.borrows, everything else is caching
72 self.borrows == other.borrows
79 /// Panics if any of the caching mechanisms have broken,
80 /// - The StackCache indices don't refer to the parallel tags,
81 /// - There are no Unique tags outside of first_unique..last_unique
82 #[cfg(feature = "expensive-debug-assertions")]
83 fn verify_cache_consistency(&self) {
84 if self.borrows.len() > CACHE_LEN {
85 for (tag, stack_idx) in self.cache.tags.iter().zip(self.cache.idx.iter()) {
86 assert_eq!(self.borrows[*stack_idx].tag, *tag);
90 for (idx, item) in self.borrows.iter().enumerate() {
91 if item.perm == Permission::Unique {
93 self.unique_range.contains(&idx),
102 /// Find the item granting the given kind of access to the given tag, and return where
103 /// it is on the stack. For wildcard tags, the given index is approximate, but if *no*
104 /// index is given it means the match was *not* in the known part of the stack.
105 /// `Ok(None)` indicates it matched the "unknown" part of the stack.
106 /// `Err` indicates it was not found.
107 pub fn find_granting(
111 exposed_tags: &FxHashSet<SbTag>,
112 ) -> Result<Option<usize>, ()> {
113 #[cfg(feature = "expensive-debug-assertions")]
114 self.verify_cache_consistency();
116 let SbTagExtra::Concrete(tag) = tag else {
117 // Handle the wildcard case.
118 // Go search the stack for an exposed tag.
122 .enumerate() // we also need to know *where* in the stack
123 .rev() // search top-to-bottom
124 .find_map(|(idx, item)| {
125 // If the item fits and *might* be this wildcard, use it.
126 if item.perm.grants(access) && exposed_tags.contains(&item.tag) {
133 return Ok(Some(idx));
135 // If we couldn't find it in the stack, check the unknown bottom.
136 return if self.unknown_bottom.is_some() { Ok(None) } else { Err(()) };
139 if let Some(idx) = self.find_granting_tagged(access, tag) {
140 return Ok(Some(idx));
143 // Couldn't find it in the stack; but if there is an unknown bottom it might be there.
144 let found = self.unknown_bottom.is_some_and(|&unknown_limit| {
145 tag.0 < unknown_limit.0 // unknown_limit is an upper bound for what can be in the unknown bottom.
147 if found { Ok(None) } else { Err(()) }
150 fn find_granting_tagged(&mut self, access: AccessKind, tag: SbTag) -> Option<usize> {
151 #[cfg(feature = "stack-cache")]
152 if let Some(idx) = self.find_granting_cache(access, tag) {
156 // If we didn't find the tag in the cache, fall back to a linear search of the
157 // whole stack, and add the tag to the stack.
158 for (stack_idx, item) in self.borrows.iter().enumerate().rev() {
159 if tag == item.tag && item.perm.grants(access) {
160 #[cfg(feature = "stack-cache")]
161 self.cache.add(stack_idx, tag);
162 return Some(stack_idx);
168 #[cfg(feature = "stack-cache")]
169 fn find_granting_cache(&mut self, access: AccessKind, tag: SbTag) -> Option<usize> {
170 // When the borrow stack is empty, there are no tags we could put into the cache that would
171 // be valid. Additionally, since lookups into the cache are a linear search it doesn't make
172 // sense to use the cache when it is no smaller than a search of the borrow stack itself.
173 if self.borrows.len() <= CACHE_LEN {
176 // Search the cache for the tag we're looking up
177 let cache_idx = self.cache.tags.iter().position(|t| *t == tag)?;
178 let stack_idx = self.cache.idx[cache_idx];
179 // If we found the tag, look up its position in the stack to see if it grants
180 // the required permission
181 if self.borrows[stack_idx].perm.grants(access) {
182 // If it does, and it's not already in the most-recently-used position, move it there.
183 // Except if the tag is in position 1, this is equivalent to just a swap, so do that.
185 self.cache.tags.swap(0, 1);
186 self.cache.idx.swap(0, 1);
187 } else if cache_idx > 1 {
188 self.cache.add(stack_idx, tag);
192 // Tag is in the cache, but it doesn't grant the required permission
197 pub fn insert(&mut self, new_idx: usize, new: Item) {
198 self.borrows.insert(new_idx, new);
200 #[cfg(feature = "stack-cache")]
201 self.insert_cache(new_idx, new);
204 #[cfg(feature = "stack-cache")]
205 fn insert_cache(&mut self, new_idx: usize, new: Item) {
206 // Adjust the possibly-unique range if an insert occurs before or within it
207 if self.unique_range.start >= new_idx {
208 self.unique_range.start += 1;
210 if self.unique_range.end >= new_idx {
211 self.unique_range.end += 1;
213 if new.perm == Permission::Unique {
214 // Make sure the possibly-unique range contains the new borrow
215 self.unique_range.start = self.unique_range.start.min(new_idx);
216 self.unique_range.end = self.unique_range.end.max(new_idx + 1);
219 // The above insert changes the meaning of every index in the cache >= new_idx, so now
220 // we need to find every one of those indexes and increment it.
221 // But if the insert is at the end (equivalent to a push), we can skip this step because
222 // it didn't change the position of any other tags.
223 if new_idx != self.borrows.len() - 1 {
224 for idx in &mut self.cache.idx {
231 // This primes the cache for the next access, which is almost always the just-added tag.
232 self.cache.add(new_idx, new.tag);
234 #[cfg(feature = "expensive-debug-assertions")]
235 self.verify_cache_consistency();
238 /// Construct a new `Stack` using the passed `Item` as the base tag.
239 pub fn new(item: Item) -> Self {
242 unknown_bottom: None,
243 #[cfg(feature = "stack-cache")]
244 cache: StackCache { idx: [0; CACHE_LEN], tags: [item.tag; CACHE_LEN] },
245 #[cfg(feature = "stack-cache")]
246 unique_range: if item.perm == Permission::Unique { 0..1 } else { 0..0 },
250 pub fn get(&self, idx: usize) -> Option<Item> {
251 self.borrows.get(idx).cloned()
254 #[allow(clippy::len_without_is_empty)] // Stacks are never empty
255 pub fn len(&self) -> usize {
259 pub fn unknown_bottom(&self) -> Option<SbTag> {
263 pub fn set_unknown_bottom(&mut self, tag: SbTag) {
264 self.borrows.clear();
265 self.unknown_bottom = Some(tag);
268 /// Find all `Unique` elements in this borrow stack above `granting_idx`, pass a copy of them
269 /// to the `visitor`, then set their `Permission` to `Disabled`.
270 pub fn disable_uniques_starting_at<V: FnMut(Item) -> crate::InterpResult<'tcx>>(
272 disable_start: usize,
274 ) -> crate::InterpResult<'tcx> {
275 #[cfg(feature = "stack-cache")]
276 let unique_range = self.unique_range.clone();
277 #[cfg(not(feature = "stack-cache"))]
278 let unique_range = 0..self.len();
280 if disable_start <= unique_range.end {
281 // add 1 so we don't disable the granting item
282 let lower = unique_range.start.max(disable_start);
283 let upper = (unique_range.end + 1).min(self.borrows.len());
284 for item in &mut self.borrows[lower..upper] {
285 if item.perm == Permission::Unique {
286 log::trace!("access: disabling item {:?}", item);
288 item.perm = Permission::Disabled;
293 #[cfg(feature = "stack-cache")]
294 if disable_start < self.unique_range.start {
295 // We disabled all Unique items
296 self.unique_range.start = 0;
297 self.unique_range.end = 0;
299 // Truncate the range to disable_start. This is + 2 because we are only removing
300 // elements after disable_start, and this range does not include the end.
301 self.unique_range.end = self.unique_range.end.min(disable_start + 1);
304 #[cfg(feature = "expensive-debug-assertions")]
305 self.verify_cache_consistency();
310 /// Produces an iterator which iterates over `range` in reverse, and when dropped removes that
311 /// range of `Item`s from this `Stack`.
312 pub fn pop_items_after<V: FnMut(Item) -> crate::InterpResult<'tcx>>(
316 ) -> crate::InterpResult<'tcx> {
317 while self.borrows.len() > start {
318 let item = self.borrows.pop().unwrap();
322 #[cfg(feature = "stack-cache")]
323 if !self.borrows.is_empty() {
324 // After we remove from the borrow stack, every aspect of our caching may be invalid, but it is
325 // also possible that the whole cache is still valid. So we call this method to repair what
326 // aspects of the cache are now invalid, instead of resetting the whole thing to a trivially
327 // valid default state.
328 let base_tag = self.borrows[0].tag;
331 // Remove invalid entries from the cache by rotating them to the end of the cache, then
332 // keep track of how many invalid elements there are and overwrite them with the base tag.
333 // The base tag here serves as a harmless default value.
334 for _ in 0..CACHE_LEN - 1 {
335 if self.cache.idx[cursor] >= start {
336 self.cache.idx[cursor..CACHE_LEN - removed].rotate_left(1);
337 self.cache.tags[cursor..CACHE_LEN - removed].rotate_left(1);
343 for i in CACHE_LEN - removed - 1..CACHE_LEN {
344 self.cache.idx[i] = 0;
345 self.cache.tags[i] = base_tag;
348 if start < self.unique_range.start.saturating_sub(1) {
349 // We removed all the Unique items
350 self.unique_range = 0..0;
352 // Ensure the range doesn't extend past the new top of the stack
353 self.unique_range.end = self.unique_range.end.min(start + 1);
356 self.unique_range = 0..0;
359 #[cfg(feature = "expensive-debug-assertions")]
360 self.verify_cache_consistency();