1 //! The virtual memory representation of the MIR interpreter.
4 use std::convert::{TryFrom, TryInto};
8 use std::ops::{Deref, Range};
11 use rustc_ast::Mutability;
12 use rustc_data_structures::intern::Interned;
13 use rustc_data_structures::sorted_map::SortedMap;
14 use rustc_span::DUMMY_SP;
15 use rustc_target::abi::{Align, HasDataLayout, Size};
18 read_target_uint, write_target_uint, AllocId, InterpError, InterpResult, Pointer, Provenance,
19 ResourceExhaustionInfo, Scalar, ScalarMaybeUninit, ScalarSizeMismatch, UndefinedBehaviorInfo,
20 UninitBytesAccess, UnsupportedOpInfo,
24 /// This type represents an Allocation in the Miri/CTFE core engine.
26 /// Its public API is rather low-level, working directly with allocation offsets and a custom error
27 /// type to account for the lack of an AllocId on this level. The Miri/CTFE core engine `memory`
28 /// module provides higher-level access.
29 // Note: for performance reasons when interning, some of the `Allocation` fields can be partially
30 // hashed. (see the `Hash` impl below for more details), so the impl is not derived.
31 #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
33 pub struct Allocation<Tag = AllocId, Extra = ()> {
34 /// The actual bytes of the allocation.
35 /// Note that the bytes of a pointer represent the offset of the pointer.
37 /// Maps from byte addresses to extra data for each pointer.
38 /// Only the first byte of a pointer is inserted into the map; i.e.,
39 /// every entry in this map applies to `pointer_size` consecutive bytes starting
40 /// at the given offset.
41 relocations: Relocations<Tag>,
42 /// Denotes which part of this allocation is initialized.
44 /// The alignment of the allocation to detect unaligned reads.
45 /// (`Align` guarantees that this is a power of two.)
47 /// `true` if the allocation is mutable.
48 /// Also used by codegen to determine if a static should be put into mutable memory,
49 /// which happens for `static mut` and `static` with interior mutability.
50 pub mutability: Mutability,
51 /// Extra state for the machine.
55 /// This is the maximum size we will hash at a time, when interning an `Allocation` and its
56 /// `InitMask`. Note, we hash that amount of bytes twice: at the start, and at the end of a buffer.
57 /// Used when these two structures are large: we only partially hash the larger fields in that
58 /// situation. See the comment at the top of their respective `Hash` impl for more details.
59 const MAX_BYTES_TO_HASH: usize = 64;
61 /// This is the maximum size (in bytes) for which a buffer will be fully hashed, when interning.
62 /// Otherwise, it will be partially hashed in 2 slices, requiring at least 2 `MAX_BYTES_TO_HASH`
64 const MAX_HASHED_BUFFER_LEN: usize = 2 * MAX_BYTES_TO_HASH;
66 // Const allocations are only hashed for interning. However, they can be large, making the hashing
67 // expensive especially since it uses `FxHash`: it's better suited to short keys, not potentially
68 // big buffers like the actual bytes of allocation. We can partially hash some fields when they're
70 impl hash::Hash for Allocation {
71 fn hash<H: hash::Hasher>(&self, state: &mut H) {
72 // Partially hash the `bytes` buffer when it is large. To limit collisions with common
73 // prefixes and suffixes, we hash the length and some slices of the buffer.
74 let byte_count = self.bytes.len();
75 if byte_count > MAX_HASHED_BUFFER_LEN {
76 // Hash the buffer's length.
77 byte_count.hash(state);
79 // And its head and tail.
80 self.bytes[..MAX_BYTES_TO_HASH].hash(state);
81 self.bytes[byte_count - MAX_BYTES_TO_HASH..].hash(state);
83 self.bytes.hash(state);
86 // Hash the other fields as usual.
87 self.relocations.hash(state);
88 self.init_mask.hash(state);
89 self.align.hash(state);
90 self.mutability.hash(state);
91 self.extra.hash(state);
95 /// Interned types generally have an `Outer` type and an `Inner` type, where
96 /// `Outer` is a newtype around `Interned<Inner>`, and all the operations are
97 /// done on `Outer`, because all occurrences are interned. E.g. `Ty` is an
98 /// outer type and `TyS` is its inner type.
100 /// Here things are different because only const allocations are interned. This
101 /// means that both the inner type (`Allocation`) and the outer type
102 /// (`ConstAllocation`) are used quite a bit.
103 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable)]
104 #[rustc_pass_by_value]
105 pub struct ConstAllocation<'tcx, Tag = AllocId, Extra = ()>(
106 pub Interned<'tcx, Allocation<Tag, Extra>>,
109 impl<'tcx> fmt::Debug for ConstAllocation<'tcx> {
110 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
111 // This matches how `Allocation` is printed. We print it like this to
112 // avoid having to update expected output in a lot of tests.
113 write!(f, "{:?}", self.inner())
117 impl<'tcx, Tag, Extra> ConstAllocation<'tcx, Tag, Extra> {
118 pub fn inner(self) -> &'tcx Allocation<Tag, Extra> {
123 /// We have our own error type that does not know about the `AllocId`; that information
124 /// is added when converting to `InterpError`.
126 pub enum AllocError {
127 /// A scalar had the wrong size.
128 ScalarSizeMismatch(ScalarSizeMismatch),
129 /// Encountered a pointer where we needed raw bytes.
131 /// Partially overwriting a pointer.
132 PartialPointerOverwrite(Size),
133 /// Using uninitialized data where it is not allowed.
134 InvalidUninitBytes(Option<UninitBytesAccess>),
136 pub type AllocResult<T = ()> = Result<T, AllocError>;
138 impl From<ScalarSizeMismatch> for AllocError {
139 fn from(s: ScalarSizeMismatch) -> Self {
140 AllocError::ScalarSizeMismatch(s)
145 pub fn to_interp_error<'tcx>(self, alloc_id: AllocId) -> InterpError<'tcx> {
148 ScalarSizeMismatch(s) => {
149 InterpError::UndefinedBehavior(UndefinedBehaviorInfo::ScalarSizeMismatch(s))
151 ReadPointerAsBytes => InterpError::Unsupported(UnsupportedOpInfo::ReadPointerAsBytes),
152 PartialPointerOverwrite(offset) => InterpError::Unsupported(
153 UnsupportedOpInfo::PartialPointerOverwrite(Pointer::new(alloc_id, offset)),
155 InvalidUninitBytes(info) => InterpError::UndefinedBehavior(
156 UndefinedBehaviorInfo::InvalidUninitBytes(info.map(|b| (alloc_id, b))),
162 /// The information that makes up a memory access: offset and size.
163 #[derive(Copy, Clone, Debug)]
164 pub struct AllocRange {
169 /// Free-starting constructor for less syntactic overhead.
171 pub fn alloc_range(start: Size, size: Size) -> AllocRange {
172 AllocRange { start, size }
177 pub fn end(self) -> Size {
178 self.start + self.size // This does overflow checking.
181 /// Returns the `subrange` within this range; panics if it is not a subrange.
183 pub fn subrange(self, subrange: AllocRange) -> AllocRange {
184 let sub_start = self.start + subrange.start;
185 let range = alloc_range(sub_start, subrange.size);
186 assert!(range.end() <= self.end(), "access outside the bounds for given AllocRange");
191 // The constructors are all without extra; the extra gets added by a machine hook later.
192 impl<Tag> Allocation<Tag> {
193 /// Creates an allocation initialized by the given bytes
194 pub fn from_bytes<'a>(
195 slice: impl Into<Cow<'a, [u8]>>,
197 mutability: Mutability,
199 let bytes = Box::<[u8]>::from(slice.into());
200 let size = Size::from_bytes(bytes.len());
203 relocations: Relocations::new(),
204 init_mask: InitMask::new(size, true),
211 pub fn from_bytes_byte_aligned_immutable<'a>(slice: impl Into<Cow<'a, [u8]>>) -> Self {
212 Allocation::from_bytes(slice, Align::ONE, Mutability::Not)
215 /// Try to create an Allocation of `size` bytes, failing if there is not enough memory
216 /// available to the compiler to do so.
217 pub fn uninit<'tcx>(size: Size, align: Align, panic_on_fail: bool) -> InterpResult<'tcx, Self> {
218 let bytes = Box::<[u8]>::try_new_zeroed_slice(size.bytes_usize()).map_err(|_| {
219 // This results in an error that can happen non-deterministically, since the memory
220 // available to the compiler can change between runs. Normally queries are always
221 // deterministic. However, we can be non-deterministic here because all uses of const
222 // evaluation (including ConstProp!) will make compilation fail (via hard error
223 // or ICE) upon encountering a `MemoryExhausted` error.
225 panic!("Allocation::uninit called with panic_on_fail had allocation failure")
227 ty::tls::with(|tcx| {
228 tcx.sess.delay_span_bug(DUMMY_SP, "exhausted memory during interpretation")
230 InterpError::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted)
232 // SAFETY: the box was zero-allocated, which is a valid initial value for Box<[u8]>
233 let bytes = unsafe { bytes.assume_init() };
236 relocations: Relocations::new(),
237 init_mask: InitMask::new(size, false),
239 mutability: Mutability::Mut,
246 /// Convert Tag and add Extra fields
247 pub fn convert_tag_add_extra<Tag, Extra, Err>(
249 cx: &impl HasDataLayout,
251 mut tagger: impl FnMut(Pointer<AllocId>) -> Result<Pointer<Tag>, Err>,
252 ) -> Result<Allocation<Tag, Extra>, Err> {
253 // Compute new pointer tags, which also adjusts the bytes.
254 let mut bytes = self.bytes;
255 let mut new_relocations = Vec::with_capacity(self.relocations.0.len());
256 let ptr_size = cx.data_layout().pointer_size.bytes_usize();
257 let endian = cx.data_layout().endian;
258 for &(offset, alloc_id) in self.relocations.iter() {
259 let idx = offset.bytes_usize();
260 let ptr_bytes = &mut bytes[idx..idx + ptr_size];
261 let bits = read_target_uint(endian, ptr_bytes).unwrap();
262 let (ptr_tag, ptr_offset) =
263 tagger(Pointer::new(alloc_id, Size::from_bytes(bits)))?.into_parts();
264 write_target_uint(endian, ptr_bytes, ptr_offset.bytes().into()).unwrap();
265 new_relocations.push((offset, ptr_tag));
267 // Create allocation.
270 relocations: Relocations::from_presorted(new_relocations),
271 init_mask: self.init_mask,
273 mutability: self.mutability,
279 /// Raw accessors. Provide access to otherwise private bytes.
280 impl<Tag, Extra> Allocation<Tag, Extra> {
281 pub fn len(&self) -> usize {
285 pub fn size(&self) -> Size {
286 Size::from_bytes(self.len())
289 /// Looks at a slice which may describe uninitialized bytes or describe a relocation. This differs
290 /// from `get_bytes_with_uninit_and_ptr` in that it does no relocation checks (even on the
292 /// This must not be used for reads affecting the interpreter execution.
293 pub fn inspect_with_uninit_and_ptr_outside_interpreter(&self, range: Range<usize>) -> &[u8] {
297 /// Returns the mask indicating which bytes are initialized.
298 pub fn init_mask(&self) -> &InitMask {
302 /// Returns the relocation list.
303 pub fn relocations(&self) -> &Relocations<Tag> {
309 impl<Tag: Provenance, Extra> Allocation<Tag, Extra> {
310 /// This is the entirely abstraction-violating way to just grab the raw bytes without
311 /// caring about relocations. It just deduplicates some code between `read_scalar`
312 /// and `get_bytes_internal`.
313 fn get_bytes_even_more_internal(&self, range: AllocRange) -> &[u8] {
314 &self.bytes[range.start.bytes_usize()..range.end().bytes_usize()]
317 /// The last argument controls whether we error out when there are uninitialized or pointer
318 /// bytes. However, we *always* error when there are relocations overlapping the edges of the
321 /// You should never call this, call `get_bytes` or `get_bytes_with_uninit_and_ptr` instead,
323 /// This function also guarantees that the resulting pointer will remain stable
324 /// even when new allocations are pushed to the `HashMap`. `mem_copy_repeatedly` relies
327 /// It is the caller's responsibility to check bounds and alignment beforehand.
328 fn get_bytes_internal(
330 cx: &impl HasDataLayout,
332 check_init_and_ptr: bool,
333 ) -> AllocResult<&[u8]> {
334 if check_init_and_ptr {
335 self.check_init(range)?;
336 self.check_relocations(cx, range)?;
338 // We still don't want relocations on the *edges*.
339 self.check_relocation_edges(cx, range)?;
342 Ok(self.get_bytes_even_more_internal(range))
345 /// Checks that these bytes are initialized and not pointer bytes, and then return them
348 /// It is the caller's responsibility to check bounds and alignment beforehand.
349 /// Most likely, you want to use the `PlaceTy` and `OperandTy`-based methods
350 /// on `InterpCx` instead.
352 pub fn get_bytes(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult<&[u8]> {
353 self.get_bytes_internal(cx, range, true)
356 /// It is the caller's responsibility to handle uninitialized and pointer bytes.
357 /// However, this still checks that there are no relocations on the *edges*.
359 /// It is the caller's responsibility to check bounds and alignment beforehand.
361 pub fn get_bytes_with_uninit_and_ptr(
363 cx: &impl HasDataLayout,
365 ) -> AllocResult<&[u8]> {
366 self.get_bytes_internal(cx, range, false)
369 /// Just calling this already marks everything as defined and removes relocations,
370 /// so be sure to actually put data there!
372 /// It is the caller's responsibility to check bounds and alignment beforehand.
373 /// Most likely, you want to use the `PlaceTy` and `OperandTy`-based methods
374 /// on `InterpCx` instead.
375 pub fn get_bytes_mut(
377 cx: &impl HasDataLayout,
379 ) -> AllocResult<&mut [u8]> {
380 self.mark_init(range, true);
381 self.clear_relocations(cx, range)?;
383 Ok(&mut self.bytes[range.start.bytes_usize()..range.end().bytes_usize()])
386 /// A raw pointer variant of `get_bytes_mut` that avoids invalidating existing aliases into this memory.
387 pub fn get_bytes_mut_ptr(
389 cx: &impl HasDataLayout,
391 ) -> AllocResult<*mut [u8]> {
392 self.mark_init(range, true);
393 self.clear_relocations(cx, range)?;
395 assert!(range.end().bytes_usize() <= self.bytes.len()); // need to do our own bounds-check
396 let begin_ptr = self.bytes.as_mut_ptr().wrapping_add(range.start.bytes_usize());
397 let len = range.end().bytes_usize() - range.start.bytes_usize();
398 Ok(ptr::slice_from_raw_parts_mut(begin_ptr, len))
402 /// Reading and writing.
403 impl<Tag: Provenance, Extra> Allocation<Tag, Extra> {
404 /// Validates that `ptr.offset` and `ptr.offset + size` do not point to the middle of a
405 /// relocation. If `allow_uninit`/`allow_ptr` is `false`, also enforces that the memory in the
406 /// given range contains no uninitialized bytes/relocations.
409 cx: &impl HasDataLayout,
414 // Check bounds and relocations on the edges.
415 self.get_bytes_with_uninit_and_ptr(cx, range)?;
416 // Check uninit and ptr.
418 self.check_init(range)?;
421 self.check_relocations(cx, range)?;
426 /// Reads a *non-ZST* scalar.
428 /// If `read_provenance` is `true`, this will also read provenance; otherwise (if the machine
429 /// supports that) provenance is entirely ignored.
431 /// ZSTs can't be read because in order to obtain a `Pointer`, we need to check
432 /// for ZSTness anyway due to integer pointers being valid for ZSTs.
434 /// It is the caller's responsibility to check bounds and alignment beforehand.
435 /// Most likely, you want to call `InterpCx::read_scalar` instead of this method.
438 cx: &impl HasDataLayout,
440 read_provenance: bool,
441 ) -> AllocResult<ScalarMaybeUninit<Tag>> {
443 assert_eq!(range.size, cx.data_layout().pointer_size);
446 // First and foremost, if anything is uninit, bail.
447 if self.is_init(range).is_err() {
448 // This inflates uninitialized bytes to the entire scalar, even if only a few
449 // bytes are uninitialized.
450 return Ok(ScalarMaybeUninit::Uninit);
453 // If we are doing a pointer read, and there is a relocation exactly where we
454 // are reading, then we can put data and relocation back together and return that.
455 if read_provenance && let Some(&prov) = self.relocations.get(&range.start) {
456 // We already checked init and relocations, so we can use this function.
457 let bytes = self.get_bytes_even_more_internal(range);
458 let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
459 let ptr = Pointer::new(prov, Size::from_bytes(bits));
460 return Ok(ScalarMaybeUninit::from_pointer(ptr, cx));
463 // If we are *not* reading a pointer, and we can just ignore relocations,
464 // then do exactly that.
465 if !read_provenance && Tag::OFFSET_IS_ADDR {
466 // We just strip provenance.
467 let bytes = self.get_bytes_even_more_internal(range);
468 let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
469 return Ok(ScalarMaybeUninit::Scalar(Scalar::from_uint(bits, range.size)));
472 // It's complicated. Better make sure there is no provenance anywhere.
473 // FIXME: If !OFFSET_IS_ADDR, this is the best we can do. But if OFFSET_IS_ADDR, then
474 // `read_pointer` is true and we ideally would distinguish the following two cases:
475 // - The entire `range` is covered by 2 relocations for the same provenance.
476 // Then we should return a pointer with that provenance.
477 // - The range has inhomogeneous provenance. Then we should return just the
479 let bytes = self.get_bytes(cx, range)?;
480 let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
481 Ok(ScalarMaybeUninit::Scalar(Scalar::from_uint(bits, range.size)))
484 /// Writes a *non-ZST* scalar.
486 /// ZSTs can't be read because in order to obtain a `Pointer`, we need to check
487 /// for ZSTness anyway due to integer pointers being valid for ZSTs.
489 /// It is the caller's responsibility to check bounds and alignment beforehand.
490 /// Most likely, you want to call `InterpCx::write_scalar` instead of this method.
491 #[instrument(skip(self, cx), level = "debug")]
494 cx: &impl HasDataLayout,
496 val: ScalarMaybeUninit<Tag>,
498 assert!(self.mutability == Mutability::Mut);
500 let val = match val {
501 ScalarMaybeUninit::Scalar(scalar) => scalar,
502 ScalarMaybeUninit::Uninit => {
503 return self.write_uninit(cx, range);
507 // `to_bits_or_ptr_internal` is the right method because we just want to store this data
508 // as-is into memory.
509 let (bytes, provenance) = match val.to_bits_or_ptr_internal(range.size)? {
511 let (provenance, offset) = val.into_parts();
512 (u128::from(offset.bytes()), Some(provenance))
514 Ok(data) => (data, None),
517 let endian = cx.data_layout().endian;
518 let dst = self.get_bytes_mut(cx, range)?;
519 write_target_uint(endian, dst, bytes).unwrap();
521 // See if we have to also write a relocation.
522 if let Some(provenance) = provenance {
523 self.relocations.0.insert(range.start, provenance);
529 /// Write "uninit" to the given memory range.
530 pub fn write_uninit(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
531 self.mark_init(range, false);
532 self.clear_relocations(cx, range)?;
538 impl<Tag: Copy, Extra> Allocation<Tag, Extra> {
539 /// Returns all relocations overlapping with the given pointer-offset pair.
540 fn get_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> &[(Size, Tag)] {
541 // We have to go back `pointer_size - 1` bytes, as that one would still overlap with
542 // the beginning of this range.
543 let start = range.start.bytes().saturating_sub(cx.data_layout().pointer_size.bytes() - 1);
544 self.relocations.range(Size::from_bytes(start)..range.end())
547 /// Returns whether this allocation has relocations overlapping with the given range.
549 /// Note: this function exists to allow `get_relocations` to be private, in order to somewhat
550 /// limit access to relocations outside of the `Allocation` abstraction.
552 pub fn has_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> bool {
553 !self.get_relocations(cx, range).is_empty()
556 /// Checks that there are no relocations overlapping with the given range.
558 fn check_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
559 if self.has_relocations(cx, range) { Err(AllocError::ReadPointerAsBytes) } else { Ok(()) }
562 /// Removes all relocations inside the given range.
563 /// If there are relocations overlapping with the edges, they
564 /// are removed as well *and* the bytes they cover are marked as
565 /// uninitialized. This is a somewhat odd "spooky action at a distance",
566 /// but it allows strictly more code to run than if we would just error
567 /// immediately in that case.
568 fn clear_relocations(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult
572 // Find the start and end of the given range and its outermost relocations.
573 let (first, last) = {
574 // Find all relocations overlapping the given range.
575 let relocations = self.get_relocations(cx, range);
576 if relocations.is_empty() {
581 relocations.first().unwrap().0,
582 relocations.last().unwrap().0 + cx.data_layout().pointer_size,
585 let start = range.start;
586 let end = range.end();
588 // We need to handle clearing the relocations from parts of a pointer.
589 // FIXME: Miri should preserve partial relocations; see
590 // https://github.com/rust-lang/miri/issues/2181.
592 if Tag::ERR_ON_PARTIAL_PTR_OVERWRITE {
593 return Err(AllocError::PartialPointerOverwrite(first));
596 "Partial pointer overwrite! De-initializing memory at offsets {first:?}..{start:?}."
598 self.init_mask.set_range(first, start, false);
601 if Tag::ERR_ON_PARTIAL_PTR_OVERWRITE {
602 return Err(AllocError::PartialPointerOverwrite(
603 last - cx.data_layout().pointer_size,
607 "Partial pointer overwrite! De-initializing memory at offsets {end:?}..{last:?}."
609 self.init_mask.set_range(end, last, false);
612 // Forget all the relocations.
613 // Since relocations do not overlap, we know that removing until `last` (exclusive) is fine,
614 // i.e., this will not remove any other relocations just after the ones we care about.
615 self.relocations.0.remove_range(first..last);
620 /// Errors if there are relocations overlapping with the edges of the
621 /// given memory range.
623 fn check_relocation_edges(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
624 self.check_relocations(cx, alloc_range(range.start, Size::ZERO))?;
625 self.check_relocations(cx, alloc_range(range.end(), Size::ZERO))?;
630 /// "Relocations" stores the provenance information of pointers stored in memory.
631 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
632 pub struct Relocations<Tag = AllocId>(SortedMap<Size, Tag>);
634 impl<Tag> Relocations<Tag> {
635 pub fn new() -> Self {
636 Relocations(SortedMap::new())
639 // The caller must guarantee that the given relocations are already sorted
640 // by address and contain no duplicates.
641 pub fn from_presorted(r: Vec<(Size, Tag)>) -> Self {
642 Relocations(SortedMap::from_presorted_elements(r))
646 impl<Tag> Deref for Relocations<Tag> {
647 type Target = SortedMap<Size, Tag>;
649 fn deref(&self) -> &Self::Target {
654 /// A partial, owned list of relocations to transfer into another allocation.
656 /// Offsets are already adjusted to the destination allocation.
657 pub struct AllocationRelocations<Tag> {
658 dest_relocations: Vec<(Size, Tag)>,
661 impl<Tag: Copy, Extra> Allocation<Tag, Extra> {
662 pub fn prepare_relocation_copy(
664 cx: &impl HasDataLayout,
668 ) -> AllocationRelocations<Tag> {
669 let relocations = self.get_relocations(cx, src);
670 if relocations.is_empty() {
671 return AllocationRelocations { dest_relocations: Vec::new() };
675 let mut new_relocations = Vec::with_capacity(relocations.len() * (count as usize));
677 // If `count` is large, this is rather wasteful -- we are allocating a big array here, which
678 // is mostly filled with redundant information since it's just N copies of the same `Tag`s
679 // at slightly adjusted offsets. The reason we do this is so that in `mark_relocation_range`
680 // we can use `insert_presorted`. That wouldn't work with an `Iterator` that just produces
681 // the right sequence of relocations for all N copies.
683 new_relocations.extend(relocations.iter().map(|&(offset, reloc)| {
684 // compute offset for current repetition
685 let dest_offset = dest + size * i; // `Size` operations
687 // shift offsets from source allocation to destination allocation
688 (offset + dest_offset) - src.start, // `Size` operations
694 AllocationRelocations { dest_relocations: new_relocations }
697 /// Applies a relocation copy.
698 /// The affected range, as defined in the parameters to `prepare_relocation_copy` is expected
699 /// to be clear of relocations.
701 /// This is dangerous to use as it can violate internal `Allocation` invariants!
702 /// It only exists to support an efficient implementation of `mem_copy_repeatedly`.
703 pub fn mark_relocation_range(&mut self, relocations: AllocationRelocations<Tag>) {
704 self.relocations.0.insert_presorted(relocations.dest_relocations);
708 ////////////////////////////////////////////////////////////////////////////////
709 // Uninitialized byte tracking
710 ////////////////////////////////////////////////////////////////////////////////
714 /// A bitmask where each bit refers to the byte with the same index. If the bit is `true`, the byte
715 /// is initialized. If it is `false` the byte is uninitialized.
716 // Note: for performance reasons when interning, some of the `InitMask` fields can be partially
717 // hashed. (see the `Hash` impl below for more details), so the impl is not derived.
718 #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
719 #[derive(HashStable)]
720 pub struct InitMask {
725 // Const allocations are only hashed for interning. However, they can be large, making the hashing
726 // expensive especially since it uses `FxHash`: it's better suited to short keys, not potentially
727 // big buffers like the allocation's init mask. We can partially hash some fields when they're
729 impl hash::Hash for InitMask {
730 fn hash<H: hash::Hasher>(&self, state: &mut H) {
731 const MAX_BLOCKS_TO_HASH: usize = MAX_BYTES_TO_HASH / std::mem::size_of::<Block>();
732 const MAX_BLOCKS_LEN: usize = MAX_HASHED_BUFFER_LEN / std::mem::size_of::<Block>();
734 // Partially hash the `blocks` buffer when it is large. To limit collisions with common
735 // prefixes and suffixes, we hash the length and some slices of the buffer.
736 let block_count = self.blocks.len();
737 if block_count > MAX_BLOCKS_LEN {
738 // Hash the buffer's length.
739 block_count.hash(state);
741 // And its head and tail.
742 self.blocks[..MAX_BLOCKS_TO_HASH].hash(state);
743 self.blocks[block_count - MAX_BLOCKS_TO_HASH..].hash(state);
745 self.blocks.hash(state);
748 // Hash the other fields as usual.
749 self.len.hash(state);
754 pub const BLOCK_SIZE: u64 = 64;
757 fn bit_index(bits: Size) -> (usize, usize) {
758 // BLOCK_SIZE is the number of bits that can fit in a `Block`.
759 // Each bit in a `Block` represents the initialization state of one byte of an allocation,
760 // so we use `.bytes()` here.
761 let bits = bits.bytes();
762 let a = bits / InitMask::BLOCK_SIZE;
763 let b = bits % InitMask::BLOCK_SIZE;
764 (usize::try_from(a).unwrap(), usize::try_from(b).unwrap())
768 fn size_from_bit_index(block: impl TryInto<u64>, bit: impl TryInto<u64>) -> Size {
769 let block = block.try_into().ok().unwrap();
770 let bit = bit.try_into().ok().unwrap();
771 Size::from_bytes(block * InitMask::BLOCK_SIZE + bit)
774 pub fn new(size: Size, state: bool) -> Self {
775 let mut m = InitMask { blocks: vec![], len: Size::ZERO };
780 pub fn set_range(&mut self, start: Size, end: Size, new_state: bool) {
783 self.grow(end - len, new_state);
785 self.set_range_inbounds(start, end, new_state);
788 pub fn set_range_inbounds(&mut self, start: Size, end: Size, new_state: bool) {
789 let (blocka, bita) = Self::bit_index(start);
790 let (blockb, bitb) = Self::bit_index(end);
791 if blocka == blockb {
792 // First set all bits except the first `bita`,
793 // then unset the last `64 - bitb` bits.
794 let range = if bitb == 0 {
797 (u64::MAX << bita) & (u64::MAX >> (64 - bitb))
800 self.blocks[blocka] |= range;
802 self.blocks[blocka] &= !range;
806 // across block boundaries
808 // Set `bita..64` to `1`.
809 self.blocks[blocka] |= u64::MAX << bita;
810 // Set `0..bitb` to `1`.
812 self.blocks[blockb] |= u64::MAX >> (64 - bitb);
814 // Fill in all the other blocks (much faster than one bit at a time).
815 for block in (blocka + 1)..blockb {
816 self.blocks[block] = u64::MAX;
819 // Set `bita..64` to `0`.
820 self.blocks[blocka] &= !(u64::MAX << bita);
821 // Set `0..bitb` to `0`.
823 self.blocks[blockb] &= !(u64::MAX >> (64 - bitb));
825 // Fill in all the other blocks (much faster than one bit at a time).
826 for block in (blocka + 1)..blockb {
827 self.blocks[block] = 0;
833 pub fn get(&self, i: Size) -> bool {
834 let (block, bit) = Self::bit_index(i);
835 (self.blocks[block] & (1 << bit)) != 0
839 pub fn set(&mut self, i: Size, new_state: bool) {
840 let (block, bit) = Self::bit_index(i);
841 self.set_bit(block, bit, new_state);
845 fn set_bit(&mut self, block: usize, bit: usize, new_state: bool) {
847 self.blocks[block] |= 1 << bit;
849 self.blocks[block] &= !(1 << bit);
853 pub fn grow(&mut self, amount: Size, new_state: bool) {
854 if amount.bytes() == 0 {
857 let unused_trailing_bits =
858 u64::try_from(self.blocks.len()).unwrap() * Self::BLOCK_SIZE - self.len.bytes();
859 if amount.bytes() > unused_trailing_bits {
860 let additional_blocks = amount.bytes() / Self::BLOCK_SIZE + 1;
862 // FIXME(oli-obk): optimize this by repeating `new_state as Block`.
863 iter::repeat(0).take(usize::try_from(additional_blocks).unwrap()),
866 let start = self.len;
868 self.set_range_inbounds(start, start + amount, new_state); // `Size` operation
871 /// Returns the index of the first bit in `start..end` (end-exclusive) that is equal to is_init.
872 fn find_bit(&self, start: Size, end: Size, is_init: bool) -> Option<Size> {
873 /// A fast implementation of `find_bit`,
874 /// which skips over an entire block at a time if it's all 0s (resp. 1s),
875 /// and finds the first 1 (resp. 0) bit inside a block using `trailing_zeros` instead of a loop.
877 /// Note that all examples below are written with 8 (instead of 64) bit blocks for simplicity,
878 /// and with the least significant bit (and lowest block) first:
880 /// 00000000|00000000
884 /// Also, if not stated, assume that `is_init = true`, that is, we are searching for the first 1 bit.
886 init_mask: &InitMask,
891 /// Search one block, returning the index of the first bit equal to `is_init`.
898 // For the following examples, assume this function was called with:
902 // Note that, for the examples in this function, the most significant bit is written first,
903 // which is backwards compared to the comments in `find_bit`/`find_bit_fast`.
905 // Invert bits so we're always looking for the first set bit.
908 let bits = if is_init { bits } else { !bits };
909 // Mask off unused start bits.
913 let bits = bits & (!0 << start_bit);
914 // Find set bit, if any.
915 // bit = trailing_zeros(0b11000000)
920 let bit = bits.trailing_zeros();
921 Some(InitMask::size_from_bit_index(block, bit))
929 // Convert `start` and `end` to block indexes and bit indexes within each block.
930 // We must convert `end` to an inclusive bound to handle block boundaries correctly.
934 // (a) 00000000|00000000 (b) 00000000|
935 // ^~~~~~~~~~~^ ^~~~~~~~~^
936 // start end start end
938 // In both cases, the block index of `end` is 1.
939 // But we do want to search block 1 in (a), and we don't in (b).
941 // We subtract 1 from both end positions to make them inclusive:
943 // (a) 00000000|00000000 (b) 00000000|
944 // ^~~~~~~~~~^ ^~~~~~~^
945 // start end_inclusive start end_inclusive
947 // For (a), the block index of `end_inclusive` is 1, and for (b), it's 0.
948 // This provides the desired behavior of searching blocks 0 and 1 for (a),
949 // and searching only block 0 for (b).
950 // There is no concern of overflows since we checked for `start >= end` above.
951 let (start_block, start_bit) = InitMask::bit_index(start);
952 let end_inclusive = Size::from_bytes(end.bytes() - 1);
953 let (end_block_inclusive, _) = InitMask::bit_index(end_inclusive);
955 // Handle first block: need to skip `start_bit` bits.
957 // We need to handle the first block separately,
958 // because there may be bits earlier in the block that should be ignored,
959 // such as the bit marked (1) in this example:
963 // (c) 01000000|00000000|00000001
964 // ^~~~~~~~~~~~~~~~~~^
967 search_block(init_mask.blocks[start_block], start_block, start_bit, is_init)
969 // If the range is less than a block, we may find a matching bit after `end`.
971 // For example, we shouldn't successfully find bit (2), because it's after `end`:
975 // (d) 00000001|00000000|00000001
979 // An alternative would be to mask off end bits in the same way as we do for start bits,
980 // but performing this check afterwards is faster and simpler to implement.
988 // Handle remaining blocks.
990 // We can skip over an entire block at once if it's all 0s (resp. 1s).
991 // The block marked (3) in this example is the first block that will be handled by this loop,
992 // and it will be skipped for that reason:
996 // (e) 01000000|00000000|00000001
997 // ^~~~~~~~~~~~~~~~~~^
999 if start_block < end_block_inclusive {
1000 // This loop is written in a specific way for performance.
1001 // Notably: `..end_block_inclusive + 1` is used for an inclusive range instead of `..=end_block_inclusive`,
1002 // and `.zip(start_block + 1..)` is used to track the index instead of `.enumerate().skip().take()`,
1003 // because both alternatives result in significantly worse codegen.
1004 // `end_block_inclusive + 1` is guaranteed not to wrap, because `end_block_inclusive <= end / BLOCK_SIZE`,
1005 // and `BLOCK_SIZE` (the number of bits per block) will always be at least 8 (1 byte).
1006 for (&bits, block) in init_mask.blocks[start_block + 1..end_block_inclusive + 1]
1008 .zip(start_block + 1..)
1010 if let Some(i) = search_block(bits, block, 0, is_init) {
1011 // If this is the last block, we may find a matching bit after `end`.
1013 // For example, we shouldn't successfully find bit (4), because it's after `end`:
1017 // (f) 00000001|00000000|00000001
1018 // ^~~~~~~~~~~~~~~~~~^
1021 // As above with example (d), we could handle the end block separately and mask off end bits,
1022 // but unconditionally searching an entire block at once and performing this check afterwards
1023 // is faster and much simpler to implement.
1036 #[cfg_attr(not(debug_assertions), allow(dead_code))]
1038 init_mask: &InitMask,
1043 (start..end).find(|&i| init_mask.get(i) == is_init)
1046 let result = find_bit_fast(self, start, end, is_init);
1050 find_bit_slow(self, start, end, is_init),
1051 "optimized implementation of find_bit is wrong for start={:?} end={:?} is_init={} init_mask={:#?}",
1062 /// A contiguous chunk of initialized or uninitialized memory.
1063 pub enum InitChunk {
1065 Uninit(Range<Size>),
1070 pub fn is_init(&self) -> bool {
1072 Self::Init(_) => true,
1073 Self::Uninit(_) => false,
1078 pub fn range(&self) -> Range<Size> {
1080 Self::Init(r) => r.clone(),
1081 Self::Uninit(r) => r.clone(),
1087 /// Checks whether the range `start..end` (end-exclusive) is entirely initialized.
1089 /// Returns `Ok(())` if it's initialized. Otherwise returns a range of byte
1090 /// indexes for the first contiguous span of the uninitialized access.
1092 pub fn is_range_initialized(&self, start: Size, end: Size) -> Result<(), Range<Size>> {
1094 return Err(self.len..end);
1097 let uninit_start = self.find_bit(start, end, false);
1099 match uninit_start {
1100 Some(uninit_start) => {
1101 let uninit_end = self.find_bit(uninit_start, end, true).unwrap_or(end);
1102 Err(uninit_start..uninit_end)
1108 /// Returns an iterator, yielding a range of byte indexes for each contiguous region
1109 /// of initialized or uninitialized bytes inside the range `start..end` (end-exclusive).
1111 /// The iterator guarantees the following:
1112 /// - Chunks are nonempty.
1113 /// - Chunks are adjacent (each range's start is equal to the previous range's end).
1114 /// - Chunks span exactly `start..end` (the first starts at `start`, the last ends at `end`).
1115 /// - Chunks alternate between [`InitChunk::Init`] and [`InitChunk::Uninit`].
1117 pub fn range_as_init_chunks(&self, start: Size, end: Size) -> InitChunkIter<'_> {
1118 assert!(end <= self.len);
1120 let is_init = if start < end {
1123 // `start..end` is empty: there are no chunks, so use some arbitrary value
1127 InitChunkIter { init_mask: self, is_init, start, end }
1131 /// Yields [`InitChunk`]s. See [`InitMask::range_as_init_chunks`].
1133 pub struct InitChunkIter<'a> {
1134 init_mask: &'a InitMask,
1135 /// Whether the next chunk we will return is initialized.
1136 /// If there are no more chunks, contains some arbitrary value.
1138 /// The current byte index into `init_mask`.
1140 /// The end byte index into `init_mask`.
1144 impl<'a> Iterator for InitChunkIter<'a> {
1145 type Item = InitChunk;
1148 fn next(&mut self) -> Option<Self::Item> {
1149 if self.start >= self.end {
1154 self.init_mask.find_bit(self.start, self.end, !self.is_init).unwrap_or(self.end);
1155 let range = self.start..end_of_chunk;
1158 Some(if self.is_init { InitChunk::Init(range) } else { InitChunk::Uninit(range) });
1160 self.is_init = !self.is_init;
1161 self.start = end_of_chunk;
1167 /// Uninitialized bytes.
1168 impl<Tag: Copy, Extra> Allocation<Tag, Extra> {
1169 /// Checks whether the given range is entirely initialized.
1171 /// Returns `Ok(())` if it's initialized. Otherwise returns the range of byte
1172 /// indexes of the first contiguous uninitialized access.
1173 fn is_init(&self, range: AllocRange) -> Result<(), Range<Size>> {
1174 self.init_mask.is_range_initialized(range.start, range.end()) // `Size` addition
1177 /// Checks that a range of bytes is initialized. If not, returns the `InvalidUninitBytes`
1178 /// error which will report the first range of bytes which is uninitialized.
1179 fn check_init(&self, range: AllocRange) -> AllocResult {
1180 self.is_init(range).map_err(|idx_range| {
1181 AllocError::InvalidUninitBytes(Some(UninitBytesAccess {
1182 access_offset: range.start,
1183 access_size: range.size,
1184 uninit_offset: idx_range.start,
1185 uninit_size: idx_range.end - idx_range.start, // `Size` subtraction
1190 fn mark_init(&mut self, range: AllocRange, is_init: bool) {
1191 if range.size.bytes() == 0 {
1194 assert!(self.mutability == Mutability::Mut);
1195 self.init_mask.set_range(range.start, range.end(), is_init);
1199 /// Run-length encoding of the uninit mask.
1200 /// Used to copy parts of a mask multiple times to another allocation.
1201 pub struct InitMaskCompressed {
1202 /// Whether the first range is initialized.
1204 /// The lengths of ranges that are run-length encoded.
1205 /// The initialization state of the ranges alternate starting with `initial`.
1206 ranges: smallvec::SmallVec<[u64; 1]>,
1209 impl InitMaskCompressed {
1210 pub fn no_bytes_init(&self) -> bool {
1211 // The `ranges` are run-length encoded and of alternating initialization state.
1212 // So if `ranges.len() > 1` then the second block is an initialized range.
1213 !self.initial && self.ranges.len() == 1
1217 /// Transferring the initialization mask to other allocations.
1218 impl<Tag, Extra> Allocation<Tag, Extra> {
1219 /// Creates a run-length encoding of the initialization mask; panics if range is empty.
1221 /// This is essentially a more space-efficient version of
1222 /// `InitMask::range_as_init_chunks(...).collect::<Vec<_>>()`.
1223 pub fn compress_uninit_range(&self, range: AllocRange) -> InitMaskCompressed {
1224 // Since we are copying `size` bytes from `src` to `dest + i * size` (`for i in 0..repeat`),
1225 // a naive initialization mask copying algorithm would repeatedly have to read the initialization mask from
1226 // the source and write it to the destination. Even if we optimized the memory accesses,
1227 // we'd be doing all of this `repeat` times.
1228 // Therefore we precompute a compressed version of the initialization mask of the source value and
1229 // then write it back `repeat` times without computing any more information from the source.
1231 // A precomputed cache for ranges of initialized / uninitialized bits
1232 // 0000010010001110 will become
1233 // `[5, 1, 2, 1, 3, 3, 1]`,
1234 // where each element toggles the state.
1236 let mut ranges = smallvec::SmallVec::<[u64; 1]>::new();
1238 let mut chunks = self.init_mask.range_as_init_chunks(range.start, range.end()).peekable();
1240 let initial = chunks.peek().expect("range should be nonempty").is_init();
1242 // Here we rely on `range_as_init_chunks` to yield alternating init/uninit chunks.
1243 for chunk in chunks {
1244 let len = chunk.range().end.bytes() - chunk.range().start.bytes();
1248 InitMaskCompressed { ranges, initial }
1251 /// Applies multiple instances of the run-length encoding to the initialization mask.
1253 /// This is dangerous to use as it can violate internal `Allocation` invariants!
1254 /// It only exists to support an efficient implementation of `mem_copy_repeatedly`.
1255 pub fn mark_compressed_init_range(
1257 defined: &InitMaskCompressed,
1261 // An optimization where we can just overwrite an entire range of initialization
1262 // bits if they are going to be uniformly `1` or `0`.
1263 if defined.ranges.len() <= 1 {
1264 self.init_mask.set_range_inbounds(
1266 range.start + range.size * repeat, // `Size` operations
1272 for mut j in 0..repeat {
1273 j *= range.size.bytes();
1274 j += range.start.bytes();
1275 let mut cur = defined.initial;
1276 for range in &defined.ranges {
1279 self.init_mask.set_range_inbounds(
1280 Size::from_bytes(old_j),
1281 Size::from_bytes(j),