1 //! The virtual memory representation of the MIR interpreter.
4 use std::convert::{TryFrom, TryInto};
8 use std::ops::{Deref, Range};
11 use rustc_ast::Mutability;
12 use rustc_data_structures::intern::Interned;
13 use rustc_data_structures::sorted_map::SortedMap;
14 use rustc_span::DUMMY_SP;
15 use rustc_target::abi::{Align, HasDataLayout, Size};
18 read_target_uint, write_target_uint, AllocId, InterpError, InterpResult, Pointer, Provenance,
19 ResourceExhaustionInfo, Scalar, ScalarMaybeUninit, ScalarSizeMismatch, UndefinedBehaviorInfo,
20 UninitBytesAccess, UnsupportedOpInfo,
24 /// This type represents an Allocation in the Miri/CTFE core engine.
26 /// Its public API is rather low-level, working directly with allocation offsets and a custom error
27 /// type to account for the lack of an AllocId on this level. The Miri/CTFE core engine `memory`
28 /// module provides higher-level access.
29 // Note: for performance reasons when interning, some of the `Allocation` fields can be partially
30 // hashed. (see the `Hash` impl below for more details), so the impl is not derived.
31 #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
33 pub struct Allocation<Tag = AllocId, Extra = ()> {
34 /// The actual bytes of the allocation.
35 /// Note that the bytes of a pointer represent the offset of the pointer.
37 /// Maps from byte addresses to extra data for each pointer.
38 /// Only the first byte of a pointer is inserted into the map; i.e.,
39 /// every entry in this map applies to `pointer_size` consecutive bytes starting
40 /// at the given offset.
41 relocations: Relocations<Tag>,
42 /// Denotes which part of this allocation is initialized.
44 /// The alignment of the allocation to detect unaligned reads.
45 /// (`Align` guarantees that this is a power of two.)
47 /// `true` if the allocation is mutable.
48 /// Also used by codegen to determine if a static should be put into mutable memory,
49 /// which happens for `static mut` and `static` with interior mutability.
50 pub mutability: Mutability,
51 /// Extra state for the machine.
55 /// This is the maximum size we will hash at a time, when interning an `Allocation` and its
56 /// `InitMask`. Note, we hash that amount of bytes twice: at the start, and at the end of a buffer.
57 /// Used when these two structures are large: we only partially hash the larger fields in that
58 /// situation. See the comment at the top of their respective `Hash` impl for more details.
59 const MAX_BYTES_TO_HASH: usize = 64;
61 /// This is the maximum size (in bytes) for which a buffer will be fully hashed, when interning.
62 /// Otherwise, it will be partially hashed in 2 slices, requiring at least 2 `MAX_BYTES_TO_HASH`
64 const MAX_HASHED_BUFFER_LEN: usize = 2 * MAX_BYTES_TO_HASH;
66 // Const allocations are only hashed for interning. However, they can be large, making the hashing
67 // expensive especially since it uses `FxHash`: it's better suited to short keys, not potentially
68 // big buffers like the actual bytes of allocation. We can partially hash some fields when they're
70 impl hash::Hash for Allocation {
71 fn hash<H: hash::Hasher>(&self, state: &mut H) {
72 // Partially hash the `bytes` buffer when it is large. To limit collisions with common
73 // prefixes and suffixes, we hash the length and some slices of the buffer.
74 let byte_count = self.bytes.len();
75 if byte_count > MAX_HASHED_BUFFER_LEN {
76 // Hash the buffer's length.
77 byte_count.hash(state);
79 // And its head and tail.
80 self.bytes[..MAX_BYTES_TO_HASH].hash(state);
81 self.bytes[byte_count - MAX_BYTES_TO_HASH..].hash(state);
83 self.bytes.hash(state);
86 // Hash the other fields as usual.
87 self.relocations.hash(state);
88 self.init_mask.hash(state);
89 self.align.hash(state);
90 self.mutability.hash(state);
91 self.extra.hash(state);
95 /// Interned types generally have an `Outer` type and an `Inner` type, where
96 /// `Outer` is a newtype around `Interned<Inner>`, and all the operations are
97 /// done on `Outer`, because all occurrences are interned. E.g. `Ty` is an
98 /// outer type and `TyS` is its inner type.
100 /// Here things are different because only const allocations are interned. This
101 /// means that both the inner type (`Allocation`) and the outer type
102 /// (`ConstAllocation`) are used quite a bit.
103 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable)]
104 #[rustc_pass_by_value]
105 pub struct ConstAllocation<'tcx, Tag = AllocId, Extra = ()>(
106 pub Interned<'tcx, Allocation<Tag, Extra>>,
109 impl<'tcx> fmt::Debug for ConstAllocation<'tcx> {
110 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
111 // This matches how `Allocation` is printed. We print it like this to
112 // avoid having to update expected output in a lot of tests.
113 write!(f, "{:?}", self.inner())
117 impl<'tcx, Tag, Extra> ConstAllocation<'tcx, Tag, Extra> {
118 pub fn inner(self) -> &'tcx Allocation<Tag, Extra> {
123 /// We have our own error type that does not know about the `AllocId`; that information
124 /// is added when converting to `InterpError`.
126 pub enum AllocError {
127 /// A scalar had the wrong size.
128 ScalarSizeMismatch(ScalarSizeMismatch),
129 /// Encountered a pointer where we needed raw bytes.
131 /// Partially overwriting a pointer.
132 PartialPointerOverwrite(Size),
133 /// Using uninitialized data where it is not allowed.
134 InvalidUninitBytes(Option<UninitBytesAccess>),
136 pub type AllocResult<T = ()> = Result<T, AllocError>;
138 impl From<ScalarSizeMismatch> for AllocError {
139 fn from(s: ScalarSizeMismatch) -> Self {
140 AllocError::ScalarSizeMismatch(s)
145 pub fn to_interp_error<'tcx>(self, alloc_id: AllocId) -> InterpError<'tcx> {
148 ScalarSizeMismatch(s) => {
149 InterpError::UndefinedBehavior(UndefinedBehaviorInfo::ScalarSizeMismatch(s))
151 ReadPointerAsBytes => InterpError::Unsupported(UnsupportedOpInfo::ReadPointerAsBytes),
152 PartialPointerOverwrite(offset) => InterpError::Unsupported(
153 UnsupportedOpInfo::PartialPointerOverwrite(Pointer::new(alloc_id, offset)),
155 InvalidUninitBytes(info) => InterpError::UndefinedBehavior(
156 UndefinedBehaviorInfo::InvalidUninitBytes(info.map(|b| (alloc_id, b))),
162 /// The information that makes up a memory access: offset and size.
163 #[derive(Copy, Clone, Debug)]
164 pub struct AllocRange {
169 /// Free-starting constructor for less syntactic overhead.
171 pub fn alloc_range(start: Size, size: Size) -> AllocRange {
172 AllocRange { start, size }
177 pub fn end(self) -> Size {
178 self.start + self.size // This does overflow checking.
181 /// Returns the `subrange` within this range; panics if it is not a subrange.
183 pub fn subrange(self, subrange: AllocRange) -> AllocRange {
184 let sub_start = self.start + subrange.start;
185 let range = alloc_range(sub_start, subrange.size);
186 assert!(range.end() <= self.end(), "access outside the bounds for given AllocRange");
191 // The constructors are all without extra; the extra gets added by a machine hook later.
192 impl<Tag> Allocation<Tag> {
193 /// Creates an allocation initialized by the given bytes
194 pub fn from_bytes<'a>(
195 slice: impl Into<Cow<'a, [u8]>>,
197 mutability: Mutability,
199 let bytes = Box::<[u8]>::from(slice.into());
200 let size = Size::from_bytes(bytes.len());
203 relocations: Relocations::new(),
204 init_mask: InitMask::new(size, true),
211 pub fn from_bytes_byte_aligned_immutable<'a>(slice: impl Into<Cow<'a, [u8]>>) -> Self {
212 Allocation::from_bytes(slice, Align::ONE, Mutability::Not)
215 /// Try to create an Allocation of `size` bytes, failing if there is not enough memory
216 /// available to the compiler to do so.
217 pub fn uninit<'tcx>(size: Size, align: Align, panic_on_fail: bool) -> InterpResult<'tcx, Self> {
218 let bytes = Box::<[u8]>::try_new_zeroed_slice(size.bytes_usize()).map_err(|_| {
219 // This results in an error that can happen non-deterministically, since the memory
220 // available to the compiler can change between runs. Normally queries are always
221 // deterministic. However, we can be non-deterministic here because all uses of const
222 // evaluation (including ConstProp!) will make compilation fail (via hard error
223 // or ICE) upon encountering a `MemoryExhausted` error.
225 panic!("Allocation::uninit called with panic_on_fail had allocation failure")
227 ty::tls::with(|tcx| {
228 tcx.sess.delay_span_bug(DUMMY_SP, "exhausted memory during interpretation")
230 InterpError::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted)
232 // SAFETY: the box was zero-allocated, which is a valid initial value for Box<[u8]>
233 let bytes = unsafe { bytes.assume_init() };
236 relocations: Relocations::new(),
237 init_mask: InitMask::new(size, false),
239 mutability: Mutability::Mut,
246 /// Convert Tag and add Extra fields
247 pub fn convert_tag_add_extra<Tag, Extra, Err>(
249 cx: &impl HasDataLayout,
251 mut tagger: impl FnMut(Pointer<AllocId>) -> Result<Pointer<Tag>, Err>,
252 ) -> Result<Allocation<Tag, Extra>, Err> {
253 // Compute new pointer tags, which also adjusts the bytes.
254 let mut bytes = self.bytes;
255 let mut new_relocations = Vec::with_capacity(self.relocations.0.len());
256 let ptr_size = cx.data_layout().pointer_size.bytes_usize();
257 let endian = cx.data_layout().endian;
258 for &(offset, alloc_id) in self.relocations.iter() {
259 let idx = offset.bytes_usize();
260 let ptr_bytes = &mut bytes[idx..idx + ptr_size];
261 let bits = read_target_uint(endian, ptr_bytes).unwrap();
262 let (ptr_tag, ptr_offset) =
263 tagger(Pointer::new(alloc_id, Size::from_bytes(bits)))?.into_parts();
264 write_target_uint(endian, ptr_bytes, ptr_offset.bytes().into()).unwrap();
265 new_relocations.push((offset, ptr_tag));
267 // Create allocation.
270 relocations: Relocations::from_presorted(new_relocations),
271 init_mask: self.init_mask,
273 mutability: self.mutability,
279 /// Raw accessors. Provide access to otherwise private bytes.
280 impl<Tag, Extra> Allocation<Tag, Extra> {
281 pub fn len(&self) -> usize {
285 pub fn size(&self) -> Size {
286 Size::from_bytes(self.len())
289 /// Looks at a slice which may describe uninitialized bytes or describe a relocation. This differs
290 /// from `get_bytes_with_uninit_and_ptr` in that it does no relocation checks (even on the
292 /// This must not be used for reads affecting the interpreter execution.
293 pub fn inspect_with_uninit_and_ptr_outside_interpreter(&self, range: Range<usize>) -> &[u8] {
297 /// Returns the mask indicating which bytes are initialized.
298 pub fn init_mask(&self) -> &InitMask {
302 /// Returns the relocation list.
303 pub fn relocations(&self) -> &Relocations<Tag> {
309 impl<Tag: Provenance, Extra> Allocation<Tag, Extra> {
310 /// This is the entirely abstraction-violating way to just grab the raw bytes without
311 /// caring about relocations. It just deduplicates some code between `read_scalar`
312 /// and `get_bytes_internal`.
313 fn get_bytes_even_more_internal(&self, range: AllocRange) -> &[u8] {
314 &self.bytes[range.start.bytes_usize()..range.end().bytes_usize()]
317 /// The last argument controls whether we error out when there are uninitialized or pointer
318 /// bytes. However, we *always* error when there are relocations overlapping the edges of the
321 /// You should never call this, call `get_bytes` or `get_bytes_with_uninit_and_ptr` instead,
323 /// This function also guarantees that the resulting pointer will remain stable
324 /// even when new allocations are pushed to the `HashMap`. `mem_copy_repeatedly` relies
327 /// It is the caller's responsibility to check bounds and alignment beforehand.
328 fn get_bytes_internal(
330 cx: &impl HasDataLayout,
332 check_init_and_ptr: bool,
333 ) -> AllocResult<&[u8]> {
334 if check_init_and_ptr {
335 self.check_init(range)?;
336 self.check_relocations(cx, range)?;
338 // We still don't want relocations on the *edges*.
339 self.check_relocation_edges(cx, range)?;
342 Ok(self.get_bytes_even_more_internal(range))
345 /// Checks that these bytes are initialized and not pointer bytes, and then return them
348 /// It is the caller's responsibility to check bounds and alignment beforehand.
349 /// Most likely, you want to use the `PlaceTy` and `OperandTy`-based methods
350 /// on `InterpCx` instead.
352 pub fn get_bytes(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult<&[u8]> {
353 self.get_bytes_internal(cx, range, true)
356 /// It is the caller's responsibility to handle uninitialized and pointer bytes.
357 /// However, this still checks that there are no relocations on the *edges*.
359 /// It is the caller's responsibility to check bounds and alignment beforehand.
361 pub fn get_bytes_with_uninit_and_ptr(
363 cx: &impl HasDataLayout,
365 ) -> AllocResult<&[u8]> {
366 self.get_bytes_internal(cx, range, false)
369 /// Just calling this already marks everything as defined and removes relocations,
370 /// so be sure to actually put data there!
372 /// It is the caller's responsibility to check bounds and alignment beforehand.
373 /// Most likely, you want to use the `PlaceTy` and `OperandTy`-based methods
374 /// on `InterpCx` instead.
375 pub fn get_bytes_mut(
377 cx: &impl HasDataLayout,
379 ) -> AllocResult<&mut [u8]> {
380 self.mark_init(range, true);
381 self.clear_relocations(cx, range)?;
383 Ok(&mut self.bytes[range.start.bytes_usize()..range.end().bytes_usize()])
386 /// A raw pointer variant of `get_bytes_mut` that avoids invalidating existing aliases into this memory.
387 pub fn get_bytes_mut_ptr(
389 cx: &impl HasDataLayout,
391 ) -> AllocResult<*mut [u8]> {
392 self.mark_init(range, true);
393 self.clear_relocations(cx, range)?;
395 assert!(range.end().bytes_usize() <= self.bytes.len()); // need to do our own bounds-check
396 let begin_ptr = self.bytes.as_mut_ptr().wrapping_add(range.start.bytes_usize());
397 let len = range.end().bytes_usize() - range.start.bytes_usize();
398 Ok(ptr::slice_from_raw_parts_mut(begin_ptr, len))
402 /// Reading and writing.
403 impl<Tag: Provenance, Extra> Allocation<Tag, Extra> {
404 /// Validates that `ptr.offset` and `ptr.offset + size` do not point to the middle of a
405 /// relocation. If `allow_uninit`/`allow_ptr` is `false`, also enforces that the memory in the
406 /// given range contains no uninitialized bytes/relocations.
409 cx: &impl HasDataLayout,
414 // Check bounds and relocations on the edges.
415 self.get_bytes_with_uninit_and_ptr(cx, range)?;
416 // Check uninit and ptr.
418 self.check_init(range)?;
421 self.check_relocations(cx, range)?;
426 /// Reads a *non-ZST* scalar.
428 /// If `read_provenance` is `true`, this will also read provenance; otherwise (if the machine
429 /// supports that) provenance is entirely ignored.
431 /// ZSTs can't be read because in order to obtain a `Pointer`, we need to check
432 /// for ZSTness anyway due to integer pointers being valid for ZSTs.
434 /// It is the caller's responsibility to check bounds and alignment beforehand.
435 /// Most likely, you want to call `InterpCx::read_scalar` instead of this method.
438 cx: &impl HasDataLayout,
440 read_provenance: bool,
441 ) -> AllocResult<ScalarMaybeUninit<Tag>> {
443 assert_eq!(range.size, cx.data_layout().pointer_size);
446 // First and foremost, if anything is uninit, bail.
447 if self.is_init(range).is_err() {
448 // This inflates uninitialized bytes to the entire scalar, even if only a few
449 // bytes are uninitialized.
450 return Ok(ScalarMaybeUninit::Uninit);
453 // If we are doing a pointer read, and there is a relocation exactly where we
454 // are reading, then we can put data and relocation back together and return that.
455 if read_provenance && let Some(&prov) = self.relocations.get(&range.start) {
456 // We already checked init and relocations, so we can use this function.
457 let bytes = self.get_bytes_even_more_internal(range);
458 let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
459 let ptr = Pointer::new(prov, Size::from_bytes(bits));
460 return Ok(ScalarMaybeUninit::from_pointer(ptr, cx));
463 // If we are *not* reading a pointer, and we can just ignore relocations,
464 // then do exactly that.
465 if !read_provenance && Tag::OFFSET_IS_ADDR {
466 // We just strip provenance.
467 let bytes = self.get_bytes_even_more_internal(range);
468 let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
469 return Ok(ScalarMaybeUninit::Scalar(Scalar::from_uint(bits, range.size)));
472 // It's complicated. Better make sure there is no provenance anywhere.
473 // FIXME: If !OFFSET_IS_ADDR, this is the best we can do. But if OFFSET_IS_ADDR, then
474 // `read_pointer` is true and we ideally would distinguish the following two cases:
475 // - The entire `range` is covered by 2 relocations for the same provenance.
476 // Then we should return a pointer with that provenance.
477 // - The range has inhomogeneous provenance. Then we should return just the
479 let bytes = self.get_bytes(cx, range)?;
480 let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
481 Ok(ScalarMaybeUninit::Scalar(Scalar::from_uint(bits, range.size)))
484 /// Writes a *non-ZST* scalar.
486 /// ZSTs can't be read because in order to obtain a `Pointer`, we need to check
487 /// for ZSTness anyway due to integer pointers being valid for ZSTs.
489 /// It is the caller's responsibility to check bounds and alignment beforehand.
490 /// Most likely, you want to call `InterpCx::write_scalar` instead of this method.
491 #[instrument(skip(self, cx), level = "debug")]
494 cx: &impl HasDataLayout,
496 val: ScalarMaybeUninit<Tag>,
498 assert!(self.mutability == Mutability::Mut);
500 let val = match val {
501 ScalarMaybeUninit::Scalar(scalar) => scalar,
502 ScalarMaybeUninit::Uninit => {
503 return self.write_uninit(cx, range);
507 // `to_bits_or_ptr_internal` is the right method because we just want to store this data
508 // as-is into memory.
509 let (bytes, provenance) = match val.to_bits_or_ptr_internal(range.size)? {
511 let (provenance, offset) = val.into_parts();
512 (u128::from(offset.bytes()), Some(provenance))
514 Ok(data) => (data, None),
517 let endian = cx.data_layout().endian;
518 let dst = self.get_bytes_mut(cx, range)?;
519 write_target_uint(endian, dst, bytes).unwrap();
521 // See if we have to also write a relocation.
522 if let Some(provenance) = provenance {
523 self.relocations.0.insert(range.start, provenance);
529 /// Write "uninit" to the given memory range.
530 pub fn write_uninit(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
531 self.mark_init(range, false);
532 self.clear_relocations(cx, range)?;
538 impl<Tag: Copy, Extra> Allocation<Tag, Extra> {
539 /// Returns all relocations overlapping with the given pointer-offset pair.
540 pub fn get_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> &[(Size, Tag)] {
541 // We have to go back `pointer_size - 1` bytes, as that one would still overlap with
542 // the beginning of this range.
543 let start = range.start.bytes().saturating_sub(cx.data_layout().pointer_size.bytes() - 1);
544 self.relocations.range(Size::from_bytes(start)..range.end())
547 /// Checks that there are no relocations overlapping with the given range.
549 fn check_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
550 if self.get_relocations(cx, range).is_empty() {
553 Err(AllocError::ReadPointerAsBytes)
557 /// Removes all relocations inside the given range.
558 /// If there are relocations overlapping with the edges, they
559 /// are removed as well *and* the bytes they cover are marked as
560 /// uninitialized. This is a somewhat odd "spooky action at a distance",
561 /// but it allows strictly more code to run than if we would just error
562 /// immediately in that case.
563 fn clear_relocations(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult
567 // Find the start and end of the given range and its outermost relocations.
568 let (first, last) = {
569 // Find all relocations overlapping the given range.
570 let relocations = self.get_relocations(cx, range);
571 if relocations.is_empty() {
576 relocations.first().unwrap().0,
577 relocations.last().unwrap().0 + cx.data_layout().pointer_size,
580 let start = range.start;
581 let end = range.end();
583 // We need to handle clearing the relocations from parts of a pointer.
584 // FIXME: Miri should preserve partial relocations; see
585 // https://github.com/rust-lang/miri/issues/2181.
587 if Tag::ERR_ON_PARTIAL_PTR_OVERWRITE {
588 return Err(AllocError::PartialPointerOverwrite(first));
591 "Partial pointer overwrite! De-initializing memory at offsets {first:?}..{start:?}."
593 self.init_mask.set_range(first, start, false);
596 if Tag::ERR_ON_PARTIAL_PTR_OVERWRITE {
597 return Err(AllocError::PartialPointerOverwrite(
598 last - cx.data_layout().pointer_size,
602 "Partial pointer overwrite! De-initializing memory at offsets {end:?}..{last:?}."
604 self.init_mask.set_range(end, last, false);
607 // Forget all the relocations.
608 // Since relocations do not overlap, we know that removing until `last` (exclusive) is fine,
609 // i.e., this will not remove any other relocations just after the ones we care about.
610 self.relocations.0.remove_range(first..last);
615 /// Errors if there are relocations overlapping with the edges of the
616 /// given memory range.
618 fn check_relocation_edges(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
619 self.check_relocations(cx, alloc_range(range.start, Size::ZERO))?;
620 self.check_relocations(cx, alloc_range(range.end(), Size::ZERO))?;
625 /// "Relocations" stores the provenance information of pointers stored in memory.
626 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
627 pub struct Relocations<Tag = AllocId>(SortedMap<Size, Tag>);
629 impl<Tag> Relocations<Tag> {
630 pub fn new() -> Self {
631 Relocations(SortedMap::new())
634 // The caller must guarantee that the given relocations are already sorted
635 // by address and contain no duplicates.
636 pub fn from_presorted(r: Vec<(Size, Tag)>) -> Self {
637 Relocations(SortedMap::from_presorted_elements(r))
641 impl<Tag> Deref for Relocations<Tag> {
642 type Target = SortedMap<Size, Tag>;
644 fn deref(&self) -> &Self::Target {
649 /// A partial, owned list of relocations to transfer into another allocation.
651 /// Offsets are already adjusted to the destination allocation.
652 pub struct AllocationRelocations<Tag> {
653 dest_relocations: Vec<(Size, Tag)>,
656 impl<Tag: Copy, Extra> Allocation<Tag, Extra> {
657 pub fn prepare_relocation_copy(
659 cx: &impl HasDataLayout,
663 ) -> AllocationRelocations<Tag> {
664 let relocations = self.get_relocations(cx, src);
665 if relocations.is_empty() {
666 return AllocationRelocations { dest_relocations: Vec::new() };
670 let mut new_relocations = Vec::with_capacity(relocations.len() * (count as usize));
672 // If `count` is large, this is rather wasteful -- we are allocating a big array here, which
673 // is mostly filled with redundant information since it's just N copies of the same `Tag`s
674 // at slightly adjusted offsets. The reason we do this is so that in `mark_relocation_range`
675 // we can use `insert_presorted`. That wouldn't work with an `Iterator` that just produces
676 // the right sequence of relocations for all N copies.
678 new_relocations.extend(relocations.iter().map(|&(offset, reloc)| {
679 // compute offset for current repetition
680 let dest_offset = dest + size * i; // `Size` operations
682 // shift offsets from source allocation to destination allocation
683 (offset + dest_offset) - src.start, // `Size` operations
689 AllocationRelocations { dest_relocations: new_relocations }
692 /// Applies a relocation copy.
693 /// The affected range, as defined in the parameters to `prepare_relocation_copy` is expected
694 /// to be clear of relocations.
696 /// This is dangerous to use as it can violate internal `Allocation` invariants!
697 /// It only exists to support an efficient implementation of `mem_copy_repeatedly`.
698 pub fn mark_relocation_range(&mut self, relocations: AllocationRelocations<Tag>) {
699 self.relocations.0.insert_presorted(relocations.dest_relocations);
703 ////////////////////////////////////////////////////////////////////////////////
704 // Uninitialized byte tracking
705 ////////////////////////////////////////////////////////////////////////////////
709 /// A bitmask where each bit refers to the byte with the same index. If the bit is `true`, the byte
710 /// is initialized. If it is `false` the byte is uninitialized.
711 // Note: for performance reasons when interning, some of the `InitMask` fields can be partially
712 // hashed. (see the `Hash` impl below for more details), so the impl is not derived.
713 #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
714 #[derive(HashStable)]
715 pub struct InitMask {
720 // Const allocations are only hashed for interning. However, they can be large, making the hashing
721 // expensive especially since it uses `FxHash`: it's better suited to short keys, not potentially
722 // big buffers like the allocation's init mask. We can partially hash some fields when they're
724 impl hash::Hash for InitMask {
725 fn hash<H: hash::Hasher>(&self, state: &mut H) {
726 const MAX_BLOCKS_TO_HASH: usize = MAX_BYTES_TO_HASH / std::mem::size_of::<Block>();
727 const MAX_BLOCKS_LEN: usize = MAX_HASHED_BUFFER_LEN / std::mem::size_of::<Block>();
729 // Partially hash the `blocks` buffer when it is large. To limit collisions with common
730 // prefixes and suffixes, we hash the length and some slices of the buffer.
731 let block_count = self.blocks.len();
732 if block_count > MAX_BLOCKS_LEN {
733 // Hash the buffer's length.
734 block_count.hash(state);
736 // And its head and tail.
737 self.blocks[..MAX_BLOCKS_TO_HASH].hash(state);
738 self.blocks[block_count - MAX_BLOCKS_TO_HASH..].hash(state);
740 self.blocks.hash(state);
743 // Hash the other fields as usual.
744 self.len.hash(state);
749 pub const BLOCK_SIZE: u64 = 64;
752 fn bit_index(bits: Size) -> (usize, usize) {
753 // BLOCK_SIZE is the number of bits that can fit in a `Block`.
754 // Each bit in a `Block` represents the initialization state of one byte of an allocation,
755 // so we use `.bytes()` here.
756 let bits = bits.bytes();
757 let a = bits / InitMask::BLOCK_SIZE;
758 let b = bits % InitMask::BLOCK_SIZE;
759 (usize::try_from(a).unwrap(), usize::try_from(b).unwrap())
763 fn size_from_bit_index(block: impl TryInto<u64>, bit: impl TryInto<u64>) -> Size {
764 let block = block.try_into().ok().unwrap();
765 let bit = bit.try_into().ok().unwrap();
766 Size::from_bytes(block * InitMask::BLOCK_SIZE + bit)
769 pub fn new(size: Size, state: bool) -> Self {
770 let mut m = InitMask { blocks: vec![], len: Size::ZERO };
775 pub fn set_range(&mut self, start: Size, end: Size, new_state: bool) {
778 self.grow(end - len, new_state);
780 self.set_range_inbounds(start, end, new_state);
783 pub fn set_range_inbounds(&mut self, start: Size, end: Size, new_state: bool) {
784 let (blocka, bita) = Self::bit_index(start);
785 let (blockb, bitb) = Self::bit_index(end);
786 if blocka == blockb {
787 // First set all bits except the first `bita`,
788 // then unset the last `64 - bitb` bits.
789 let range = if bitb == 0 {
792 (u64::MAX << bita) & (u64::MAX >> (64 - bitb))
795 self.blocks[blocka] |= range;
797 self.blocks[blocka] &= !range;
801 // across block boundaries
803 // Set `bita..64` to `1`.
804 self.blocks[blocka] |= u64::MAX << bita;
805 // Set `0..bitb` to `1`.
807 self.blocks[blockb] |= u64::MAX >> (64 - bitb);
809 // Fill in all the other blocks (much faster than one bit at a time).
810 for block in (blocka + 1)..blockb {
811 self.blocks[block] = u64::MAX;
814 // Set `bita..64` to `0`.
815 self.blocks[blocka] &= !(u64::MAX << bita);
816 // Set `0..bitb` to `0`.
818 self.blocks[blockb] &= !(u64::MAX >> (64 - bitb));
820 // Fill in all the other blocks (much faster than one bit at a time).
821 for block in (blocka + 1)..blockb {
822 self.blocks[block] = 0;
828 pub fn get(&self, i: Size) -> bool {
829 let (block, bit) = Self::bit_index(i);
830 (self.blocks[block] & (1 << bit)) != 0
834 pub fn set(&mut self, i: Size, new_state: bool) {
835 let (block, bit) = Self::bit_index(i);
836 self.set_bit(block, bit, new_state);
840 fn set_bit(&mut self, block: usize, bit: usize, new_state: bool) {
842 self.blocks[block] |= 1 << bit;
844 self.blocks[block] &= !(1 << bit);
848 pub fn grow(&mut self, amount: Size, new_state: bool) {
849 if amount.bytes() == 0 {
852 let unused_trailing_bits =
853 u64::try_from(self.blocks.len()).unwrap() * Self::BLOCK_SIZE - self.len.bytes();
854 if amount.bytes() > unused_trailing_bits {
855 let additional_blocks = amount.bytes() / Self::BLOCK_SIZE + 1;
857 // FIXME(oli-obk): optimize this by repeating `new_state as Block`.
858 iter::repeat(0).take(usize::try_from(additional_blocks).unwrap()),
861 let start = self.len;
863 self.set_range_inbounds(start, start + amount, new_state); // `Size` operation
866 /// Returns the index of the first bit in `start..end` (end-exclusive) that is equal to is_init.
867 fn find_bit(&self, start: Size, end: Size, is_init: bool) -> Option<Size> {
868 /// A fast implementation of `find_bit`,
869 /// which skips over an entire block at a time if it's all 0s (resp. 1s),
870 /// and finds the first 1 (resp. 0) bit inside a block using `trailing_zeros` instead of a loop.
872 /// Note that all examples below are written with 8 (instead of 64) bit blocks for simplicity,
873 /// and with the least significant bit (and lowest block) first:
875 /// 00000000|00000000
879 /// Also, if not stated, assume that `is_init = true`, that is, we are searching for the first 1 bit.
881 init_mask: &InitMask,
886 /// Search one block, returning the index of the first bit equal to `is_init`.
893 // For the following examples, assume this function was called with:
897 // Note that, for the examples in this function, the most significant bit is written first,
898 // which is backwards compared to the comments in `find_bit`/`find_bit_fast`.
900 // Invert bits so we're always looking for the first set bit.
903 let bits = if is_init { bits } else { !bits };
904 // Mask off unused start bits.
908 let bits = bits & (!0 << start_bit);
909 // Find set bit, if any.
910 // bit = trailing_zeros(0b11000000)
915 let bit = bits.trailing_zeros();
916 Some(InitMask::size_from_bit_index(block, bit))
924 // Convert `start` and `end` to block indexes and bit indexes within each block.
925 // We must convert `end` to an inclusive bound to handle block boundaries correctly.
929 // (a) 00000000|00000000 (b) 00000000|
930 // ^~~~~~~~~~~^ ^~~~~~~~~^
931 // start end start end
933 // In both cases, the block index of `end` is 1.
934 // But we do want to search block 1 in (a), and we don't in (b).
936 // We subtract 1 from both end positions to make them inclusive:
938 // (a) 00000000|00000000 (b) 00000000|
939 // ^~~~~~~~~~^ ^~~~~~~^
940 // start end_inclusive start end_inclusive
942 // For (a), the block index of `end_inclusive` is 1, and for (b), it's 0.
943 // This provides the desired behavior of searching blocks 0 and 1 for (a),
944 // and searching only block 0 for (b).
945 // There is no concern of overflows since we checked for `start >= end` above.
946 let (start_block, start_bit) = InitMask::bit_index(start);
947 let end_inclusive = Size::from_bytes(end.bytes() - 1);
948 let (end_block_inclusive, _) = InitMask::bit_index(end_inclusive);
950 // Handle first block: need to skip `start_bit` bits.
952 // We need to handle the first block separately,
953 // because there may be bits earlier in the block that should be ignored,
954 // such as the bit marked (1) in this example:
958 // (c) 01000000|00000000|00000001
959 // ^~~~~~~~~~~~~~~~~~^
962 search_block(init_mask.blocks[start_block], start_block, start_bit, is_init)
964 // If the range is less than a block, we may find a matching bit after `end`.
966 // For example, we shouldn't successfully find bit (2), because it's after `end`:
970 // (d) 00000001|00000000|00000001
974 // An alternative would be to mask off end bits in the same way as we do for start bits,
975 // but performing this check afterwards is faster and simpler to implement.
983 // Handle remaining blocks.
985 // We can skip over an entire block at once if it's all 0s (resp. 1s).
986 // The block marked (3) in this example is the first block that will be handled by this loop,
987 // and it will be skipped for that reason:
991 // (e) 01000000|00000000|00000001
992 // ^~~~~~~~~~~~~~~~~~^
994 if start_block < end_block_inclusive {
995 // This loop is written in a specific way for performance.
996 // Notably: `..end_block_inclusive + 1` is used for an inclusive range instead of `..=end_block_inclusive`,
997 // and `.zip(start_block + 1..)` is used to track the index instead of `.enumerate().skip().take()`,
998 // because both alternatives result in significantly worse codegen.
999 // `end_block_inclusive + 1` is guaranteed not to wrap, because `end_block_inclusive <= end / BLOCK_SIZE`,
1000 // and `BLOCK_SIZE` (the number of bits per block) will always be at least 8 (1 byte).
1001 for (&bits, block) in init_mask.blocks[start_block + 1..end_block_inclusive + 1]
1003 .zip(start_block + 1..)
1005 if let Some(i) = search_block(bits, block, 0, is_init) {
1006 // If this is the last block, we may find a matching bit after `end`.
1008 // For example, we shouldn't successfully find bit (4), because it's after `end`:
1012 // (f) 00000001|00000000|00000001
1013 // ^~~~~~~~~~~~~~~~~~^
1016 // As above with example (d), we could handle the end block separately and mask off end bits,
1017 // but unconditionally searching an entire block at once and performing this check afterwards
1018 // is faster and much simpler to implement.
1031 #[cfg_attr(not(debug_assertions), allow(dead_code))]
1033 init_mask: &InitMask,
1038 (start..end).find(|&i| init_mask.get(i) == is_init)
1041 let result = find_bit_fast(self, start, end, is_init);
1045 find_bit_slow(self, start, end, is_init),
1046 "optimized implementation of find_bit is wrong for start={:?} end={:?} is_init={} init_mask={:#?}",
1057 /// A contiguous chunk of initialized or uninitialized memory.
1058 pub enum InitChunk {
1060 Uninit(Range<Size>),
1065 pub fn is_init(&self) -> bool {
1067 Self::Init(_) => true,
1068 Self::Uninit(_) => false,
1073 pub fn range(&self) -> Range<Size> {
1075 Self::Init(r) => r.clone(),
1076 Self::Uninit(r) => r.clone(),
1082 /// Checks whether the range `start..end` (end-exclusive) is entirely initialized.
1084 /// Returns `Ok(())` if it's initialized. Otherwise returns a range of byte
1085 /// indexes for the first contiguous span of the uninitialized access.
1087 pub fn is_range_initialized(&self, start: Size, end: Size) -> Result<(), Range<Size>> {
1089 return Err(self.len..end);
1092 let uninit_start = self.find_bit(start, end, false);
1094 match uninit_start {
1095 Some(uninit_start) => {
1096 let uninit_end = self.find_bit(uninit_start, end, true).unwrap_or(end);
1097 Err(uninit_start..uninit_end)
1103 /// Returns an iterator, yielding a range of byte indexes for each contiguous region
1104 /// of initialized or uninitialized bytes inside the range `start..end` (end-exclusive).
1106 /// The iterator guarantees the following:
1107 /// - Chunks are nonempty.
1108 /// - Chunks are adjacent (each range's start is equal to the previous range's end).
1109 /// - Chunks span exactly `start..end` (the first starts at `start`, the last ends at `end`).
1110 /// - Chunks alternate between [`InitChunk::Init`] and [`InitChunk::Uninit`].
1112 pub fn range_as_init_chunks(&self, start: Size, end: Size) -> InitChunkIter<'_> {
1113 assert!(end <= self.len);
1115 let is_init = if start < end {
1118 // `start..end` is empty: there are no chunks, so use some arbitrary value
1122 InitChunkIter { init_mask: self, is_init, start, end }
1126 /// Yields [`InitChunk`]s. See [`InitMask::range_as_init_chunks`].
1128 pub struct InitChunkIter<'a> {
1129 init_mask: &'a InitMask,
1130 /// Whether the next chunk we will return is initialized.
1131 /// If there are no more chunks, contains some arbitrary value.
1133 /// The current byte index into `init_mask`.
1135 /// The end byte index into `init_mask`.
1139 impl<'a> Iterator for InitChunkIter<'a> {
1140 type Item = InitChunk;
1143 fn next(&mut self) -> Option<Self::Item> {
1144 if self.start >= self.end {
1149 self.init_mask.find_bit(self.start, self.end, !self.is_init).unwrap_or(self.end);
1150 let range = self.start..end_of_chunk;
1153 Some(if self.is_init { InitChunk::Init(range) } else { InitChunk::Uninit(range) });
1155 self.is_init = !self.is_init;
1156 self.start = end_of_chunk;
1162 /// Uninitialized bytes.
1163 impl<Tag: Copy, Extra> Allocation<Tag, Extra> {
1164 /// Checks whether the given range is entirely initialized.
1166 /// Returns `Ok(())` if it's initialized. Otherwise returns the range of byte
1167 /// indexes of the first contiguous uninitialized access.
1168 fn is_init(&self, range: AllocRange) -> Result<(), Range<Size>> {
1169 self.init_mask.is_range_initialized(range.start, range.end()) // `Size` addition
1172 /// Checks that a range of bytes is initialized. If not, returns the `InvalidUninitBytes`
1173 /// error which will report the first range of bytes which is uninitialized.
1174 fn check_init(&self, range: AllocRange) -> AllocResult {
1175 self.is_init(range).map_err(|idx_range| {
1176 AllocError::InvalidUninitBytes(Some(UninitBytesAccess {
1177 access_offset: range.start,
1178 access_size: range.size,
1179 uninit_offset: idx_range.start,
1180 uninit_size: idx_range.end - idx_range.start, // `Size` subtraction
1185 fn mark_init(&mut self, range: AllocRange, is_init: bool) {
1186 if range.size.bytes() == 0 {
1189 assert!(self.mutability == Mutability::Mut);
1190 self.init_mask.set_range(range.start, range.end(), is_init);
1194 /// Run-length encoding of the uninit mask.
1195 /// Used to copy parts of a mask multiple times to another allocation.
1196 pub struct InitMaskCompressed {
1197 /// Whether the first range is initialized.
1199 /// The lengths of ranges that are run-length encoded.
1200 /// The initialization state of the ranges alternate starting with `initial`.
1201 ranges: smallvec::SmallVec<[u64; 1]>,
1204 impl InitMaskCompressed {
1205 pub fn no_bytes_init(&self) -> bool {
1206 // The `ranges` are run-length encoded and of alternating initialization state.
1207 // So if `ranges.len() > 1` then the second block is an initialized range.
1208 !self.initial && self.ranges.len() == 1
1212 /// Transferring the initialization mask to other allocations.
1213 impl<Tag, Extra> Allocation<Tag, Extra> {
1214 /// Creates a run-length encoding of the initialization mask; panics if range is empty.
1216 /// This is essentially a more space-efficient version of
1217 /// `InitMask::range_as_init_chunks(...).collect::<Vec<_>>()`.
1218 pub fn compress_uninit_range(&self, range: AllocRange) -> InitMaskCompressed {
1219 // Since we are copying `size` bytes from `src` to `dest + i * size` (`for i in 0..repeat`),
1220 // a naive initialization mask copying algorithm would repeatedly have to read the initialization mask from
1221 // the source and write it to the destination. Even if we optimized the memory accesses,
1222 // we'd be doing all of this `repeat` times.
1223 // Therefore we precompute a compressed version of the initialization mask of the source value and
1224 // then write it back `repeat` times without computing any more information from the source.
1226 // A precomputed cache for ranges of initialized / uninitialized bits
1227 // 0000010010001110 will become
1228 // `[5, 1, 2, 1, 3, 3, 1]`,
1229 // where each element toggles the state.
1231 let mut ranges = smallvec::SmallVec::<[u64; 1]>::new();
1233 let mut chunks = self.init_mask.range_as_init_chunks(range.start, range.end()).peekable();
1235 let initial = chunks.peek().expect("range should be nonempty").is_init();
1237 // Here we rely on `range_as_init_chunks` to yield alternating init/uninit chunks.
1238 for chunk in chunks {
1239 let len = chunk.range().end.bytes() - chunk.range().start.bytes();
1243 InitMaskCompressed { ranges, initial }
1246 /// Applies multiple instances of the run-length encoding to the initialization mask.
1248 /// This is dangerous to use as it can violate internal `Allocation` invariants!
1249 /// It only exists to support an efficient implementation of `mem_copy_repeatedly`.
1250 pub fn mark_compressed_init_range(
1252 defined: &InitMaskCompressed,
1256 // An optimization where we can just overwrite an entire range of initialization
1257 // bits if they are going to be uniformly `1` or `0`.
1258 if defined.ranges.len() <= 1 {
1259 self.init_mask.set_range_inbounds(
1261 range.start + range.size * repeat, // `Size` operations
1267 for mut j in 0..repeat {
1268 j *= range.size.bytes();
1269 j += range.start.bytes();
1270 let mut cur = defined.initial;
1271 for range in &defined.ranges {
1274 self.init_mask.set_range_inbounds(
1275 Size::from_bytes(old_j),
1276 Size::from_bytes(j),