1 //! Computations on places -- field projections, going from mir::Place, and writing
3 //! All high-level functions to write to memory work on places as destinations.
5 use std::convert::TryFrom;
8 use rustc_ast::Mutability;
9 use rustc_macros::HashStable;
10 use rustc_middle::mir;
11 use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt, TyAndLayout};
12 use rustc_middle::ty::{self, Ty};
13 use rustc_target::abi::{Abi, Align, FieldsShape, TagEncoding};
14 use rustc_target::abi::{HasDataLayout, Size, VariantIdx, Variants};
17 alloc_range, mir_assign_valid_types, AllocId, AllocRef, AllocRefMut, CheckInAllocMsg,
18 ConstAlloc, ImmTy, Immediate, InterpCx, InterpResult, LocalValue, Machine, MemoryKind, OpTy,
19 Operand, Pointer, PointerArithmetic, Provenance, Scalar, ScalarMaybeUninit,
22 #[derive(Copy, Clone, Hash, PartialEq, Eq, HashStable, Debug)]
23 /// Information required for the sound usage of a `MemPlace`.
24 pub enum MemPlaceMeta<Tag: Provenance = AllocId> {
25 /// The unsized payload (e.g. length for slices or vtable pointer for trait objects).
27 /// `Sized` types or unsized `extern type`
29 /// The address of this place may not be taken. This protects the `MemPlace` from coming from
30 /// a ZST Operand without a backing allocation and being converted to an integer address. This
31 /// should be impossible, because you can't take the address of an operand, but this is a second
32 /// protection layer ensuring that we don't mess up.
36 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
37 rustc_data_structures::static_assert_size!(MemPlaceMeta, 24);
39 impl<Tag: Provenance> MemPlaceMeta<Tag> {
40 pub fn unwrap_meta(self) -> Scalar<Tag> {
43 Self::None | Self::Poison => {
44 bug!("expected wide pointer extra data (e.g. slice length or trait object vtable)")
48 fn has_meta(self) -> bool {
50 Self::Meta(_) => true,
51 Self::None | Self::Poison => false,
56 #[derive(Copy, Clone, Hash, PartialEq, Eq, HashStable, Debug)]
57 pub struct MemPlace<Tag: Provenance = AllocId> {
58 /// The pointer can be a pure integer, with the `None` tag.
59 pub ptr: Pointer<Option<Tag>>,
61 /// Metadata for unsized places. Interpretation is up to the type.
62 /// Must not be present for sized types, but can be missing for unsized types
63 /// (e.g., `extern type`).
64 pub meta: MemPlaceMeta<Tag>,
67 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
68 rustc_data_structures::static_assert_size!(MemPlace, 48);
70 #[derive(Copy, Clone, Hash, PartialEq, Eq, HashStable, Debug)]
71 pub enum Place<Tag: Provenance = AllocId> {
72 /// A place referring to a value allocated in the `Memory` system.
75 /// To support alloc-free locals, we are able to write directly to a local.
76 /// (Without that optimization, we'd just always be a `MemPlace`.)
77 Local { frame: usize, local: mir::Local },
80 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
81 rustc_data_structures::static_assert_size!(Place, 56);
83 #[derive(Copy, Clone, Debug)]
84 pub struct PlaceTy<'tcx, Tag: Provenance = AllocId> {
85 place: Place<Tag>, // Keep this private; it helps enforce invariants.
86 pub layout: TyAndLayout<'tcx>,
89 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
90 rustc_data_structures::static_assert_size!(PlaceTy<'_>, 72);
92 impl<'tcx, Tag: Provenance> std::ops::Deref for PlaceTy<'tcx, Tag> {
93 type Target = Place<Tag>;
95 fn deref(&self) -> &Place<Tag> {
100 /// A MemPlace with its layout. Constructing it is only possible in this module.
101 #[derive(Copy, Clone, Hash, Eq, PartialEq, Debug)]
102 pub struct MPlaceTy<'tcx, Tag: Provenance = AllocId> {
103 mplace: MemPlace<Tag>,
104 pub layout: TyAndLayout<'tcx>,
107 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
108 rustc_data_structures::static_assert_size!(MPlaceTy<'_>, 64);
110 impl<'tcx, Tag: Provenance> std::ops::Deref for MPlaceTy<'tcx, Tag> {
111 type Target = MemPlace<Tag>;
113 fn deref(&self) -> &MemPlace<Tag> {
118 impl<'tcx, Tag: Provenance> From<MPlaceTy<'tcx, Tag>> for PlaceTy<'tcx, Tag> {
120 fn from(mplace: MPlaceTy<'tcx, Tag>) -> Self {
121 PlaceTy { place: Place::Ptr(mplace.mplace), layout: mplace.layout }
125 impl<Tag: Provenance> MemPlace<Tag> {
127 pub fn from_ptr(ptr: Pointer<Option<Tag>>, align: Align) -> Self {
128 MemPlace { ptr, align, meta: MemPlaceMeta::None }
131 /// Adjust the provenance of the main pointer (metadata is unaffected).
132 pub fn map_provenance(self, f: impl FnOnce(Option<Tag>) -> Option<Tag>) -> Self {
133 MemPlace { ptr: self.ptr.map_provenance(f), ..self }
136 /// Turn a mplace into a (thin or wide) pointer, as a reference, pointing to the same space.
137 /// This is the inverse of `ref_to_mplace`.
139 pub fn to_ref(self, cx: &impl HasDataLayout) -> Immediate<Tag> {
141 MemPlaceMeta::None => Immediate::from(Scalar::from_maybe_pointer(self.ptr, cx)),
142 MemPlaceMeta::Meta(meta) => {
143 Immediate::ScalarPair(Scalar::from_maybe_pointer(self.ptr, cx).into(), meta.into())
145 MemPlaceMeta::Poison => bug!(
146 "MPlaceTy::dangling may never be used to produce a \
147 place that will have the address of its pointee taken"
156 meta: MemPlaceMeta<Tag>,
157 cx: &impl HasDataLayout,
158 ) -> InterpResult<'tcx, Self> {
160 ptr: self.ptr.offset(offset, cx)?,
161 align: self.align.restrict_for_offset(offset),
167 impl<'tcx, Tag: Provenance> MPlaceTy<'tcx, Tag> {
168 /// Produces a MemPlace that works for ZST but nothing else
170 pub fn dangling(layout: TyAndLayout<'tcx>) -> Self {
171 let align = layout.align.abi;
172 let ptr = Pointer::new(None, Size::from_bytes(align.bytes())); // no provenance, absolute address
173 // `Poison` this to make sure that the pointer value `ptr` is never observable by the program.
174 MPlaceTy { mplace: MemPlace { ptr, align, meta: MemPlaceMeta::Poison }, layout }
181 meta: MemPlaceMeta<Tag>,
182 layout: TyAndLayout<'tcx>,
183 cx: &impl HasDataLayout,
184 ) -> InterpResult<'tcx, Self> {
185 Ok(MPlaceTy { mplace: self.mplace.offset(offset, meta, cx)?, layout })
189 pub fn from_aligned_ptr(ptr: Pointer<Option<Tag>>, layout: TyAndLayout<'tcx>) -> Self {
190 MPlaceTy { mplace: MemPlace::from_ptr(ptr, layout.align.abi), layout }
194 pub(super) fn len(&self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> {
195 if self.layout.is_unsized() {
196 // We need to consult `meta` metadata
197 match self.layout.ty.kind() {
198 ty::Slice(..) | ty::Str => self.mplace.meta.unwrap_meta().to_machine_usize(cx),
199 _ => bug!("len not supported on unsized type {:?}", self.layout.ty),
202 // Go through the layout. There are lots of types that support a length,
203 // e.g., SIMD types. (But not all repr(simd) types even have FieldsShape::Array!)
204 match self.layout.fields {
205 FieldsShape::Array { count, .. } => Ok(count),
206 _ => bug!("len not supported on sized type {:?}", self.layout.ty),
212 pub(super) fn vtable(&self) -> Scalar<Tag> {
213 match self.layout.ty.kind() {
214 ty::Dynamic(..) => self.mplace.meta.unwrap_meta(),
215 _ => bug!("vtable not supported on type {:?}", self.layout.ty),
220 // These are defined here because they produce a place.
221 impl<'tcx, Tag: Provenance> OpTy<'tcx, Tag> {
223 /// Note: do not call `as_ref` on the resulting place. This function should only be used to
224 /// read from the resulting mplace, not to get its address back.
225 pub fn try_as_mplace(&self) -> Result<MPlaceTy<'tcx, Tag>, ImmTy<'tcx, Tag>> {
227 Operand::Indirect(mplace) => Ok(MPlaceTy { mplace, layout: self.layout }),
228 Operand::Immediate(_) if self.layout.is_zst() => Ok(MPlaceTy::dangling(self.layout)),
229 Operand::Immediate(imm) => Err(ImmTy::from_immediate(imm, self.layout)),
234 /// Note: do not call `as_ref` on the resulting place. This function should only be used to
235 /// read from the resulting mplace, not to get its address back.
236 pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Tag> {
237 self.try_as_mplace().unwrap()
241 impl<Tag: Provenance> Place<Tag> {
243 pub fn assert_mem_place(self) -> MemPlace<Tag> {
245 Place::Ptr(mplace) => mplace,
246 _ => bug!("assert_mem_place: expected Place::Ptr, got {:?}", self),
251 impl<'tcx, Tag: Provenance> PlaceTy<'tcx, Tag> {
253 pub fn assert_mem_place(self) -> MPlaceTy<'tcx, Tag> {
254 MPlaceTy { mplace: self.place.assert_mem_place(), layout: self.layout }
258 // separating the pointer tag for `impl Trait`, see https://github.com/rust-lang/rust/issues/54385
259 impl<'mir, 'tcx: 'mir, Tag, M> InterpCx<'mir, 'tcx, M>
261 // FIXME: Working around https://github.com/rust-lang/rust/issues/54385
262 Tag: Provenance + Eq + Hash + 'static,
263 M: Machine<'mir, 'tcx, PointerTag = Tag>,
265 /// Take a value, which represents a (thin or wide) reference, and make it a place.
266 /// Alignment is just based on the type. This is the inverse of `MemPlace::to_ref()`.
268 /// Only call this if you are sure the place is "valid" (aligned and inbounds), or do not
269 /// want to ever use the place for memory access!
270 /// Generally prefer `deref_operand`.
271 pub fn ref_to_mplace(
273 val: &ImmTy<'tcx, M::PointerTag>,
274 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
276 val.layout.ty.builtin_deref(true).expect("`ref_to_mplace` called on non-ptr type").ty;
277 let layout = self.layout_of(pointee_type)?;
278 let (ptr, meta) = match **val {
279 Immediate::Scalar(ptr) => (ptr, MemPlaceMeta::None),
280 Immediate::ScalarPair(ptr, meta) => (ptr, MemPlaceMeta::Meta(meta.check_init()?)),
283 let mplace = MemPlace {
284 ptr: self.scalar_to_ptr(ptr.check_init()?),
285 // We could use the run-time alignment here. For now, we do not, because
286 // the point of tracking the alignment here is to make sure that the *static*
287 // alignment information emitted with the loads is correct. The run-time
288 // alignment can only be more restrictive.
289 align: layout.align.abi,
292 Ok(MPlaceTy { mplace, layout })
295 /// Take an operand, representing a pointer, and dereference it to a place -- that
296 /// will always be a MemPlace. Lives in `place.rs` because it creates a place.
297 pub fn deref_operand(
299 src: &OpTy<'tcx, M::PointerTag>,
300 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
301 let val = self.read_immediate(src)?;
302 trace!("deref to {} on {:?}", val.layout.ty, *val);
303 let mplace = self.ref_to_mplace(&val)?;
304 self.check_mplace_access(mplace, CheckInAllocMsg::DerefTest)?;
309 pub(super) fn get_alloc(
311 place: &MPlaceTy<'tcx, M::PointerTag>,
312 ) -> InterpResult<'tcx, Option<AllocRef<'_, 'tcx, M::PointerTag, M::AllocExtra>>> {
313 assert!(!place.layout.is_unsized());
314 assert!(!place.meta.has_meta());
315 let size = place.layout.size;
316 self.memory.get(place.ptr, size, place.align)
320 pub(super) fn get_alloc_mut(
322 place: &MPlaceTy<'tcx, M::PointerTag>,
323 ) -> InterpResult<'tcx, Option<AllocRefMut<'_, 'tcx, M::PointerTag, M::AllocExtra>>> {
324 assert!(!place.layout.is_unsized());
325 assert!(!place.meta.has_meta());
326 let size = place.layout.size;
327 self.memory.get_mut(place.ptr, size, place.align)
330 /// Check if this mplace is dereferenceable and sufficiently aligned.
331 fn check_mplace_access(
333 mplace: MPlaceTy<'tcx, M::PointerTag>,
334 msg: CheckInAllocMsg,
335 ) -> InterpResult<'tcx> {
336 let (size, align) = self
337 .size_and_align_of_mplace(&mplace)?
338 .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
339 assert!(mplace.mplace.align <= align, "dynamic alignment less strict than static one?");
340 let align = M::enforce_alignment(&self.memory.extra).then_some(align);
341 self.memory.check_ptr_access_align(mplace.ptr, size, align.unwrap_or(Align::ONE), msg)?;
345 /// Offset a pointer to project to a field of a struct/union. Unlike `place_field`, this is
346 /// always possible without allocating, so it can take `&self`. Also return the field's layout.
347 /// This supports both struct and array fields.
349 /// This also works for arrays, but then the `usize` index type is restricting.
350 /// For indexing into arrays, use `mplace_index`.
354 base: &MPlaceTy<'tcx, M::PointerTag>,
356 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
357 let offset = base.layout.fields.offset(field);
358 let field_layout = base.layout.field(self, field);
360 // Offset may need adjustment for unsized fields.
361 let (meta, offset) = if field_layout.is_unsized() {
362 // Re-use parent metadata to determine dynamic field layout.
363 // With custom DSTS, this *will* execute user-defined code, but the same
364 // happens at run-time so that's okay.
365 match self.size_and_align_of(&base.meta, &field_layout)? {
366 Some((_, align)) => (base.meta, offset.align_to(align)),
368 // For unsized types with an extern type tail we perform no adjustments.
369 // NOTE: keep this in sync with `PlaceRef::project_field` in the codegen backend.
370 assert!(matches!(base.meta, MemPlaceMeta::None));
375 // base.meta could be present; we might be accessing a sized field of an unsized
377 (MemPlaceMeta::None, offset)
380 // We do not look at `base.layout.align` nor `field_layout.align`, unlike
381 // codegen -- mostly to see if we can get away with that
382 base.offset(offset, meta, field_layout, self)
385 /// Index into an array.
389 base: &MPlaceTy<'tcx, M::PointerTag>,
391 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
392 // Not using the layout method because we want to compute on u64
393 match base.layout.fields {
394 FieldsShape::Array { stride, .. } => {
395 let len = base.len(self)?;
397 // This can only be reached in ConstProp and non-rustc-MIR.
398 throw_ub!(BoundsCheckFailed { len, index });
400 let offset = stride * index; // `Size` multiplication
401 // All fields have the same layout.
402 let field_layout = base.layout.field(self, 0);
404 assert!(!field_layout.is_unsized());
405 base.offset(offset, MemPlaceMeta::None, field_layout, self)
409 "`mplace_index` called on non-array type {:?}",
415 // Iterates over all fields of an array. Much more efficient than doing the
416 // same by repeatedly calling `mplace_array`.
417 pub(super) fn mplace_array_fields<'a>(
419 base: &'a MPlaceTy<'tcx, Tag>,
420 ) -> InterpResult<'tcx, impl Iterator<Item = InterpResult<'tcx, MPlaceTy<'tcx, Tag>>> + 'a>
422 let len = base.len(self)?; // also asserts that we have a type where this makes sense
423 let FieldsShape::Array { stride, .. } = base.layout.fields else {
424 span_bug!(self.cur_span(), "mplace_array_fields: expected an array layout");
426 let layout = base.layout.field(self, 0);
427 let dl = &self.tcx.data_layout;
428 // `Size` multiplication
429 Ok((0..len).map(move |i| base.offset(stride * i, MemPlaceMeta::None, layout, dl)))
434 base: &MPlaceTy<'tcx, M::PointerTag>,
438 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
439 let len = base.len(self)?; // also asserts that we have a type where this makes sense
440 let actual_to = if from_end {
441 if from.checked_add(to).map_or(true, |to| to > len) {
442 // This can only be reached in ConstProp and non-rustc-MIR.
443 throw_ub!(BoundsCheckFailed { len: len, index: from.saturating_add(to) });
445 len.checked_sub(to).unwrap()
450 // Not using layout method because that works with usize, and does not work with slices
451 // (that have count 0 in their layout).
452 let from_offset = match base.layout.fields {
453 FieldsShape::Array { stride, .. } => stride * from, // `Size` multiplication is checked
455 span_bug!(self.cur_span(), "unexpected layout of index access: {:#?}", base.layout)
459 // Compute meta and new layout
460 let inner_len = actual_to.checked_sub(from).unwrap();
461 let (meta, ty) = match base.layout.ty.kind() {
462 // It is not nice to match on the type, but that seems to be the only way to
464 ty::Array(inner, _) => (MemPlaceMeta::None, self.tcx.mk_array(*inner, inner_len)),
466 let len = Scalar::from_machine_usize(inner_len, self);
467 (MemPlaceMeta::Meta(len), base.layout.ty)
470 span_bug!(self.cur_span(), "cannot subslice non-array type: `{:?}`", base.layout.ty)
473 let layout = self.layout_of(ty)?;
474 base.offset(from_offset, meta, layout, self)
477 pub(crate) fn mplace_downcast(
479 base: &MPlaceTy<'tcx, M::PointerTag>,
481 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
482 // Downcasts only change the layout
483 assert!(!base.meta.has_meta());
484 Ok(MPlaceTy { layout: base.layout.for_variant(self, variant), ..*base })
487 /// Project into an mplace
488 pub(super) fn mplace_projection(
490 base: &MPlaceTy<'tcx, M::PointerTag>,
491 proj_elem: mir::PlaceElem<'tcx>,
492 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
493 use rustc_middle::mir::ProjectionElem::*;
495 Field(field, _) => self.mplace_field(base, field.index())?,
496 Downcast(_, variant) => self.mplace_downcast(base, variant)?,
497 Deref => self.deref_operand(&base.into())?,
500 let layout = self.layout_of(self.tcx.types.usize)?;
501 let n = self.access_local(self.frame(), local, Some(layout))?;
502 let n = self.read_scalar(&n)?;
503 let n = n.to_machine_usize(self)?;
504 self.mplace_index(base, n)?
507 ConstantIndex { offset, min_length, from_end } => {
508 let n = base.len(self)?;
510 // This can only be reached in ConstProp and non-rustc-MIR.
511 throw_ub!(BoundsCheckFailed { len: min_length, index: n });
514 let index = if from_end {
515 assert!(0 < offset && offset <= min_length);
516 n.checked_sub(offset).unwrap()
518 assert!(offset < min_length);
522 self.mplace_index(base, index)?
525 Subslice { from, to, from_end } => self.mplace_subslice(base, from, to, from_end)?,
529 /// Converts a repr(simd) place into a place where `place_index` accesses the SIMD elements.
530 /// Also returns the number of elements.
531 pub fn mplace_to_simd(
533 base: &MPlaceTy<'tcx, M::PointerTag>,
534 ) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::PointerTag>, u64)> {
535 // Basically we just transmute this place into an array following simd_size_and_type.
536 // (Transmuting is okay since this is an in-memory place. We also double-check the size
538 let (len, e_ty) = base.layout.ty.simd_size_and_type(*self.tcx);
539 let array = self.tcx.mk_array(e_ty, len);
540 let layout = self.layout_of(array)?;
541 assert_eq!(layout.size, base.layout.size);
542 Ok((MPlaceTy { layout, ..*base }, len))
545 /// Gets the place of a field inside the place, and also the field's type.
546 /// Just a convenience function, but used quite a bit.
547 /// This is the only projection that might have a side-effect: We cannot project
548 /// into the field of a local `ScalarPair`, we have to first allocate it.
551 base: &PlaceTy<'tcx, M::PointerTag>,
553 ) -> InterpResult<'tcx, PlaceTy<'tcx, M::PointerTag>> {
554 // FIXME: We could try to be smarter and avoid allocation for fields that span the
556 let mplace = self.force_allocation(base)?;
557 Ok(self.mplace_field(&mplace, field)?.into())
562 base: &PlaceTy<'tcx, M::PointerTag>,
564 ) -> InterpResult<'tcx, PlaceTy<'tcx, M::PointerTag>> {
565 let mplace = self.force_allocation(base)?;
566 Ok(self.mplace_index(&mplace, index)?.into())
569 pub fn place_downcast(
571 base: &PlaceTy<'tcx, M::PointerTag>,
573 ) -> InterpResult<'tcx, PlaceTy<'tcx, M::PointerTag>> {
574 // Downcast just changes the layout
575 Ok(match base.place {
576 Place::Ptr(mplace) => {
577 self.mplace_downcast(&MPlaceTy { mplace, layout: base.layout }, variant)?.into()
579 Place::Local { .. } => {
580 let layout = base.layout.for_variant(self, variant);
581 PlaceTy { layout, ..*base }
586 /// Projects into a place.
587 pub fn place_projection(
589 base: &PlaceTy<'tcx, M::PointerTag>,
590 &proj_elem: &mir::ProjectionElem<mir::Local, Ty<'tcx>>,
591 ) -> InterpResult<'tcx, PlaceTy<'tcx, M::PointerTag>> {
592 use rustc_middle::mir::ProjectionElem::*;
594 Field(field, _) => self.place_field(base, field.index())?,
595 Downcast(_, variant) => self.place_downcast(base, variant)?,
596 Deref => self.deref_operand(&self.place_to_op(base)?)?.into(),
597 // For the other variants, we have to force an allocation.
598 // This matches `operand_projection`.
599 Subslice { .. } | ConstantIndex { .. } | Index(_) => {
600 let mplace = self.force_allocation(base)?;
601 self.mplace_projection(&mplace, proj_elem)?.into()
606 /// Converts a repr(simd) place into a place where `place_index` accesses the SIMD elements.
607 /// Also returns the number of elements.
608 pub fn place_to_simd(
610 base: &PlaceTy<'tcx, M::PointerTag>,
611 ) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::PointerTag>, u64)> {
612 let mplace = self.force_allocation(base)?;
613 self.mplace_to_simd(&mplace)
616 /// Computes a place. You should only use this if you intend to write into this
617 /// place; for reading, a more efficient alternative is `eval_place_for_read`.
620 place: mir::Place<'tcx>,
621 ) -> InterpResult<'tcx, PlaceTy<'tcx, M::PointerTag>> {
622 let mut place_ty = PlaceTy {
623 // This works even for dead/uninitialized locals; we check further when writing
624 place: Place::Local { frame: self.frame_idx(), local: place.local },
625 layout: self.layout_of_local(self.frame(), place.local, None)?,
628 for elem in place.projection.iter() {
629 place_ty = self.place_projection(&place_ty, &elem)?
632 trace!("{:?}", self.dump_place(place_ty.place));
633 // Sanity-check the type we ended up with.
634 debug_assert!(mir_assign_valid_types(
637 self.layout_of(self.subst_from_current_frame_and_normalize_erasing_regions(
638 place.ty(&self.frame().body.local_decls, *self.tcx).ty
645 /// Write an immediate to a place
647 pub fn write_immediate(
649 src: Immediate<M::PointerTag>,
650 dest: &PlaceTy<'tcx, M::PointerTag>,
651 ) -> InterpResult<'tcx> {
652 self.write_immediate_no_validate(src, dest)?;
654 if M::enforce_validity(self) {
655 // Data got changed, better make sure it matches the type!
656 self.validate_operand(&self.place_to_op(dest)?)?;
662 /// Write a scalar to a place
666 val: impl Into<ScalarMaybeUninit<M::PointerTag>>,
667 dest: &PlaceTy<'tcx, M::PointerTag>,
668 ) -> InterpResult<'tcx> {
669 self.write_immediate(Immediate::Scalar(val.into()), dest)
672 /// Write a pointer to a place
674 pub fn write_pointer(
676 ptr: impl Into<Pointer<Option<M::PointerTag>>>,
677 dest: &PlaceTy<'tcx, M::PointerTag>,
678 ) -> InterpResult<'tcx> {
679 self.write_scalar(Scalar::from_maybe_pointer(ptr.into(), self), dest)
682 /// Write an immediate to a place.
683 /// If you use this you are responsible for validating that things got copied at the
685 fn write_immediate_no_validate(
687 src: Immediate<M::PointerTag>,
688 dest: &PlaceTy<'tcx, M::PointerTag>,
689 ) -> InterpResult<'tcx> {
690 if cfg!(debug_assertions) {
691 // This is a very common path, avoid some checks in release mode
692 assert!(!dest.layout.is_unsized(), "Cannot write unsized data");
694 Immediate::Scalar(ScalarMaybeUninit::Scalar(Scalar::Ptr(..))) => assert_eq!(
697 "Size mismatch when writing pointer"
699 Immediate::Scalar(ScalarMaybeUninit::Scalar(Scalar::Int(int))) => {
700 assert_eq!(int.size(), dest.layout.size, "Size mismatch when writing bits")
702 Immediate::Scalar(ScalarMaybeUninit::Uninit) => {} // uninit can have any size
703 Immediate::ScalarPair(_, _) => {
704 // FIXME: Can we check anything here?
708 trace!("write_immediate: {:?} <- {:?}: {}", *dest, src, dest.layout.ty);
710 // See if we can avoid an allocation. This is the counterpart to `try_read_immediate`,
711 // but not factored as a separate function.
712 let mplace = match dest.place {
713 Place::Local { frame, local } => {
714 match M::access_local_mut(self, frame, local)? {
716 // Local can be updated in-place.
717 *local = LocalValue::Live(Operand::Immediate(src));
721 // The local is in memory, go on below.
726 Place::Ptr(mplace) => mplace, // already referring to memory
728 let dest = MPlaceTy { mplace, layout: dest.layout };
730 // This is already in memory, write there.
731 self.write_immediate_to_mplace_no_validate(src, &dest)
734 /// Write an immediate to memory.
735 /// If you use this you are responsible for validating that things got copied at the
737 fn write_immediate_to_mplace_no_validate(
739 value: Immediate<M::PointerTag>,
740 dest: &MPlaceTy<'tcx, M::PointerTag>,
741 ) -> InterpResult<'tcx> {
742 // Note that it is really important that the type here is the right one, and matches the
743 // type things are read at. In case `src_val` is a `ScalarPair`, we don't do any magic here
744 // to handle padding properly, which is only correct if we never look at this data with the
747 // Invalid places are a thing: the return place of a diverging function
749 let Some(mut alloc) = self.get_alloc_mut(dest)? else {
754 // FIXME: We should check that there are dest.layout.size many bytes available in
755 // memory. The code below is not sufficient, with enough padding it might not
756 // cover all the bytes!
758 Immediate::Scalar(scalar) => {
759 match dest.layout.abi {
760 Abi::Scalar(_) => {} // fine
763 "write_immediate_to_mplace: invalid Scalar layout: {:#?}",
767 alloc.write_scalar(alloc_range(Size::ZERO, dest.layout.size), scalar)
769 Immediate::ScalarPair(a_val, b_val) => {
770 // We checked `ptr_align` above, so all fields will have the alignment they need.
771 // We would anyway check against `ptr_align.restrict_for_offset(b_offset)`,
772 // which `ptr.offset(b_offset)` cannot possibly fail to satisfy.
773 let (a, b) = match dest.layout.abi {
774 Abi::ScalarPair(a, b) => (a.value, b.value),
777 "write_immediate_to_mplace: invalid ScalarPair layout: {:#?}",
781 let (a_size, b_size) = (a.size(&tcx), b.size(&tcx));
782 let b_offset = a_size.align_to(b.align(&tcx).abi);
784 // It is tempting to verify `b_offset` against `layout.fields.offset(1)`,
785 // but that does not work: We could be a newtype around a pair, then the
786 // fields do not match the `ScalarPair` components.
788 alloc.write_scalar(alloc_range(Size::ZERO, a_size), a_val)?;
789 alloc.write_scalar(alloc_range(b_offset, b_size), b_val)
794 /// Copies the data from an operand to a place. This does not support transmuting!
795 /// Use `copy_op_transmute` if the layouts could disagree.
799 src: &OpTy<'tcx, M::PointerTag>,
800 dest: &PlaceTy<'tcx, M::PointerTag>,
801 ) -> InterpResult<'tcx> {
802 self.copy_op_no_validate(src, dest)?;
804 if M::enforce_validity(self) {
805 // Data got changed, better make sure it matches the type!
806 self.validate_operand(&self.place_to_op(dest)?)?;
812 /// Copies the data from an operand to a place. This does not support transmuting!
813 /// Use `copy_op_transmute` if the layouts could disagree.
814 /// Also, if you use this you are responsible for validating that things get copied at the
816 fn copy_op_no_validate(
818 src: &OpTy<'tcx, M::PointerTag>,
819 dest: &PlaceTy<'tcx, M::PointerTag>,
820 ) -> InterpResult<'tcx> {
821 // We do NOT compare the types for equality, because well-typed code can
822 // actually "transmute" `&mut T` to `&T` in an assignment without a cast.
823 if !mir_assign_valid_types(*self.tcx, self.param_env, src.layout, dest.layout) {
826 "type mismatch when copying!\nsrc: {:?},\ndest: {:?}",
832 // Let us see if the layout is simple so we take a shortcut, avoid force_allocation.
833 let src = match self.try_read_immediate(src)? {
835 assert!(!src.layout.is_unsized(), "cannot have unsized immediates");
836 // Yay, we got a value that we can write directly.
837 // FIXME: Add a check to make sure that if `src` is indirect,
838 // it does not overlap with `dest`.
839 return self.write_immediate_no_validate(*src_val, dest);
841 Err(mplace) => mplace,
843 // Slow path, this does not fit into an immediate. Just memcpy.
844 trace!("copy_op: {:?} <- {:?}: {}", *dest, src, dest.layout.ty);
846 // This interprets `src.meta` with the `dest` local's layout, if an unsized local
847 // is being initialized!
848 let (dest, size) = self.force_allocation_maybe_sized(dest, src.meta)?;
849 let size = size.unwrap_or_else(|| {
851 !dest.layout.is_unsized(),
852 "Cannot copy into already initialized unsized place"
856 assert_eq!(src.meta, dest.meta, "Can only copy between equally-sized instances");
859 .copy(src.ptr, src.align, dest.ptr, dest.align, size, /*nonoverlapping*/ true)
862 /// Copies the data from an operand to a place. The layouts may disagree, but they must
863 /// have the same size.
864 pub fn copy_op_transmute(
866 src: &OpTy<'tcx, M::PointerTag>,
867 dest: &PlaceTy<'tcx, M::PointerTag>,
868 ) -> InterpResult<'tcx> {
869 if mir_assign_valid_types(*self.tcx, self.param_env, src.layout, dest.layout) {
870 // Fast path: Just use normal `copy_op`
871 return self.copy_op(src, dest);
873 // We still require the sizes to match.
874 if src.layout.size != dest.layout.size {
875 // FIXME: This should be an assert instead of an error, but if we transmute within an
876 // array length computation, `typeck` may not have yet been run and errored out. In fact
877 // most likey we *are* running `typeck` right now. Investigate whether we can bail out
878 // on `typeck_results().has_errors` at all const eval entry points.
879 debug!("Size mismatch when transmuting!\nsrc: {:#?}\ndest: {:#?}", src, dest);
880 self.tcx.sess.delay_span_bug(
882 "size-changing transmute, should have been caught by transmute checking",
884 throw_inval!(TransmuteSizeDiff(src.layout.ty, dest.layout.ty));
886 // Unsized copies rely on interpreting `src.meta` with `dest.layout`, we want
887 // to avoid that here.
889 !src.layout.is_unsized() && !dest.layout.is_unsized(),
890 "Cannot transmute unsized data"
893 // The hard case is `ScalarPair`. `src` is already read from memory in this case,
894 // using `src.layout` to figure out which bytes to use for the 1st and 2nd field.
895 // We have to write them to `dest` at the offsets they were *read at*, which is
896 // not necessarily the same as the offsets in `dest.layout`!
897 // Hence we do the copy with the source layout on both sides. We also make sure to write
898 // into memory, because if `dest` is a local we would not even have a way to write
899 // at the `src` offsets; the fact that we came from a different layout would
901 let dest = self.force_allocation(dest)?;
902 self.copy_op_no_validate(
904 &PlaceTy::from(MPlaceTy { mplace: *dest, layout: src.layout }),
907 if M::enforce_validity(self) {
908 // Data got changed, better make sure it matches the type!
909 self.validate_operand(&dest.into())?;
915 /// Ensures that a place is in memory, and returns where it is.
916 /// If the place currently refers to a local that doesn't yet have a matching allocation,
917 /// create such an allocation.
918 /// This is essentially `force_to_memplace`.
920 /// This supports unsized types and returns the computed size to avoid some
921 /// redundant computation when copying; use `force_allocation` for a simpler, sized-only
923 pub fn force_allocation_maybe_sized(
925 place: &PlaceTy<'tcx, M::PointerTag>,
926 meta: MemPlaceMeta<M::PointerTag>,
927 ) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::PointerTag>, Option<Size>)> {
928 let (mplace, size) = match place.place {
929 Place::Local { frame, local } => {
930 match M::access_local_mut(self, frame, local)? {
931 Ok(&mut local_val) => {
932 // We need to make an allocation.
934 // We need the layout of the local. We can NOT use the layout we got,
935 // that might e.g., be an inner field of a struct with `Scalar` layout,
936 // that has different alignment than the outer field.
938 self.layout_of_local(&self.stack()[frame], local, None)?;
939 // We also need to support unsized types, and hence cannot use `allocate`.
940 let (size, align) = self
941 .size_and_align_of(&meta, &local_layout)?
942 .expect("Cannot allocate for non-dyn-sized type");
943 let ptr = self.memory.allocate(size, align, MemoryKind::Stack)?;
944 let mplace = MemPlace { ptr: ptr.into(), align, meta };
945 if let LocalValue::Live(Operand::Immediate(value)) = local_val {
946 // Preserve old value.
947 // We don't have to validate as we can assume the local
948 // was already valid for its type.
949 let mplace = MPlaceTy { mplace, layout: local_layout };
950 self.write_immediate_to_mplace_no_validate(value, &mplace)?;
952 // Now we can call `access_mut` again, asserting it goes well,
953 // and actually overwrite things.
954 *M::access_local_mut(self, frame, local).unwrap().unwrap() =
955 LocalValue::Live(Operand::Indirect(mplace));
958 Err(mplace) => (mplace, None), // this already was an indirect local
961 Place::Ptr(mplace) => (mplace, None),
963 // Return with the original layout, so that the caller can go on
964 Ok((MPlaceTy { mplace, layout: place.layout }, size))
968 pub fn force_allocation(
970 place: &PlaceTy<'tcx, M::PointerTag>,
971 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
972 Ok(self.force_allocation_maybe_sized(place, MemPlaceMeta::None)?.0)
977 layout: TyAndLayout<'tcx>,
978 kind: MemoryKind<M::MemoryKind>,
979 ) -> InterpResult<'static, MPlaceTy<'tcx, M::PointerTag>> {
980 let ptr = self.memory.allocate(layout.size, layout.align.abi, kind)?;
981 Ok(MPlaceTy::from_aligned_ptr(ptr.into(), layout))
984 /// Returns a wide MPlace of type `&'static [mut] str` to a new 1-aligned allocation.
988 kind: MemoryKind<M::MemoryKind>,
990 ) -> MPlaceTy<'tcx, M::PointerTag> {
991 let ptr = self.memory.allocate_bytes(str.as_bytes(), Align::ONE, kind, mutbl);
992 let meta = Scalar::from_machine_usize(u64::try_from(str.len()).unwrap(), self);
994 MemPlace { ptr: ptr.into(), align: Align::ONE, meta: MemPlaceMeta::Meta(meta) };
996 let ty = self.tcx.mk_ref(
997 self.tcx.lifetimes.re_static,
998 ty::TypeAndMut { ty: self.tcx.types.str_, mutbl },
1000 let layout = self.layout_of(ty).unwrap();
1001 MPlaceTy { mplace, layout }
1004 /// Writes the discriminant of the given variant.
1005 pub fn write_discriminant(
1007 variant_index: VariantIdx,
1008 dest: &PlaceTy<'tcx, M::PointerTag>,
1009 ) -> InterpResult<'tcx> {
1010 // This must be an enum or generator.
1011 match dest.layout.ty.kind() {
1012 ty::Adt(adt, _) => assert!(adt.is_enum()),
1013 ty::Generator(..) => {}
1016 "write_discriminant called on non-variant-type (neither enum nor generator)"
1019 // Layout computation excludes uninhabited variants from consideration
1020 // therefore there's no way to represent those variants in the given layout.
1021 // Essentially, uninhabited variants do not have a tag that corresponds to their
1022 // discriminant, so we cannot do anything here.
1023 // When evaluating we will always error before even getting here, but ConstProp 'executes'
1024 // dead code, so we cannot ICE here.
1025 if dest.layout.for_variant(self, variant_index).abi.is_uninhabited() {
1026 throw_ub!(UninhabitedEnumVariantWritten)
1029 match dest.layout.variants {
1030 Variants::Single { index } => {
1031 assert_eq!(index, variant_index);
1033 Variants::Multiple {
1034 tag_encoding: TagEncoding::Direct,
1039 // No need to validate that the discriminant here because the
1040 // `TyAndLayout::for_variant()` call earlier already checks the variant is valid.
1043 dest.layout.ty.discriminant_for_variant(*self.tcx, variant_index).unwrap().val;
1045 // raw discriminants for enums are isize or bigger during
1046 // their computation, but the in-memory tag is the smallest possible
1048 let size = tag_layout.value.size(self);
1049 let tag_val = size.truncate(discr_val);
1051 let tag_dest = self.place_field(dest, tag_field)?;
1052 self.write_scalar(Scalar::from_uint(tag_val, size), &tag_dest)?;
1054 Variants::Multiple {
1056 TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
1061 // No need to validate that the discriminant here because the
1062 // `TyAndLayout::for_variant()` call earlier already checks the variant is valid.
1064 if variant_index != dataful_variant {
1065 let variants_start = niche_variants.start().as_u32();
1066 let variant_index_relative = variant_index
1068 .checked_sub(variants_start)
1069 .expect("overflow computing relative variant idx");
1070 // We need to use machine arithmetic when taking into account `niche_start`:
1071 // tag_val = variant_index_relative + niche_start_val
1072 let tag_layout = self.layout_of(tag_layout.value.to_int_ty(*self.tcx))?;
1073 let niche_start_val = ImmTy::from_uint(niche_start, tag_layout);
1074 let variant_index_relative_val =
1075 ImmTy::from_uint(variant_index_relative, tag_layout);
1076 let tag_val = self.binary_op(
1078 &variant_index_relative_val,
1082 let niche_dest = self.place_field(dest, tag_field)?;
1083 self.write_immediate(*tag_val, &niche_dest)?;
1091 pub fn raw_const_to_mplace(
1093 raw: ConstAlloc<'tcx>,
1094 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
1095 // This must be an allocation in `tcx`
1096 let _ = self.tcx.global_alloc(raw.alloc_id);
1097 let ptr = self.global_base_pointer(Pointer::from(raw.alloc_id))?;
1098 let layout = self.layout_of(raw.ty)?;
1099 Ok(MPlaceTy::from_aligned_ptr(ptr.into(), layout))
1102 /// Turn a place with a `dyn Trait` type into a place with the actual dynamic type.
1103 /// Also return some more information so drop doesn't have to run the same code twice.
1104 pub(super) fn unpack_dyn_trait(
1106 mplace: &MPlaceTy<'tcx, M::PointerTag>,
1107 ) -> InterpResult<'tcx, (ty::Instance<'tcx>, MPlaceTy<'tcx, M::PointerTag>)> {
1108 let vtable = self.scalar_to_ptr(mplace.vtable()); // also sanity checks the type
1109 let (instance, ty) = self.read_drop_type_from_vtable(vtable)?;
1110 let layout = self.layout_of(ty)?;
1112 // More sanity checks
1113 if cfg!(debug_assertions) {
1114 let (size, align) = self.read_size_and_align_from_vtable(vtable)?;
1115 assert_eq!(size, layout.size);
1116 // only ABI alignment is preserved
1117 assert_eq!(align, layout.align.abi);
1120 let mplace = MPlaceTy { mplace: MemPlace { meta: MemPlaceMeta::None, ..**mplace }, layout };
1121 Ok((instance, mplace))