1 //! This module specifies the type based interner for constants.
3 //! After a const evaluation has computed a value, before we destroy the const evaluator's session
4 //! memory, we need to extract all memory allocations to the global memory pool so they stay around.
6 use super::validity::RefTracking;
7 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
9 use rustc_middle::mir::interpret::InterpResult;
10 use rustc_middle::ty::{self, query::TyCtxtAt, Ty};
12 use rustc_ast::ast::Mutability;
14 use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, Scalar, ValueVisitor};
16 pub trait CompileTimeMachine<'mir, 'tcx> = Machine<
24 MemoryMap = FxHashMap<AllocId, (MemoryKind<!>, Allocation)>,
27 struct InternVisitor<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> {
28 /// The ectx from which we intern.
29 ecx: &'rt mut InterpCx<'mir, 'tcx, M>,
30 /// Previously encountered safe references.
31 ref_tracking: &'rt mut RefTracking<(MPlaceTy<'tcx>, InternMode)>,
32 /// A list of all encountered allocations. After type-based interning, we traverse this list to
33 /// also intern allocations that are only referenced by a raw pointer or inside a union.
34 leftover_allocations: &'rt mut FxHashSet<AllocId>,
35 /// The root kind of the value that we're looking at. This field is never mutated and only used
36 /// for sanity assertions that will ICE when `const_qualif` screws up.
38 /// This field stores whether we are *currently* inside an `UnsafeCell`. This can affect
39 /// the intern mode of references we encounter.
40 inside_unsafe_cell: bool,
42 /// This flag is to avoid triggering UnsafeCells are not allowed behind references in constants
44 /// It's a copy of `mir::Body`'s ignore_interior_mut_in_const_validation field
45 ignore_interior_mut_in_const: bool,
48 #[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)]
50 /// A static and its current mutability. Below shared references inside a `static mut`,
51 /// this is *immutable*, and below mutable references inside an `UnsafeCell`, this
53 Static(hir::Mutability),
54 /// The "base value" of a const, which can have `UnsafeCell` (as in `const FOO: Cell<i32>`),
55 /// but that interior mutability is simply ignored.
57 /// The "inner values" of a const with references, where `UnsafeCell` is an error.
61 /// Signalling data structure to ensure we don't recurse
62 /// into the memory of other constants or statics
65 fn mutable_memory_in_const(tcx: TyCtxtAt<'_>, kind: &str) {
66 // FIXME: show this in validation instead so we can point at where in the value the error is?
67 tcx.sess.span_err(tcx.span, &format!("mutable memory ({}) is not allowed in constant", kind));
70 /// Intern an allocation without looking at its children.
71 /// `mode` is the mode of the environment where we found this pointer.
72 /// `mutablity` is the mutability of the place to be interned; even if that says
73 /// `immutable` things might become mutable if `ty` is not frozen.
74 /// `ty` can be `None` if there is no potential interior mutability
75 /// to account for (e.g. for vtables).
76 fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>>(
77 ecx: &'rt mut InterpCx<'mir, 'tcx, M>,
78 leftover_allocations: &'rt mut FxHashSet<AllocId>,
82 ) -> Option<IsStaticOrFn> {
83 trace!("intern_shallow {:?} with {:?}", alloc_id, mode);
86 let (kind, mut alloc) = match ecx.memory.alloc_map.remove(&alloc_id) {
89 // Pointer not found in local memory map. It is either a pointer to the global
91 // If the pointer is dangling (neither in local nor global memory), we leave it
92 // to validation to error -- it has the much better error messages, pointing out where
93 // in the value the dangling reference lies.
94 // The `delay_span_bug` ensures that we don't forget such a check in validation.
95 if tcx.get_global_alloc(alloc_id).is_none() {
96 tcx.sess.delay_span_bug(ecx.tcx.span, "tried to intern dangling pointer");
98 // treat dangling pointers like other statics
99 // just to stop trying to recurse into them
100 return Some(IsStaticOrFn);
103 // This match is just a canary for future changes to `MemoryKind`, which most likely need
104 // changes in this function.
106 MemoryKind::Stack | MemoryKind::Vtable | MemoryKind::CallerLocation => {}
108 // Set allocation mutability as appropriate. This is used by LLVM to put things into
109 // read-only memory, and also by Miri when evaluating other globals that
111 if let InternMode::Static(mutability) = mode {
112 // For this, we need to take into account `UnsafeCell`. When `ty` is `None`, we assume
113 // no interior mutability.
114 let frozen = ty.map_or(true, |ty| ty.is_freeze(ecx.tcx, ecx.param_env));
115 // For statics, allocation mutability is the combination of the place mutability and
116 // the type mutability.
117 // The entire allocation needs to be mutable if it contains an `UnsafeCell` anywhere.
118 let immutable = mutability == Mutability::Not && frozen;
120 alloc.mutability = Mutability::Not;
122 // Just making sure we are not "upgrading" an immutable allocation to mutable.
123 assert_eq!(alloc.mutability, Mutability::Mut);
126 // No matter what, *constants are never mutable*. Mutating them is UB.
127 // See const_eval::machine::MemoryExtra::can_access_statics for why
128 // immutability is so important.
130 // There are no sensible checks we can do here; grep for `mutable_memory_in_const` to
131 // find the checks we are doing elsewhere to avoid even getting here for memory
132 // that "wants" to be mutable.
133 alloc.mutability = Mutability::Not;
135 // link the alloc id to the actual allocation
136 let alloc = tcx.intern_const_alloc(alloc);
137 leftover_allocations.extend(alloc.relocations().iter().map(|&(_, ((), reloc))| reloc));
138 tcx.set_alloc_id_memory(alloc_id, alloc);
142 impl<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> InternVisitor<'rt, 'mir, 'tcx, M> {
147 ty: Option<Ty<'tcx>>,
148 ) -> Option<IsStaticOrFn> {
149 intern_shallow(self.ecx, self.leftover_allocations, alloc_id, mode, ty)
153 impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
154 for InternVisitor<'rt, 'mir, 'tcx, M>
156 type V = MPlaceTy<'tcx>;
159 fn ecx(&self) -> &InterpCx<'mir, 'tcx, M> {
165 mplace: MPlaceTy<'tcx>,
166 fields: impl Iterator<Item = InterpResult<'tcx, Self::V>>,
167 ) -> InterpResult<'tcx> {
168 if let Some(def) = mplace.layout.ty.ty_adt_def() {
169 if Some(def.did) == self.ecx.tcx.lang_items().unsafe_cell_type() {
170 if self.mode == InternMode::ConstInner && !self.ignore_interior_mut_in_const {
171 // We do not actually make this memory mutable. But in case the user
172 // *expected* it to be mutable, make sure we error. This is just a
173 // sanity check to prevent users from accidentally exploiting the UB
174 // they caused. It also helps us to find cases where const-checking
175 // failed to prevent an `UnsafeCell` (but as `ignore_interior_mut_in_const`
176 // shows that part is not airtight).
177 mutable_memory_in_const(self.ecx.tcx, "`UnsafeCell`");
179 // We are crossing over an `UnsafeCell`, we can mutate again. This means that
180 // References we encounter inside here are interned as pointing to mutable
182 // Remember the `old` value to handle nested `UnsafeCell`.
183 let old = std::mem::replace(&mut self.inside_unsafe_cell, true);
184 let walked = self.walk_aggregate(mplace, fields);
185 self.inside_unsafe_cell = old;
189 self.walk_aggregate(mplace, fields)
192 fn visit_value(&mut self, mplace: MPlaceTy<'tcx>) -> InterpResult<'tcx> {
193 // Handle Reference types, as these are the only relocations supported by const eval.
194 // Raw pointers (and boxes) are handled by the `leftover_relocations` logic.
195 let tcx = self.ecx.tcx;
196 let ty = mplace.layout.ty;
197 if let ty::Ref(_, referenced_ty, ref_mutability) = ty.kind {
198 let value = self.ecx.read_immediate(mplace.into())?;
199 let mplace = self.ecx.ref_to_mplace(value)?;
200 assert_eq!(mplace.layout.ty, referenced_ty);
201 // Handle trait object vtables.
202 if let ty::Dynamic(..) =
203 tcx.struct_tail_erasing_lifetimes(referenced_ty, self.ecx.param_env).kind
205 // Validation will error (with a better message) on an invalid vtable pointer
206 // so we can safely not do anything if this is not a real pointer.
207 if let Scalar::Ptr(vtable) = mplace.meta.unwrap_meta() {
208 // Explicitly choose const mode here, since vtables are immutable, even
209 // if the reference of the fat pointer is mutable.
210 self.intern_shallow(vtable.alloc_id, InternMode::ConstInner, None);
212 // Let validation show the error message, but make sure it *does* error.
214 .delay_span_bug(tcx.span, "vtables pointers cannot be integer pointers");
217 // Check if we have encountered this pointer+layout combination before.
218 // Only recurse for allocation-backed pointers.
219 if let Scalar::Ptr(ptr) = mplace.ptr {
220 // Compute the mode with which we intern this.
221 let ref_mode = match self.mode {
222 InternMode::Static(mutbl) => {
223 // In statics, merge outer mutability with reference mutability and
224 // take into account whether we are in an `UnsafeCell`.
226 // The only way a mutable reference actually works as a mutable reference is
227 // by being in a `static mut` directly or behind another mutable reference.
228 // If there's an immutable reference or we are inside a `static`, then our
229 // mutable reference is equivalent to an immutable one. As an example:
230 // `&&mut Foo` is semantically equivalent to `&&Foo`
231 match ref_mutability {
232 _ if self.inside_unsafe_cell => {
233 // Inside an `UnsafeCell` is like inside a `static mut`, the "outer"
234 // mutability does not matter.
235 InternMode::Static(ref_mutability)
238 // A shared reference, things become immutable.
239 // We do *not* consier `freeze` here -- that is done more precisely
240 // when traversing the referenced data (by tracking `UnsafeCell`).
241 InternMode::Static(Mutability::Not)
244 // Mutable reference.
245 InternMode::Static(mutbl)
249 InternMode::ConstBase | InternMode::ConstInner => {
250 // Ignore `UnsafeCell`, everything is immutable. Do some sanity checking
251 // for mutable references that we encounter -- they must all be ZST.
252 // This helps to prevent users from accidentally exploiting UB that they
253 // caused (by somehow getting a mutable reference in a `const`).
254 if ref_mutability == Mutability::Mut {
255 match referenced_ty.kind {
256 ty::Array(_, n) if n.eval_usize(*tcx, self.ecx.param_env) == 0 => {}
258 if mplace.meta.unwrap_meta().to_machine_usize(self.ecx)?
260 _ => mutable_memory_in_const(tcx, "`&mut`"),
263 // A shared reference. We cannot check `freeze` here due to references
264 // like `&dyn Trait` that are actually immutable. We do check for
265 // concrete `UnsafeCell` when traversing the pointee though (if it is
266 // a new allocation, not yet interned).
268 // Go on with the "inner" rules.
269 InternMode::ConstInner
272 match self.intern_shallow(ptr.alloc_id, ref_mode, Some(referenced_ty)) {
273 // No need to recurse, these are interned already and statics may have
274 // cycles, so we don't want to recurse there
275 Some(IsStaticOrFn) => {}
276 // intern everything referenced by this value. The mutability is taken from the
277 // reference. It is checked above that mutable references only happen in
279 None => self.ref_tracking.track((mplace, ref_mode), || ()),
284 // Not a reference -- proceed recursively.
285 self.walk_value(mplace)
290 #[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)]
291 pub enum InternKind {
292 /// The `mutability` of the static, ignoring the type which may have interior mutability.
293 Static(hir::Mutability),
298 /// Intern `ret` and everything it references.
300 /// This *cannot raise an interpreter error*. Doing so is left to validation, which
301 /// tracks where in the value we are and thus can show much better error messages.
302 /// Any errors here would anyway be turned into `const_err` lints, whereas validation failures
304 pub fn intern_const_alloc_recursive<M: CompileTimeMachine<'mir, 'tcx>>(
305 ecx: &mut InterpCx<'mir, 'tcx, M>,
306 intern_kind: InternKind,
308 ignore_interior_mut_in_const: bool,
313 let base_intern_mode = match intern_kind {
314 InternKind::Static(mutbl) => InternMode::Static(mutbl),
315 // FIXME: what about array lengths, array initializers?
316 InternKind::Constant | InternKind::Promoted => InternMode::ConstBase,
319 // Type based interning.
320 // `ref_tracking` tracks typed references we have already interned and still need to crawl for
321 // more typed information inside them.
322 // `leftover_allocations` collects *all* allocations we see, because some might not
323 // be available in a typed way. They get interned at the end.
324 let mut ref_tracking = RefTracking::empty();
325 let leftover_allocations = &mut FxHashSet::default();
327 // start with the outermost allocation
330 leftover_allocations,
331 // The outermost allocation must exist, because we allocated it with
332 // `Memory::allocate`.
333 ret.ptr.assert_ptr().alloc_id,
338 ref_tracking.track((ret, base_intern_mode), || ());
340 while let Some(((mplace, mode), _)) = ref_tracking.todo.pop() {
341 let res = InternVisitor {
342 ref_tracking: &mut ref_tracking,
345 leftover_allocations,
346 ignore_interior_mut_in_const,
347 inside_unsafe_cell: false,
349 .visit_value(mplace);
350 // We deliberately *ignore* interpreter errors here. When there is a problem, the remaining
351 // references are "leftover"-interned, and later validation will show a proper error
352 // and point at the right part of the value causing the problem.
356 ecx.tcx.sess.delay_span_bug(
359 "error during interning should later cause validation failure: {}",
363 // Some errors shouldn't come up because creating them causes
364 // an allocation, which we should avoid. When that happens,
365 // dedicated error variants should be introduced instead.
367 !error.kind.allocates(),
368 "interning encountered allocating error: {}",
375 // Intern the rest of the allocations as mutable. These might be inside unions, padding, raw
376 // pointers, ... So we can't intern them according to their type rules
378 let mut todo: Vec<_> = leftover_allocations.iter().cloned().collect();
379 while let Some(alloc_id) = todo.pop() {
380 if let Some((_, mut alloc)) = ecx.memory.alloc_map.remove(&alloc_id) {
381 // We can't call the `intern_shallow` method here, as its logic is tailored to safe
382 // references and a `leftover_allocations` set (where we only have a todo-list here).
383 // So we hand-roll the interning logic here again.
385 // Statics may contain mutable allocations even behind relocations.
386 // Even for immutable statics it would be ok to have mutable allocations behind
387 // raw pointers, e.g. for `static FOO: *const AtomicUsize = &AtomicUsize::new(42)`.
388 InternKind::Static(_) => {}
389 // Raw pointers in promoteds may only point to immutable things so we mark
390 // everything as immutable.
391 // It is UB to mutate through a raw pointer obtained via an immutable reference:
392 // Since all references and pointers inside a promoted must by their very definition
393 // be created from an immutable reference (and promotion also excludes interior
394 // mutability), mutating through them would be UB.
395 // There's no way we can check whether the user is using raw pointers correctly,
396 // so all we can do is mark this as immutable here.
397 InternKind::Promoted => {
398 // See const_eval::machine::MemoryExtra::can_access_statics for why
399 // immutability is so important.
400 alloc.mutability = Mutability::Not;
402 InternKind::Constant => {
403 // If it's a constant, we should not have any "leftovers" as everything
404 // is tracked by const-checking.
405 // FIXME: downgrade this to a warning? It rejects some legitimate consts,
406 // such as `const CONST_RAW: *const Vec<i32> = &Vec::new() as *const _;`.
409 .span_err(ecx.tcx.span, "untyped pointers are not allowed in constant");
410 // For better errors later, mark the allocation as immutable.
411 alloc.mutability = Mutability::Not;
414 let alloc = tcx.intern_const_alloc(alloc);
415 tcx.set_alloc_id_memory(alloc_id, alloc);
416 for &(_, ((), reloc)) in alloc.relocations().iter() {
417 if leftover_allocations.insert(reloc) {
421 } else if ecx.memory.dead_alloc_map.contains_key(&alloc_id) {
422 // Codegen does not like dangling pointers, and generally `tcx` assumes that
423 // all allocations referenced anywhere actually exist. So, make sure we error here.
424 ecx.tcx.sess.span_err(ecx.tcx.span, "encountered dangling pointer in final constant");
425 } else if ecx.tcx.get_global_alloc(alloc_id).is_none() {
426 // We have hit an `AllocId` that is neither in local or global memory and isn't
427 // marked as dangling by local memory. That should be impossible.
428 span_bug!(ecx.tcx.span, "encountered unknown alloc id {:?}", alloc_id);