X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=src%2Fconstant.rs;h=2a2573aad295025f14f1d743c63b64845d9412db;hb=e3091d74d876bbb6b02cf218b064ee47d1551e41;hp=ab099292bf4804bc91833d3b535cda06108432bd;hpb=f0bb30f8a1ac6107555dfc8fe830d42f469af7f8;p=rust.git diff --git a/src/constant.rs b/src/constant.rs index ab099292bf4..2a2573aad29 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -1,32 +1,37 @@ -use std::borrow::Cow; +//! Handling of `static`s, `const`s and promoted allocations -use rustc::mir::interpret::{ - read_target_uint, AllocId, Allocation, ConstValue, GlobalAlloc, GlobalId, InterpResult, Scalar, -}; -use rustc::ty::{layout::Align, Const, ConstKind}; -use rustc_mir::interpret::{ - ImmTy, InterpCx, Machine, Memory, MemoryKind, OpTy, PlaceTy, Pointer, StackPopCleanup, - StackPopInfo, +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_errors::ErrorReported; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mir::interpret::{ + read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Scalar, }; +use rustc_middle::ty::ConstKind; +use rustc_span::DUMMY_SP; +use cranelift_codegen::ir::GlobalValueData; use cranelift_module::*; use crate::prelude::*; -#[derive(Default)] -pub struct ConstantCx { - todo: HashSet, - done: HashSet, +pub(crate) struct ConstantCx { + todo: Vec, + done: FxHashSet, + anon_allocs: FxHashMap, } -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Copy, Clone, Debug)] enum TodoItem { Alloc(AllocId), Static(DefId), } impl ConstantCx { - pub fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut Module) { + pub(crate) fn new() -> Self { + ConstantCx { todo: vec![], done: FxHashSet::default(), anon_allocs: FxHashMap::default() } + } + + pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) { //println!("todo {:?}", self.todo); define_all_allocs(tcx, module, &mut self); //println!("done {:?}", self.done); @@ -34,281 +39,343 @@ pub fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut Module) { } } -pub fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) { - constants_cx.todo.insert(TodoItem::Static(def_id)); +pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool { + let mut all_constants_ok = true; + for constant in &fx.mir.required_consts { + let const_ = match fx.monomorphize(constant.literal) { + ConstantKind::Ty(ct) => ct, + ConstantKind::Val(..) => continue, + }; + match const_.val { + ConstKind::Value(_) => {} + ConstKind::Unevaluated(unevaluated) => { + if let Err(err) = + fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) + { + all_constants_ok = false; + match err { + ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted => { + fx.tcx.sess.span_err(constant.span, "erroneous constant encountered"); + } + ErrorHandled::TooGeneric => { + span_bug!( + constant.span, + "codgen encountered polymorphic constant: {:?}", + err + ); + } + } + } + } + ConstKind::Param(_) + | ConstKind::Infer(_) + | ConstKind::Bound(_, _) + | ConstKind::Placeholder(_) + | ConstKind::Error(_) => unreachable!("{:?}", const_), + } + } + all_constants_ok } -pub fn codegen_static_ref<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Backend>, +pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) { + let mut constants_cx = ConstantCx::new(); + constants_cx.todo.push(TodoItem::Static(def_id)); + constants_cx.finalize(tcx, module); +} + +pub(crate) fn codegen_tls_ref<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, def_id: DefId, - ty: Ty<'tcx>, -) -> CPlace<'tcx> { - let linkage = crate::linkage::get_static_ref_linkage(fx.tcx, def_id); - let data_id = data_id_for_static(fx.tcx, fx.module, def_id, linkage); - cplace_for_dataid(fx, ty, data_id) + layout: TyAndLayout<'tcx>, +) -> CValue<'tcx> { + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("tls {:?}", def_id)); + } + let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id); + CValue::by_val(tls_ptr, layout) } -pub fn trans_promoted<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Backend>, - instance: Instance<'tcx>, - promoted: Promoted, - dest_ty: Ty<'tcx>, +fn codegen_static_ref<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + def_id: DefId, + layout: TyAndLayout<'tcx>, ) -> CPlace<'tcx> { - match fx.tcx.const_eval(ParamEnv::reveal_all().and(GlobalId { - instance, - promoted: Some(promoted), - })) { - Ok(const_) => { - let cplace = trans_const_place(fx, const_); - debug_assert_eq!(cplace.layout(), fx.layout_of(dest_ty)); - cplace - } - Err(_) => crate::trap::trap_unreachable_ret_place( - fx, - fx.layout_of(dest_ty), - "[panic] Tried to get value of promoted value with errored during const eval.", - ), + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", def_id)); } + let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); + assert!(!layout.is_unsized(), "unsized statics aren't supported"); + assert!( + matches!( + fx.bcx.func.global_values[local_data_id], + GlobalValueData::Symbol { tls: false, .. } + ), + "tls static referenced without Rvalue::ThreadLocalRef" + ); + CPlace::for_ptr(crate::pointer::Pointer::new(global_ptr), layout) } -pub fn trans_constant<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Backend>, +pub(crate) fn codegen_constant<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, constant: &Constant<'tcx>, ) -> CValue<'tcx> { - let const_ = force_eval_const(fx, &constant.literal); - trans_const_value(fx, const_) -} - -pub fn force_eval_const<'tcx>( - fx: &FunctionCx<'_, 'tcx, impl Backend>, - const_: &'tcx Const, -) -> &'tcx Const<'tcx> { - match const_.val { - ConstKind::Unevaluated(def_id, ref substs) => { - let param_env = ParamEnv::reveal_all(); - let substs = fx.monomorphize(substs); - let instance = Instance::resolve(fx.tcx, param_env, def_id, substs).unwrap(); - let cid = GlobalId { - instance, - promoted: None, - }; - fx.tcx.const_eval(param_env.and(cid)).unwrap() + let const_ = match fx.monomorphize(constant.literal) { + ConstantKind::Ty(ct) => ct, + ConstantKind::Val(val, ty) => return codegen_const_value(fx, val, ty), + }; + let const_val = match const_.val { + ConstKind::Value(const_val) => const_val, + ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) + if fx.tcx.is_static(def.did) => + { + assert!(substs.is_empty()); + assert!(promoted.is_none()); + + return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty)).to_cvalue(fx); } - _ => fx.monomorphize(&const_), - } + ConstKind::Unevaluated(unevaluated) => { + match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) { + Ok(const_val) => const_val, + Err(_) => { + span_bug!(constant.span, "erroneous constant not captured by required_consts"); + } + } + } + ConstKind::Param(_) + | ConstKind::Infer(_) + | ConstKind::Bound(_, _) + | ConstKind::Placeholder(_) + | ConstKind::Error(_) => unreachable!("{:?}", const_), + }; + + codegen_const_value(fx, const_val, const_.ty) } -pub fn trans_const_value<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Backend>, - const_: &'tcx Const<'tcx>, +pub(crate) fn codegen_const_value<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + const_val: ConstValue<'tcx>, + ty: Ty<'tcx>, ) -> CValue<'tcx> { - let ty = fx.monomorphize(&const_.ty); let layout = fx.layout_of(ty); - match ty.kind { - ty::Bool | ty::Uint(_) => { - let bits = const_.val.try_to_bits(layout.size).unwrap(); - CValue::const_val(fx, ty, bits) - } - ty::Int(_) => { - let bits = const_.val.try_to_bits(layout.size).unwrap(); - CValue::const_val( - fx, - ty, - rustc::mir::interpret::sign_extend(bits, layout.size), - ) - } - ty::Float(fty) => { - let bits = const_.val.try_to_bits(layout.size).unwrap(); - let val = match fty { - FloatTy::F32 => fx - .bcx - .ins() - .f32const(Ieee32::with_bits(u32::try_from(bits).unwrap())), - FloatTy::F64 => fx - .bcx - .ins() - .f64const(Ieee64::with_bits(u64::try_from(bits).unwrap())), - }; - CValue::by_val(val, layout) - } - ty::FnDef(_def_id, _substs) => CValue::by_ref( - fx.bcx - .ins() - .iconst(fx.pointer_type, fx.pointer_type.bytes() as i64), + assert!(!layout.is_unsized(), "sized const value"); + + if layout.is_zst() { + return CValue::by_ref(crate::Pointer::dangling(layout.align.pref), layout); + } + + match const_val { + ConstValue::Scalar(x) => match x { + Scalar::Int(int) => { + if fx.clif_type(layout.ty).is_some() { + return CValue::const_val(fx, layout, int); + } else { + let raw_val = int.to_bits(int.size()).unwrap(); + let val = match int.size().bytes() { + 1 => fx.bcx.ins().iconst(types::I8, raw_val as i64), + 2 => fx.bcx.ins().iconst(types::I16, raw_val as i64), + 4 => fx.bcx.ins().iconst(types::I32, raw_val as i64), + 8 => fx.bcx.ins().iconst(types::I64, raw_val as i64), + 16 => { + let lsb = fx.bcx.ins().iconst(types::I64, raw_val as u64 as i64); + let msb = + fx.bcx.ins().iconst(types::I64, (raw_val >> 64) as u64 as i64); + fx.bcx.ins().iconcat(lsb, msb) + } + _ => unreachable!(), + }; + + let place = CPlace::new_stack_slot(fx, layout); + place.to_ptr().store(fx, val, MemFlags::trusted()); + place.to_cvalue(fx) + } + } + Scalar::Ptr(ptr) => { + let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id); + let base_addr = match alloc_kind { + Some(GlobalAlloc::Memory(alloc)) => { + let data_id = data_id_for_alloc_id( + &mut fx.constants_cx, + fx.module, + ptr.alloc_id, + alloc.mutability, + ); + let local_data_id = + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id)); + } + fx.bcx.ins().global_value(fx.pointer_type, local_data_id) + } + Some(GlobalAlloc::Function(instance)) => { + let func_id = crate::abi::import_function(fx.tcx, fx.module, instance); + let local_func_id = + fx.module.declare_func_in_func(func_id, &mut fx.bcx.func); + fx.bcx.ins().func_addr(fx.pointer_type, local_func_id) + } + Some(GlobalAlloc::Static(def_id)) => { + assert!(fx.tcx.is_static(def_id)); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", def_id)); + } + fx.bcx.ins().global_value(fx.pointer_type, local_data_id) + } + None => bug!("missing allocation {:?}", ptr.alloc_id), + }; + let val = if ptr.offset.bytes() != 0 { + fx.bcx.ins().iadd_imm(base_addr, i64::try_from(ptr.offset.bytes()).unwrap()) + } else { + base_addr + }; + CValue::by_val(val, layout) + } + }, + ConstValue::ByRef { alloc, offset } => CValue::by_ref( + pointer_for_allocation(fx, alloc) + .offset_i64(fx, i64::try_from(offset.bytes()).unwrap()), layout, ), - _ => trans_const_place(fx, const_).to_cvalue(fx), + ConstValue::Slice { data, start, end } => { + let ptr = pointer_for_allocation(fx, data) + .offset_i64(fx, i64::try_from(start).unwrap()) + .get_addr(fx); + let len = fx + .bcx + .ins() + .iconst(fx.pointer_type, i64::try_from(end.checked_sub(start).unwrap()).unwrap()); + CValue::by_val_pair(ptr, len, layout) + } } } -fn trans_const_place<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Backend>, - const_: &'tcx Const<'tcx>, -) -> CPlace<'tcx> { - // Adapted from https://github.com/rust-lang/rust/pull/53671/files#diff-e0b58bb6712edaa8595ad7237542c958L551 - let result = || -> InterpResult<'tcx, &'tcx Allocation> { - let mut ecx = InterpCx::new( - fx.tcx.at(DUMMY_SP), - ty::ParamEnv::reveal_all(), - TransPlaceInterpreter, - (), - ); - ecx.push_stack_frame( - fx.instance, - DUMMY_SP, - fx.mir, - None, - StackPopCleanup::None { cleanup: false }, - ) - .unwrap(); - let op = ecx.eval_operand( - &Operand::Constant(Box::new(Constant { - span: DUMMY_SP, - user_ty: None, - literal: const_, - })), - None, - )?; - let ptr = ecx.allocate(op.layout, MemoryKind::Stack); - ecx.copy_op(op, ptr.into())?; - let alloc = ecx - .memory - .get_raw(ptr.to_ref().to_scalar()?.to_ptr()?.alloc_id)?; - Ok(fx.tcx.intern_const_alloc(alloc.clone())) - }; - let alloc = result().expect("unable to convert ConstKind to Allocation"); +pub(crate) fn pointer_for_allocation<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + alloc: &'tcx Allocation, +) -> crate::pointer::Pointer { + let alloc_id = fx.tcx.create_memory_alloc(alloc); + let data_id = + data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability); - //println!("const value: {:?} allocation: {:?}", value, alloc); - let alloc_id = fx.tcx.alloc_map.lock().create_memory_alloc(alloc); - fx.constants_cx.todo.insert(TodoItem::Alloc(alloc_id)); - let data_id = data_id_for_alloc_id(fx.module, alloc_id, alloc.align); - cplace_for_dataid(fx, const_.ty, data_id) + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", alloc_id)); + } + let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); + crate::pointer::Pointer::new(global_ptr) } -fn data_id_for_alloc_id( - module: &mut Module, +pub(crate) fn data_id_for_alloc_id( + cx: &mut ConstantCx, + module: &mut dyn Module, alloc_id: AllocId, - align: Align, + mutability: rustc_hir::Mutability, ) -> DataId { - module - .declare_data( - &format!("__alloc_{}", alloc_id.0), - Linkage::Local, - false, - Some(align.bytes() as u8), - ) - .unwrap() + cx.todo.push(TodoItem::Alloc(alloc_id)); + *cx.anon_allocs.entry(alloc_id).or_insert_with(|| { + module.declare_anonymous_data(mutability == rustc_hir::Mutability::Mut, false).unwrap() + }) } fn data_id_for_static( tcx: TyCtxt<'_>, - module: &mut Module, + module: &mut dyn Module, def_id: DefId, - linkage: Linkage, + definition: bool, ) -> DataId { - let instance = Instance::mono(tcx, def_id); - let symbol_name = tcx.symbol_name(instance).name.as_str(); - let ty = instance.ty(tcx); + let rlinkage = tcx.codegen_fn_attrs(def_id).linkage; + let linkage = if definition { + crate::linkage::get_static_linkage(tcx, def_id) + } else if rlinkage == Some(rustc_middle::mir::mono::Linkage::ExternalWeak) + || rlinkage == Some(rustc_middle::mir::mono::Linkage::WeakAny) + { + Linkage::Preemptible + } else { + Linkage::Import + }; + + let instance = Instance::mono(tcx, def_id).polymorphize(tcx); + let symbol_name = tcx.symbol_name(instance).name; + let ty = instance.ty(tcx, ParamEnv::reveal_all()); let is_mutable = if tcx.is_mutable_static(def_id) { true } else { - !ty.is_freeze(tcx, ParamEnv::reveal_all(), DUMMY_SP) + !ty.is_freeze(tcx.at(DUMMY_SP), ParamEnv::reveal_all()) }; - let align = tcx - .layout_of(ParamEnv::reveal_all().and(ty)) - .unwrap() - .align - .pref - .bytes(); + let align = tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap().align.pref.bytes(); + + let attrs = tcx.codegen_fn_attrs(def_id); let data_id = module .declare_data( &*symbol_name, linkage, is_mutable, - Some(align.try_into().unwrap()), + attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL), ) .unwrap(); - if linkage == Linkage::Preemptible { - if let ty::RawPtr(_) = ty.kind { - } else { - tcx.sess.span_fatal( - tcx.def_span(def_id), - "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", - ) - } - + if rlinkage.is_some() { + // Comment copied from https://github.com/rust-lang/rust/blob/45060c2a66dfd667f88bd8b94261b28a58d85bd5/src/librustc_codegen_llvm/consts.rs#L141 + // Declare an internal global `extern_with_linkage_foo` which + // is initialized with the address of `foo`. If `foo` is + // discarded during linking (for example, if `foo` has weak + // linkage and there are no definitions), then + // `extern_with_linkage_foo` will instead be initialized to + // zero. + + let ref_name = format!("_rust_extern_with_linkage_{}", symbol_name); + let ref_data_id = module.declare_data(&ref_name, Linkage::Local, false, false).unwrap(); let mut data_ctx = DataContext::new(); - let zero_bytes = std::iter::repeat(0) - .take(pointer_ty(tcx).bytes() as usize) - .collect::>() - .into_boxed_slice(); - data_ctx.define(zero_bytes); - match module.define_data(data_id, &data_ctx) { - // Everytime a weak static is referenced, there will be a zero pointer definition, + data_ctx.set_align(align); + let data = module.declare_data_in_data(data_id, &mut data_ctx); + data_ctx.define(std::iter::repeat(0).take(pointer_ty(tcx).bytes() as usize).collect()); + data_ctx.write_data_addr(0, data, 0); + match module.define_data(ref_data_id, &data_ctx) { + // Every time the static is referenced there will be another definition of this global, // so duplicate definitions are expected and allowed. Err(ModuleError::DuplicateDefinition(_)) => {} res => res.unwrap(), } + ref_data_id + } else { + data_id } - - data_id } -fn cplace_for_dataid<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Backend>, - ty: Ty<'tcx>, - data_id: DataId, -) -> CPlace<'tcx> { - let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); - let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); - let layout = fx.layout_of(fx.monomorphize(&ty)); - assert!(!layout.is_unsized(), "unsized statics aren't supported"); - CPlace::for_addr(global_ptr, layout) -} - -fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut Module, cx: &mut ConstantCx) { - let memory = Memory::::new(tcx.at(DUMMY_SP), ()); - - while let Some(todo_item) = pop_set(&mut cx.todo) { - let (data_id, alloc) = match todo_item { +fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut ConstantCx) { + while let Some(todo_item) = cx.todo.pop() { + let (data_id, alloc, section_name) = match todo_item { TodoItem::Alloc(alloc_id) => { //println!("alloc_id {}", alloc_id); - let alloc = memory.get_raw(alloc_id).unwrap(); - let data_id = data_id_for_alloc_id(module, alloc_id, alloc.align); - (data_id, alloc) + let alloc = match tcx.get_global_alloc(alloc_id).unwrap() { + GlobalAlloc::Memory(alloc) => alloc, + GlobalAlloc::Function(_) | GlobalAlloc::Static(_) => unreachable!(), + }; + let data_id = *cx.anon_allocs.entry(alloc_id).or_insert_with(|| { + module + .declare_anonymous_data( + alloc.mutability == rustc_hir::Mutability::Mut, + false, + ) + .unwrap() + }); + (data_id, alloc, None) } TodoItem::Static(def_id) => { //println!("static {:?}", def_id); - if tcx.is_foreign_item(def_id) { - continue; - } - - let instance = ty::Instance::mono(tcx, def_id); - let cid = GlobalId { - instance, - promoted: None, - }; - let const_ = tcx.const_eval(ParamEnv::reveal_all().and(cid)).unwrap(); + let section_name = tcx.codegen_fn_attrs(def_id).link_section.map(|s| s.as_str()); - let alloc = match const_.val { - ConstKind::Value(ConstValue::ByRef { alloc, offset }) if offset.bytes() == 0 => alloc, - _ => bug!("static const eval returned {:#?}", const_), - }; + let alloc = tcx.eval_static_initializer(def_id).unwrap(); - let data_id = data_id_for_static( - tcx, - module, - def_id, - if tcx.is_reachable_non_generic(def_id) { - Linkage::Export - } else { - Linkage::Local - }, - ); - (data_id, alloc) + let data_id = data_id_for_static(tcx, module, def_id, true); + (data_id, alloc, section_name) } }; @@ -318,12 +385,25 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut Module, cx: &mu } let mut data_ctx = DataContext::new(); - - let mut bytes = alloc.inspect_with_undef_and_ptr_outside_interpreter(0..alloc.len()).to_vec(); - // The machO backend of faerie doesn't align data objects correctly unless we do this. - while bytes.len() as u64 % 16 != 0 { - bytes.push(0xde); + data_ctx.set_align(alloc.align.bytes()); + + if let Some(section_name) = section_name { + let (segment_name, section_name) = if tcx.sess.target.is_like_osx { + if let Some(names) = section_name.split_once(',') { + names + } else { + tcx.sess.fatal(&format!( + "#[link_section = \"{}\"] is not valid for macos target: must be segment and section separated by comma", + section_name + )); + } + } else { + ("", &*section_name) + }; + data_ctx.set_segment_section(segment_name, section_name); } + + let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec(); data_ctx.define(bytes.into_boxed_slice()); for &(offset, (_tag, reloc)) in alloc.relocations().iter() { @@ -331,15 +411,13 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut Module, cx: &mu let endianness = tcx.data_layout.endian; let offset = offset.bytes() as usize; let ptr_size = tcx.data_layout.pointer_size; - let bytes = &alloc.inspect_with_undef_and_ptr_outside_interpreter(offset..offset + ptr_size.bytes() as usize); + let bytes = &alloc.inspect_with_uninit_and_ptr_outside_interpreter( + offset..offset + ptr_size.bytes() as usize, + ); read_target_uint(endianness, bytes).unwrap() }; - // Don't inline `reloc_target_alloc` into the match. That would cause `tcx.alloc_map` - // to be locked for the duration of the match. `data_id_for_static` however may try - // to lock `tcx.alloc_map` itself while calculating the layout of the target static. - // This would cause a panic in single threaded rustc and a deadlock for parallel rustc. - let reloc_target_alloc = tcx.alloc_map.lock().get(reloc).unwrap(); + let reloc_target_alloc = tcx.get_global_alloc(reloc).unwrap(); let data_id = match reloc_target_alloc { GlobalAlloc::Function(instance) => { assert_eq!(addend, 0); @@ -348,20 +426,22 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut Module, cx: &mu data_ctx.write_function_addr(offset.bytes() as u32, local_func_id); continue; } - GlobalAlloc::Memory(_) => { - cx.todo.insert(TodoItem::Alloc(reloc)); - data_id_for_alloc_id(module, reloc, alloc.align) + GlobalAlloc::Memory(target_alloc) => { + data_id_for_alloc_id(cx, module, reloc, target_alloc.mutability) } GlobalAlloc::Static(def_id) => { + if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) + { + tcx.sess.fatal(&format!( + "Allocation {:?} contains reference to TLS value {:?}", + alloc, def_id + )); + } + // Don't push a `TodoItem::Static` here, as it will cause statics used by // multiple crates to be duplicated between them. It isn't necessary anyway, // as it will get pushed by `codegen_static` when necessary. - data_id_for_static( - tcx, - module, - def_id, - crate::linkage::get_static_ref_linkage(tcx, def_id), - ) + data_id_for_static(tcx, module, def_id, false) } }; @@ -376,136 +456,94 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut Module, cx: &mu assert!(cx.todo.is_empty(), "{:?}", cx.todo); } -fn pop_set(set: &mut HashSet) -> Option { - if let Some(elem) = set.iter().next().map(|elem| *elem) { - set.remove(&elem); - Some(elem) - } else { - None - } -} - -struct TransPlaceInterpreter; - -impl<'mir, 'tcx> Machine<'mir, 'tcx> for TransPlaceInterpreter { - type MemoryKinds = !; - type ExtraFnVal = !; - type PointerTag = (); - type AllocExtra = (); - type MemoryExtra = (); - type FrameExtra = (); - type MemoryMap = FxHashMap, Allocation<()>)>; - - const CHECK_ALIGN: bool = true; - const STATIC_KIND: Option = None; - - fn enforce_validity(_: &InterpCx<'mir, 'tcx, Self>) -> bool { - false - } - - fn before_terminator(_: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> { - panic!(); - } - - fn find_fn( - _: &mut InterpCx<'mir, 'tcx, Self>, - _: Instance<'tcx>, - _: &[OpTy<'tcx>], - _: Option<(PlaceTy<'tcx>, BasicBlock)>, - _: Option, - ) -> InterpResult<'tcx, Option<&'mir Body<'tcx>>> { - panic!(); - } - - fn call_intrinsic( - _: &mut InterpCx<'mir, 'tcx, Self>, - _: Span, - _: Instance<'tcx>, - _: &[OpTy<'tcx>], - _: Option<(PlaceTy<'tcx>, BasicBlock)>, - _: Option, - ) -> InterpResult<'tcx> { - panic!(); - } - - fn find_foreign_static(_: TyCtxt<'tcx>, _: DefId) -> InterpResult<'tcx, Cow<'tcx, Allocation>> { - panic!(); - } - - fn binary_ptr_op( - _: &InterpCx<'mir, 'tcx, Self>, - _: mir::BinOp, - _: ImmTy<'tcx>, - _: ImmTy<'tcx>, - ) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> { - panic!(); - } - - fn ptr_to_int(_: &Memory<'mir, 'tcx, Self>, _: Pointer<()>) -> InterpResult<'tcx, u64> { - panic!(); - } - - fn box_alloc(_: &mut InterpCx<'mir, 'tcx, Self>, _: PlaceTy<'tcx>) -> InterpResult<'tcx> { - panic!(); - } - - fn tag_allocation<'b>( - _: &(), - _: AllocId, - alloc: Cow<'b, Allocation>, - _: Option>, - ) -> (Cow<'b, Allocation<(), ()>>, ()) { - (alloc, ()) - } - - fn tag_static_base_pointer(_: &(), _: AllocId) -> Self::PointerTag { - () - } - - fn call_extra_fn( - _: &mut InterpCx<'mir, 'tcx, Self>, - _: !, - _: &[OpTy<'tcx, ()>], - _: Option<(PlaceTy<'tcx, ()>, BasicBlock)>, - _: Option, - ) -> InterpResult<'tcx> { - unreachable!(); - } - - fn stack_push(_: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> { - Ok(()) - } - - fn stack_pop(_: &mut InterpCx<'mir, 'tcx, Self>, _: (), _: bool) -> InterpResult<'tcx, StackPopInfo> { - Ok(StackPopInfo::Normal) - } -} - -pub fn mir_operand_get_const_val<'tcx>( - fx: &FunctionCx<'_, 'tcx, impl Backend>, +pub(crate) fn mir_operand_get_const_val<'tcx>( + fx: &FunctionCx<'_, '_, 'tcx>, operand: &Operand<'tcx>, -) -> Option<&'tcx Const<'tcx>> { - let place = match operand { - Operand::Copy(place) | Operand::Move(place) => place, - Operand::Constant(const_) => return Some(force_eval_const(fx, const_.literal)), - }; - - assert!(place.projection.is_empty()); - let static_ = match &place.base { - PlaceBase::Static(static_) => static_, - PlaceBase::Local(_) => return None, - }; - - Some(match &static_.kind { - StaticKind::Static => unimplemented!(), - StaticKind::Promoted(promoted, substs) => { - let instance = Instance::new(static_.def_id, fx.monomorphize(substs)); - fx.tcx - .const_eval(ParamEnv::reveal_all().and(GlobalId { - instance, - promoted: Some(*promoted), - })) - .unwrap() +) -> Option> { + match operand { + Operand::Constant(const_) => match const_.literal { + ConstantKind::Ty(const_) => { + fx.monomorphize(const_).eval(fx.tcx, ParamEnv::reveal_all()).val.try_to_value() + } + ConstantKind::Val(val, _) => Some(val), + }, + // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored + // inside a temporary before being passed to the intrinsic requiring the const argument. + // This code tries to find a single constant defining definition of the referenced local. + Operand::Copy(place) | Operand::Move(place) => { + if !place.projection.is_empty() { + return None; + } + let mut computed_const_val = None; + for bb_data in fx.mir.basic_blocks() { + for stmt in &bb_data.statements { + match &stmt.kind { + StatementKind::Assign(local_and_rvalue) if &local_and_rvalue.0 == place => { + match &local_and_rvalue.1 { + Rvalue::Cast(CastKind::Misc, operand, ty) => { + if computed_const_val.is_some() { + return None; // local assigned twice + } + if !matches!(ty.kind(), ty::Uint(_) | ty::Int(_)) { + return None; + } + let const_val = mir_operand_get_const_val(fx, operand)?; + if fx.layout_of(ty).size + != const_val.try_to_scalar_int()?.size() + { + return None; + } + computed_const_val = Some(const_val); + } + Rvalue::Use(operand) => { + computed_const_val = mir_operand_get_const_val(fx, operand) + } + _ => return None, + } + } + StatementKind::SetDiscriminant { place: stmt_place, variant_index: _ } + if &**stmt_place == place => + { + return None; + } + StatementKind::LlvmInlineAsm(_) | StatementKind::CopyNonOverlapping(_) => { + return None; + } // conservative handling + StatementKind::Assign(_) + | StatementKind::FakeRead(_) + | StatementKind::SetDiscriminant { .. } + | StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Retag(_, _) + | StatementKind::AscribeUserType(_, _) + | StatementKind::Coverage(_) + | StatementKind::Nop => {} + } + } + match &bb_data.terminator().kind { + TerminatorKind::Goto { .. } + | TerminatorKind::SwitchInt { .. } + | TerminatorKind::Resume + | TerminatorKind::Abort + | TerminatorKind::Return + | TerminatorKind::Unreachable + | TerminatorKind::Drop { .. } + | TerminatorKind::Assert { .. } => {} + TerminatorKind::DropAndReplace { .. } + | TerminatorKind::Yield { .. } + | TerminatorKind::GeneratorDrop + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } => unreachable!(), + TerminatorKind::InlineAsm { .. } => return None, + TerminatorKind::Call { destination: Some((call_place, _)), .. } + if call_place == place => + { + return None; + } + TerminatorKind::Call { .. } => {} + } + } + computed_const_val } - }) + } }