--- /dev/null
- use rustc_codegen_ssa::base::to_immediate;
+use crate::common::Funclet;
+use crate::context::CodegenCx;
+use crate::llvm::{self, BasicBlock, False};
+use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope};
+use crate::type_::Type;
+use crate::type_of::LayoutLlvmExt;
+use crate::value::Value;
+use libc::{c_char, c_uint};
- OperandValue::Immediate(to_immediate(self, llval, place.layout))
+use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, TypeKind};
+use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
+use rustc_codegen_ssa::mir::place::PlaceRef;
+use rustc_codegen_ssa::traits::*;
+use rustc_codegen_ssa::MemFlags;
+use rustc_data_structures::const_cstr;
+use rustc_data_structures::small_c_str::SmallCStr;
+use rustc_hir::def_id::DefId;
+use rustc_middle::ty::layout::TyAndLayout;
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::sym;
+use rustc_target::abi::{self, Align, Size};
+use rustc_target::spec::{HasTargetSpec, Target};
+use std::borrow::Cow;
+use std::ffi::CStr;
+use std::iter::TrustedLen;
+use std::ops::{Deref, Range};
+use std::ptr;
+use tracing::debug;
+
+// All Builders must have an llfn associated with them
+#[must_use]
+pub struct Builder<'a, 'll, 'tcx> {
+ pub llbuilder: &'ll mut llvm::Builder<'ll>,
+ pub cx: &'a CodegenCx<'ll, 'tcx>,
+}
+
+impl Drop for Builder<'a, 'll, 'tcx> {
+ fn drop(&mut self) {
+ unsafe {
+ llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
+ }
+ }
+}
+
+// FIXME(eddyb) use a checked constructor when they become `const fn`.
+const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") };
+
+/// Empty string, to be used where LLVM expects an instruction name, indicating
+/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
+// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
+const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr();
+
+impl BackendTypes for Builder<'_, 'll, 'tcx> {
+ type Value = <CodegenCx<'ll, 'tcx> as BackendTypes>::Value;
+ type Function = <CodegenCx<'ll, 'tcx> as BackendTypes>::Function;
+ type BasicBlock = <CodegenCx<'ll, 'tcx> as BackendTypes>::BasicBlock;
+ type Type = <CodegenCx<'ll, 'tcx> as BackendTypes>::Type;
+ type Funclet = <CodegenCx<'ll, 'tcx> as BackendTypes>::Funclet;
+
+ type DIScope = <CodegenCx<'ll, 'tcx> as BackendTypes>::DIScope;
+ type DIVariable = <CodegenCx<'ll, 'tcx> as BackendTypes>::DIVariable;
+}
+
+impl abi::HasDataLayout for Builder<'_, '_, '_> {
+ fn data_layout(&self) -> &abi::TargetDataLayout {
+ self.cx.data_layout()
+ }
+}
+
+impl ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
+ fn tcx(&self) -> TyCtxt<'tcx> {
+ self.cx.tcx
+ }
+}
+
+impl ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> {
+ fn param_env(&self) -> ty::ParamEnv<'tcx> {
+ self.cx.param_env()
+ }
+}
+
+impl HasTargetSpec for Builder<'_, '_, 'tcx> {
+ fn target_spec(&self) -> &Target {
+ &self.cx.target_spec()
+ }
+}
+
+impl abi::LayoutOf for Builder<'_, '_, 'tcx> {
+ type Ty = Ty<'tcx>;
+ type TyAndLayout = TyAndLayout<'tcx>;
+
+ fn layout_of(&self, ty: Ty<'tcx>) -> Self::TyAndLayout {
+ self.cx.layout_of(ty)
+ }
+}
+
+impl Deref for Builder<'_, 'll, 'tcx> {
+ type Target = CodegenCx<'ll, 'tcx>;
+
+ fn deref(&self) -> &Self::Target {
+ self.cx
+ }
+}
+
+impl HasCodegen<'tcx> for Builder<'_, 'll, 'tcx> {
+ type CodegenCx = CodegenCx<'ll, 'tcx>;
+}
+
+macro_rules! builder_methods_for_value_instructions {
+ ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
+ $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
+ unsafe {
+ llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
+ }
+ })+
+ }
+}
+
+impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
+ fn new_block<'b>(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &'b str) -> Self {
+ let mut bx = Builder::with_cx(cx);
+ let llbb = unsafe {
+ let name = SmallCStr::new(name);
+ llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
+ };
+ bx.position_at_end(llbb);
+ bx
+ }
+
+ fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self {
+ // Create a fresh builder from the crate context.
+ let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) };
+ Builder { llbuilder, cx }
+ }
+
+ fn build_sibling_block(&self, name: &str) -> Self {
+ Builder::new_block(self.cx, self.llfn(), name)
+ }
+
+ fn llbb(&self) -> &'ll BasicBlock {
+ unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
+ }
+
+ fn position_at_end(&mut self, llbb: &'ll BasicBlock) {
+ unsafe {
+ llvm::LLVMPositionBuilderAtEnd(self.llbuilder, llbb);
+ }
+ }
+
+ fn ret_void(&mut self) {
+ unsafe {
+ llvm::LLVMBuildRetVoid(self.llbuilder);
+ }
+ }
+
+ fn ret(&mut self, v: &'ll Value) {
+ unsafe {
+ llvm::LLVMBuildRet(self.llbuilder, v);
+ }
+ }
+
+ fn br(&mut self, dest: &'ll BasicBlock) {
+ unsafe {
+ llvm::LLVMBuildBr(self.llbuilder, dest);
+ }
+ }
+
+ fn cond_br(
+ &mut self,
+ cond: &'ll Value,
+ then_llbb: &'ll BasicBlock,
+ else_llbb: &'ll BasicBlock,
+ ) {
+ unsafe {
+ llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
+ }
+ }
+
+ fn switch(
+ &mut self,
+ v: &'ll Value,
+ else_llbb: &'ll BasicBlock,
+ cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)> + TrustedLen,
+ ) {
+ let switch =
+ unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
+ for (on_val, dest) in cases {
+ let on_val = self.const_uint_big(self.val_ty(v), on_val);
+ unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
+ }
+ }
+
+ fn invoke(
+ &mut self,
+ llfn: &'ll Value,
+ args: &[&'ll Value],
+ then: &'ll BasicBlock,
+ catch: &'ll BasicBlock,
+ funclet: Option<&Funclet<'ll>>,
+ ) -> &'ll Value {
+ debug!("invoke {:?} with args ({:?})", llfn, args);
+
+ let args = self.check_call("invoke", llfn, args);
+ let bundle = funclet.map(|funclet| funclet.bundle());
+ let bundle = bundle.as_ref().map(|b| &*b.raw);
+
+ unsafe {
+ llvm::LLVMRustBuildInvoke(
+ self.llbuilder,
+ llfn,
+ args.as_ptr(),
+ args.len() as c_uint,
+ then,
+ catch,
+ bundle,
+ UNNAMED,
+ )
+ }
+ }
+
+ fn unreachable(&mut self) {
+ unsafe {
+ llvm::LLVMBuildUnreachable(self.llbuilder);
+ }
+ }
+
+ builder_methods_for_value_instructions! {
+ add(a, b) => LLVMBuildAdd,
+ fadd(a, b) => LLVMBuildFAdd,
+ sub(a, b) => LLVMBuildSub,
+ fsub(a, b) => LLVMBuildFSub,
+ mul(a, b) => LLVMBuildMul,
+ fmul(a, b) => LLVMBuildFMul,
+ udiv(a, b) => LLVMBuildUDiv,
+ exactudiv(a, b) => LLVMBuildExactUDiv,
+ sdiv(a, b) => LLVMBuildSDiv,
+ exactsdiv(a, b) => LLVMBuildExactSDiv,
+ fdiv(a, b) => LLVMBuildFDiv,
+ urem(a, b) => LLVMBuildURem,
+ srem(a, b) => LLVMBuildSRem,
+ frem(a, b) => LLVMBuildFRem,
+ shl(a, b) => LLVMBuildShl,
+ lshr(a, b) => LLVMBuildLShr,
+ ashr(a, b) => LLVMBuildAShr,
+ and(a, b) => LLVMBuildAnd,
+ or(a, b) => LLVMBuildOr,
+ xor(a, b) => LLVMBuildXor,
+ neg(x) => LLVMBuildNeg,
+ fneg(x) => LLVMBuildFNeg,
+ not(x) => LLVMBuildNot,
+ unchecked_sadd(x, y) => LLVMBuildNSWAdd,
+ unchecked_uadd(x, y) => LLVMBuildNUWAdd,
+ unchecked_ssub(x, y) => LLVMBuildNSWSub,
+ unchecked_usub(x, y) => LLVMBuildNUWSub,
+ unchecked_smul(x, y) => LLVMBuildNSWMul,
+ unchecked_umul(x, y) => LLVMBuildNUWMul,
+ }
+
+ fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+ unsafe {
+ let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
+ llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+ instr
+ }
+ }
+
+ fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+ unsafe {
+ let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
+ llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+ instr
+ }
+ }
+
+ fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+ unsafe {
+ let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
+ llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+ instr
+ }
+ }
+
+ fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+ unsafe {
+ let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
+ llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+ instr
+ }
+ }
+
+ fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+ unsafe {
+ let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
+ llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+ instr
+ }
+ }
+
+ fn checked_binop(
+ &mut self,
+ oop: OverflowOp,
+ ty: Ty<'_>,
+ lhs: Self::Value,
+ rhs: Self::Value,
+ ) -> (Self::Value, Self::Value) {
+ use rustc_ast::IntTy::*;
+ use rustc_ast::UintTy::*;
+ use rustc_middle::ty::{Int, Uint};
+
+ let new_kind = match ty.kind {
+ Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.ptr_width)),
+ Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.ptr_width)),
+ ref t @ (Uint(_) | Int(_)) => t.clone(),
+ _ => panic!("tried to get overflow intrinsic for op applied to non-int type"),
+ };
+
+ let name = match oop {
+ OverflowOp::Add => match new_kind {
+ Int(I8) => "llvm.sadd.with.overflow.i8",
+ Int(I16) => "llvm.sadd.with.overflow.i16",
+ Int(I32) => "llvm.sadd.with.overflow.i32",
+ Int(I64) => "llvm.sadd.with.overflow.i64",
+ Int(I128) => "llvm.sadd.with.overflow.i128",
+
+ Uint(U8) => "llvm.uadd.with.overflow.i8",
+ Uint(U16) => "llvm.uadd.with.overflow.i16",
+ Uint(U32) => "llvm.uadd.with.overflow.i32",
+ Uint(U64) => "llvm.uadd.with.overflow.i64",
+ Uint(U128) => "llvm.uadd.with.overflow.i128",
+
+ _ => unreachable!(),
+ },
+ OverflowOp::Sub => match new_kind {
+ Int(I8) => "llvm.ssub.with.overflow.i8",
+ Int(I16) => "llvm.ssub.with.overflow.i16",
+ Int(I32) => "llvm.ssub.with.overflow.i32",
+ Int(I64) => "llvm.ssub.with.overflow.i64",
+ Int(I128) => "llvm.ssub.with.overflow.i128",
+
+ Uint(U8) => "llvm.usub.with.overflow.i8",
+ Uint(U16) => "llvm.usub.with.overflow.i16",
+ Uint(U32) => "llvm.usub.with.overflow.i32",
+ Uint(U64) => "llvm.usub.with.overflow.i64",
+ Uint(U128) => "llvm.usub.with.overflow.i128",
+
+ _ => unreachable!(),
+ },
+ OverflowOp::Mul => match new_kind {
+ Int(I8) => "llvm.smul.with.overflow.i8",
+ Int(I16) => "llvm.smul.with.overflow.i16",
+ Int(I32) => "llvm.smul.with.overflow.i32",
+ Int(I64) => "llvm.smul.with.overflow.i64",
+ Int(I128) => "llvm.smul.with.overflow.i128",
+
+ Uint(U8) => "llvm.umul.with.overflow.i8",
+ Uint(U16) => "llvm.umul.with.overflow.i16",
+ Uint(U32) => "llvm.umul.with.overflow.i32",
+ Uint(U64) => "llvm.umul.with.overflow.i64",
+ Uint(U128) => "llvm.umul.with.overflow.i128",
+
+ _ => unreachable!(),
+ },
+ };
+
+ let intrinsic = self.get_intrinsic(&name);
+ let res = self.call(intrinsic, &[lhs, rhs], None);
+ (self.extract_value(res, 0), self.extract_value(res, 1))
+ }
+
++ fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
++ if self.cx().val_ty(val) == self.cx().type_i1() {
++ self.zext(val, self.cx().type_i8())
++ } else {
++ val
++ }
++ }
++ fn to_immediate_scalar(&mut self, val: Self::Value, scalar: &abi::Scalar) -> Self::Value {
++ if scalar.is_bool() {
++ return self.trunc(val, self.cx().type_i1());
++ }
++ val
++ }
++
+ fn alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value {
+ let mut bx = Builder::with_cx(self.cx);
+ bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
+ bx.dynamic_alloca(ty, align)
+ }
+
+ fn dynamic_alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value {
+ unsafe {
+ let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED);
+ llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
+ alloca
+ }
+ }
+
+ fn array_alloca(&mut self, ty: &'ll Type, len: &'ll Value, align: Align) -> &'ll Value {
+ unsafe {
+ let alloca = llvm::LLVMBuildArrayAlloca(self.llbuilder, ty, len, UNNAMED);
+ llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
+ alloca
+ }
+ }
+
+ fn load(&mut self, ptr: &'ll Value, align: Align) -> &'ll Value {
+ unsafe {
+ let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
+ llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
+ load
+ }
+ }
+
+ fn volatile_load(&mut self, ptr: &'ll Value) -> &'ll Value {
+ unsafe {
+ let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
+ llvm::LLVMSetVolatile(load, llvm::True);
+ load
+ }
+ }
+
+ fn atomic_load(
+ &mut self,
+ ptr: &'ll Value,
+ order: rustc_codegen_ssa::common::AtomicOrdering,
+ size: Size,
+ ) -> &'ll Value {
+ unsafe {
+ let load = llvm::LLVMRustBuildAtomicLoad(
+ self.llbuilder,
+ ptr,
+ UNNAMED,
+ AtomicOrdering::from_generic(order),
+ );
+ // LLVM requires the alignment of atomic loads to be at least the size of the type.
+ llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
+ load
+ }
+ }
+
+ fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
+ debug!("PlaceRef::load: {:?}", place);
+
+ assert_eq!(place.llextra.is_some(), place.layout.is_unsized());
+
+ if place.layout.is_zst() {
+ return OperandRef::new_zst(self, place.layout);
+ }
+
+ fn scalar_load_metadata<'a, 'll, 'tcx>(
+ bx: &mut Builder<'a, 'll, 'tcx>,
+ load: &'ll Value,
+ scalar: &abi::Scalar,
+ ) {
+ let vr = scalar.valid_range.clone();
+ match scalar.value {
+ abi::Int(..) => {
+ let range = scalar.valid_range_exclusive(bx);
+ if range.start != range.end {
+ bx.range_metadata(load, range);
+ }
+ }
+ abi::Pointer if vr.start() < vr.end() && !vr.contains(&0) => {
+ bx.nonnull_metadata(load);
+ }
+ _ => {}
+ }
+ }
+
+ let val = if let Some(llextra) = place.llextra {
+ OperandValue::Ref(place.llval, Some(llextra), place.align)
+ } else if place.layout.is_llvm_immediate() {
+ let mut const_llval = None;
+ unsafe {
+ if let Some(global) = llvm::LLVMIsAGlobalVariable(place.llval) {
+ if llvm::LLVMIsGlobalConstant(global) == llvm::True {
+ const_llval = llvm::LLVMGetInitializer(global);
+ }
+ }
+ }
+ let llval = const_llval.unwrap_or_else(|| {
+ let load = self.load(place.llval, place.align);
+ if let abi::Abi::Scalar(ref scalar) = place.layout.abi {
+ scalar_load_metadata(self, load, scalar);
+ }
+ load
+ });
- if scalar.is_bool() { self.trunc(load, self.type_i1()) } else { load }
++ OperandValue::Immediate(self.to_immediate(llval, place.layout))
+ } else if let abi::Abi::ScalarPair(ref a, ref b) = place.layout.abi {
+ let b_offset = a.value.size(self).align_to(b.value.align(self).abi);
+
+ let mut load = |i, scalar: &abi::Scalar, align| {
+ let llptr = self.struct_gep(place.llval, i as u64);
+ let load = self.load(llptr, align);
+ scalar_load_metadata(self, load, scalar);
++ self.to_immediate_scalar(load, scalar)
+ };
+
+ OperandValue::Pair(
+ load(0, a, place.align),
+ load(1, b, place.align.restrict_for_offset(b_offset)),
+ )
+ } else {
+ OperandValue::Ref(place.llval, None, place.align)
+ };
+
+ OperandRef { val, layout: place.layout }
+ }
+
+ fn write_operand_repeatedly(
+ mut self,
+ cg_elem: OperandRef<'tcx, &'ll Value>,
+ count: u64,
+ dest: PlaceRef<'tcx, &'ll Value>,
+ ) -> Self {
+ let zero = self.const_usize(0);
+ let count = self.const_usize(count);
+ let start = dest.project_index(&mut self, zero).llval;
+ let end = dest.project_index(&mut self, count).llval;
+
+ let mut header_bx = self.build_sibling_block("repeat_loop_header");
+ let mut body_bx = self.build_sibling_block("repeat_loop_body");
+ let next_bx = self.build_sibling_block("repeat_loop_next");
+
+ self.br(header_bx.llbb());
+ let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]);
+
+ let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end);
+ header_bx.cond_br(keep_going, body_bx.llbb(), next_bx.llbb());
+
+ let align = dest.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size);
+ cg_elem
+ .val
+ .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align));
+
+ let next = body_bx.inbounds_gep(current, &[self.const_usize(1)]);
+ body_bx.br(header_bx.llbb());
+ header_bx.add_incoming_to_phi(current, next, body_bx.llbb());
+
+ next_bx
+ }
+
+ fn range_metadata(&mut self, load: &'ll Value, range: Range<u128>) {
+ if self.sess().target.target.arch == "amdgpu" {
+ // amdgpu/LLVM does something weird and thinks a i64 value is
+ // split into a v2i32, halving the bitwidth LLVM expects,
+ // tripping an assertion. So, for now, just disable this
+ // optimization.
+ return;
+ }
+
+ unsafe {
+ let llty = self.cx.val_ty(load);
+ let v = [
+ self.cx.const_uint_big(llty, range.start),
+ self.cx.const_uint_big(llty, range.end),
+ ];
+
+ llvm::LLVMSetMetadata(
+ load,
+ llvm::MD_range as c_uint,
+ llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint),
+ );
+ }
+ }
+
+ fn nonnull_metadata(&mut self, load: &'ll Value) {
+ unsafe {
+ llvm::LLVMSetMetadata(
+ load,
+ llvm::MD_nonnull as c_uint,
+ llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0),
+ );
+ }
+ }
+
+ fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
+ self.store_with_flags(val, ptr, align, MemFlags::empty())
+ }
+
+ fn store_with_flags(
+ &mut self,
+ val: &'ll Value,
+ ptr: &'ll Value,
+ align: Align,
+ flags: MemFlags,
+ ) -> &'ll Value {
+ debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
+ let ptr = self.check_store(val, ptr);
+ unsafe {
+ let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
+ let align =
+ if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
+ llvm::LLVMSetAlignment(store, align);
+ if flags.contains(MemFlags::VOLATILE) {
+ llvm::LLVMSetVolatile(store, llvm::True);
+ }
+ if flags.contains(MemFlags::NONTEMPORAL) {
+ // According to LLVM [1] building a nontemporal store must
+ // *always* point to a metadata value of the integer 1.
+ //
+ // [1]: http://llvm.org/docs/LangRef.html#store-instruction
+ let one = self.cx.const_i32(1);
+ let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1);
+ llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node);
+ }
+ store
+ }
+ }
+
+ fn atomic_store(
+ &mut self,
+ val: &'ll Value,
+ ptr: &'ll Value,
+ order: rustc_codegen_ssa::common::AtomicOrdering,
+ size: Size,
+ ) {
+ debug!("Store {:?} -> {:?}", val, ptr);
+ let ptr = self.check_store(val, ptr);
+ unsafe {
+ let store = llvm::LLVMRustBuildAtomicStore(
+ self.llbuilder,
+ val,
+ ptr,
+ AtomicOrdering::from_generic(order),
+ );
+ // LLVM requires the alignment of atomic stores to be at least the size of the type.
+ llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
+ }
+ }
+
+ fn gep(&mut self, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
+ unsafe {
+ llvm::LLVMBuildGEP(
+ self.llbuilder,
+ ptr,
+ indices.as_ptr(),
+ indices.len() as c_uint,
+ UNNAMED,
+ )
+ }
+ }
+
+ fn inbounds_gep(&mut self, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
+ unsafe {
+ llvm::LLVMBuildInBoundsGEP(
+ self.llbuilder,
+ ptr,
+ indices.as_ptr(),
+ indices.len() as c_uint,
+ UNNAMED,
+ )
+ }
+ }
+
+ fn struct_gep(&mut self, ptr: &'ll Value, idx: u64) -> &'ll Value {
+ assert_eq!(idx as c_uint as u64, idx);
+ unsafe { llvm::LLVMBuildStructGEP(self.llbuilder, ptr, idx as c_uint, UNNAMED) }
+ }
+
+ /* Casts */
+ fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> {
+ // WebAssembly has saturating floating point to integer casts if the
+ // `nontrapping-fptoint` target feature is activated. We'll use those if
+ // they are available.
+ if self.sess().target.target.arch == "wasm32"
+ && self.sess().target_features.contains(&sym::nontrapping_dash_fptoint)
+ {
+ let src_ty = self.cx.val_ty(val);
+ let float_width = self.cx.float_width(src_ty);
+ let int_width = self.cx.int_width(dest_ty);
+ let name = match (int_width, float_width) {
+ (32, 32) => Some("llvm.wasm.trunc.saturate.unsigned.i32.f32"),
+ (32, 64) => Some("llvm.wasm.trunc.saturate.unsigned.i32.f64"),
+ (64, 32) => Some("llvm.wasm.trunc.saturate.unsigned.i64.f32"),
+ (64, 64) => Some("llvm.wasm.trunc.saturate.unsigned.i64.f64"),
+ _ => None,
+ };
+ if let Some(name) = name {
+ let intrinsic = self.get_intrinsic(name);
+ return Some(self.call(intrinsic, &[val], None));
+ }
+ }
+ None
+ }
+
+ fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> {
+ // WebAssembly has saturating floating point to integer casts if the
+ // `nontrapping-fptoint` target feature is activated. We'll use those if
+ // they are available.
+ if self.sess().target.target.arch == "wasm32"
+ && self.sess().target_features.contains(&sym::nontrapping_dash_fptoint)
+ {
+ let src_ty = self.cx.val_ty(val);
+ let float_width = self.cx.float_width(src_ty);
+ let int_width = self.cx.int_width(dest_ty);
+ let name = match (int_width, float_width) {
+ (32, 32) => Some("llvm.wasm.trunc.saturate.signed.i32.f32"),
+ (32, 64) => Some("llvm.wasm.trunc.saturate.signed.i32.f64"),
+ (64, 32) => Some("llvm.wasm.trunc.saturate.signed.i64.f32"),
+ (64, 64) => Some("llvm.wasm.trunc.saturate.signed.i64.f64"),
+ _ => None,
+ };
+ if let Some(name) = name {
+ let intrinsic = self.get_intrinsic(name);
+ return Some(self.call(intrinsic, &[val], None));
+ }
+ }
+ None
+ }
+
+ fn fptosui_may_trap(&self, val: &'ll Value, dest_ty: &'ll Type) -> bool {
+ // Most of the time we'll be generating the `fptosi` or `fptoui`
+ // instruction for floating-point-to-integer conversions. These
+ // instructions by definition in LLVM do not trap. For the WebAssembly
+ // target, however, we'll lower in some cases to intrinsic calls instead
+ // which may trap. If we detect that this is a situation where we'll be
+ // using the intrinsics then we report that the call map trap, which
+ // callers might need to handle.
+ if !self.wasm_and_missing_nontrapping_fptoint() {
+ return false;
+ }
+ let src_ty = self.cx.val_ty(val);
+ let float_width = self.cx.float_width(src_ty);
+ let int_width = self.cx.int_width(dest_ty);
+ match (int_width, float_width) {
+ (32, 32) | (32, 64) | (64, 32) | (64, 64) => true,
+ _ => false,
+ }
+ }
+
+ fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ // When we can, use the native wasm intrinsics which have tighter
+ // codegen. Note that this has a semantic difference in that the
+ // intrinsic can trap whereas `fptoui` never traps. That difference,
+ // however, is handled by `fptosui_may_trap` above.
+ //
+ // Note that we skip the wasm intrinsics for vector types where `fptoui`
+ // must be used instead.
+ if self.wasm_and_missing_nontrapping_fptoint() {
+ let src_ty = self.cx.val_ty(val);
+ if self.cx.type_kind(src_ty) != TypeKind::Vector {
+ let float_width = self.cx.float_width(src_ty);
+ let int_width = self.cx.int_width(dest_ty);
+ let name = match (int_width, float_width) {
+ (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"),
+ (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"),
+ (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"),
+ (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"),
+ _ => None,
+ };
+ if let Some(name) = name {
+ let intrinsic = self.get_intrinsic(name);
+ return self.call(intrinsic, &[val], None);
+ }
+ }
+ }
+ unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ if self.wasm_and_missing_nontrapping_fptoint() {
+ let src_ty = self.cx.val_ty(val);
+ if self.cx.type_kind(src_ty) != TypeKind::Vector {
+ let float_width = self.cx.float_width(src_ty);
+ let int_width = self.cx.int_width(dest_ty);
+ let name = match (int_width, float_width) {
+ (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"),
+ (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"),
+ (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"),
+ (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"),
+ _ => None,
+ };
+ if let Some(name) = name {
+ let intrinsic = self.get_intrinsic(name);
+ return self.call(intrinsic, &[val], None);
+ }
+ }
+ }
+ unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) }
+ }
+
+ fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ /* Comparisons */
+ fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+ let op = llvm::IntPredicate::from_generic(op);
+ unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
+ }
+
+ fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
+ }
+
+ /* Miscellaneous instructions */
+ fn memcpy(
+ &mut self,
+ dst: &'ll Value,
+ dst_align: Align,
+ src: &'ll Value,
+ src_align: Align,
+ size: &'ll Value,
+ flags: MemFlags,
+ ) {
+ if flags.contains(MemFlags::NONTEMPORAL) {
+ // HACK(nox): This is inefficient but there is no nontemporal memcpy.
+ let val = self.load(src, src_align);
+ let ptr = self.pointercast(dst, self.type_ptr_to(self.val_ty(val)));
+ self.store_with_flags(val, ptr, dst_align, flags);
+ return;
+ }
+ let size = self.intcast(size, self.type_isize(), false);
+ let is_volatile = flags.contains(MemFlags::VOLATILE);
+ let dst = self.pointercast(dst, self.type_i8p());
+ let src = self.pointercast(src, self.type_i8p());
+ unsafe {
+ llvm::LLVMRustBuildMemCpy(
+ self.llbuilder,
+ dst,
+ dst_align.bytes() as c_uint,
+ src,
+ src_align.bytes() as c_uint,
+ size,
+ is_volatile,
+ );
+ }
+ }
+
+ fn memmove(
+ &mut self,
+ dst: &'ll Value,
+ dst_align: Align,
+ src: &'ll Value,
+ src_align: Align,
+ size: &'ll Value,
+ flags: MemFlags,
+ ) {
+ if flags.contains(MemFlags::NONTEMPORAL) {
+ // HACK(nox): This is inefficient but there is no nontemporal memmove.
+ let val = self.load(src, src_align);
+ let ptr = self.pointercast(dst, self.type_ptr_to(self.val_ty(val)));
+ self.store_with_flags(val, ptr, dst_align, flags);
+ return;
+ }
+ let size = self.intcast(size, self.type_isize(), false);
+ let is_volatile = flags.contains(MemFlags::VOLATILE);
+ let dst = self.pointercast(dst, self.type_i8p());
+ let src = self.pointercast(src, self.type_i8p());
+ unsafe {
+ llvm::LLVMRustBuildMemMove(
+ self.llbuilder,
+ dst,
+ dst_align.bytes() as c_uint,
+ src,
+ src_align.bytes() as c_uint,
+ size,
+ is_volatile,
+ );
+ }
+ }
+
+ fn memset(
+ &mut self,
+ ptr: &'ll Value,
+ fill_byte: &'ll Value,
+ size: &'ll Value,
+ align: Align,
+ flags: MemFlags,
+ ) {
+ let is_volatile = flags.contains(MemFlags::VOLATILE);
+ let ptr = self.pointercast(ptr, self.type_i8p());
+ unsafe {
+ llvm::LLVMRustBuildMemSet(
+ self.llbuilder,
+ ptr,
+ align.bytes() as c_uint,
+ fill_byte,
+ size,
+ is_volatile,
+ );
+ }
+ }
+
+ fn select(
+ &mut self,
+ cond: &'ll Value,
+ then_val: &'ll Value,
+ else_val: &'ll Value,
+ ) -> &'ll Value {
+ unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
+ }
+
+ #[allow(dead_code)]
+ fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
+ }
+
+ fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
+ }
+
+ fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
+ unsafe {
+ let elt_ty = self.cx.val_ty(elt);
+ let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
+ let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
+ let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
+ self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
+ }
+ }
+
+ fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
+ assert_eq!(idx as c_uint as u64, idx);
+ unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
+ }
+
+ fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
+ assert_eq!(idx as c_uint as u64, idx);
+ unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
+ }
+
+ fn landing_pad(
+ &mut self,
+ ty: &'ll Type,
+ pers_fn: &'ll Value,
+ num_clauses: usize,
+ ) -> &'ll Value {
+ unsafe {
+ llvm::LLVMBuildLandingPad(self.llbuilder, ty, pers_fn, num_clauses as c_uint, UNNAMED)
+ }
+ }
+
+ fn set_cleanup(&mut self, landing_pad: &'ll Value) {
+ unsafe {
+ llvm::LLVMSetCleanup(landing_pad, llvm::True);
+ }
+ }
+
+ fn resume(&mut self, exn: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMBuildResume(self.llbuilder, exn) }
+ }
+
+ fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
+ let name = const_cstr!("cleanuppad");
+ let ret = unsafe {
+ llvm::LLVMRustBuildCleanupPad(
+ self.llbuilder,
+ parent,
+ args.len() as c_uint,
+ args.as_ptr(),
+ name.as_ptr(),
+ )
+ };
+ Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
+ }
+
+ fn cleanup_ret(
+ &mut self,
+ funclet: &Funclet<'ll>,
+ unwind: Option<&'ll BasicBlock>,
+ ) -> &'ll Value {
+ let ret =
+ unsafe { llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) };
+ ret.expect("LLVM does not have support for cleanupret")
+ }
+
+ fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
+ let name = const_cstr!("catchpad");
+ let ret = unsafe {
+ llvm::LLVMRustBuildCatchPad(
+ self.llbuilder,
+ parent,
+ args.len() as c_uint,
+ args.as_ptr(),
+ name.as_ptr(),
+ )
+ };
+ Funclet::new(ret.expect("LLVM does not have support for catchpad"))
+ }
+
+ fn catch_switch(
+ &mut self,
+ parent: Option<&'ll Value>,
+ unwind: Option<&'ll BasicBlock>,
+ num_handlers: usize,
+ ) -> &'ll Value {
+ let name = const_cstr!("catchswitch");
+ let ret = unsafe {
+ llvm::LLVMRustBuildCatchSwitch(
+ self.llbuilder,
+ parent,
+ unwind,
+ num_handlers as c_uint,
+ name.as_ptr(),
+ )
+ };
+ ret.expect("LLVM does not have support for catchswitch")
+ }
+
+ fn add_handler(&mut self, catch_switch: &'ll Value, handler: &'ll BasicBlock) {
+ unsafe {
+ llvm::LLVMRustAddHandler(catch_switch, handler);
+ }
+ }
+
+ fn set_personality_fn(&mut self, personality: &'ll Value) {
+ unsafe {
+ llvm::LLVMSetPersonalityFn(self.llfn(), personality);
+ }
+ }
+
+ // Atomic Operations
+ fn atomic_cmpxchg(
+ &mut self,
+ dst: &'ll Value,
+ cmp: &'ll Value,
+ src: &'ll Value,
+ order: rustc_codegen_ssa::common::AtomicOrdering,
+ failure_order: rustc_codegen_ssa::common::AtomicOrdering,
+ weak: bool,
+ ) -> &'ll Value {
+ let weak = if weak { llvm::True } else { llvm::False };
+ unsafe {
+ llvm::LLVMRustBuildAtomicCmpXchg(
+ self.llbuilder,
+ dst,
+ cmp,
+ src,
+ AtomicOrdering::from_generic(order),
+ AtomicOrdering::from_generic(failure_order),
+ weak,
+ )
+ }
+ }
+ fn atomic_rmw(
+ &mut self,
+ op: rustc_codegen_ssa::common::AtomicRmwBinOp,
+ dst: &'ll Value,
+ src: &'ll Value,
+ order: rustc_codegen_ssa::common::AtomicOrdering,
+ ) -> &'ll Value {
+ unsafe {
+ llvm::LLVMBuildAtomicRMW(
+ self.llbuilder,
+ AtomicRmwBinOp::from_generic(op),
+ dst,
+ src,
+ AtomicOrdering::from_generic(order),
+ False,
+ )
+ }
+ }
+
+ fn atomic_fence(
+ &mut self,
+ order: rustc_codegen_ssa::common::AtomicOrdering,
+ scope: rustc_codegen_ssa::common::SynchronizationScope,
+ ) {
+ unsafe {
+ llvm::LLVMRustBuildAtomicFence(
+ self.llbuilder,
+ AtomicOrdering::from_generic(order),
+ SynchronizationScope::from_generic(scope),
+ );
+ }
+ }
+
+ fn set_invariant_load(&mut self, load: &'ll Value) {
+ unsafe {
+ llvm::LLVMSetMetadata(
+ load,
+ llvm::MD_invariant_load as c_uint,
+ llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0),
+ );
+ }
+ }
+
+ fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
+ self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size);
+ }
+
+ fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
+ self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
+ }
+
+ fn instrprof_increment(
+ &mut self,
+ fn_name: &'ll Value,
+ hash: &'ll Value,
+ num_counters: &'ll Value,
+ index: &'ll Value,
+ ) {
+ debug!(
+ "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})",
+ fn_name, hash, num_counters, index
+ );
+
+ let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) };
+ let args = &[fn_name, hash, num_counters, index];
+ let args = self.check_call("call", llfn, args);
+
+ unsafe {
+ let _ = llvm::LLVMRustBuildCall(
+ self.llbuilder,
+ llfn,
+ args.as_ptr() as *const &llvm::Value,
+ args.len() as c_uint,
+ None,
+ );
+ }
+ }
+
+ fn call(
+ &mut self,
+ llfn: &'ll Value,
+ args: &[&'ll Value],
+ funclet: Option<&Funclet<'ll>>,
+ ) -> &'ll Value {
+ debug!("call {:?} with args ({:?})", llfn, args);
+
+ let args = self.check_call("call", llfn, args);
+ let bundle = funclet.map(|funclet| funclet.bundle());
+ let bundle = bundle.as_ref().map(|b| &*b.raw);
+
+ unsafe {
+ llvm::LLVMRustBuildCall(
+ self.llbuilder,
+ llfn,
+ args.as_ptr() as *const &llvm::Value,
+ args.len() as c_uint,
+ bundle,
+ )
+ }
+ }
+
+ fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
+ }
+
+ fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
+ self.cx
+ }
+
+ unsafe fn delete_basic_block(&mut self, bb: &'ll BasicBlock) {
+ llvm::LLVMDeleteBasicBlock(bb);
+ }
+
+ fn do_not_inline(&mut self, llret: &'ll Value) {
+ llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret);
+ }
+}
+
+impl StaticBuilderMethods for Builder<'a, 'll, 'tcx> {
+ fn get_static(&mut self, def_id: DefId) -> &'ll Value {
+ // Forward to the `get_static` method of `CodegenCx`
+ self.cx().get_static(def_id)
+ }
+}
+
+impl Builder<'a, 'll, 'tcx> {
+ pub fn llfn(&self) -> &'ll Value {
+ unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
+ }
+
+ fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
+ unsafe {
+ llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
+ }
+ }
+
+ pub fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
+ }
+
+ pub fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) }
+ }
+
+ pub fn insert_element(
+ &mut self,
+ vec: &'ll Value,
+ elt: &'ll Value,
+ idx: &'ll Value,
+ ) -> &'ll Value {
+ unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
+ }
+
+ pub fn shuffle_vector(
+ &mut self,
+ v1: &'ll Value,
+ v2: &'ll Value,
+ mask: &'ll Value,
+ ) -> &'ll Value {
+ unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
+ }
+
+ pub fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) }
+ }
+ pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
+ }
+ pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
+ unsafe {
+ let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
+ llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+ instr
+ }
+ }
+ pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
+ unsafe {
+ let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
+ llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+ instr
+ }
+ }
+ pub fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) }
+ }
+ pub fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) }
+ }
+ pub fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) }
+ }
+ pub fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) }
+ }
+ pub fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) }
+ }
+ pub fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
+ unsafe {
+ llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false)
+ }
+ }
+ pub fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
+ unsafe {
+ llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false)
+ }
+ }
+ pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value {
+ unsafe {
+ let instr =
+ llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true);
+ llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+ instr
+ }
+ }
+ pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value {
+ unsafe {
+ let instr =
+ llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true);
+ llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+ instr
+ }
+ }
+ pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) }
+ }
+ pub fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
+ unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) }
+ }
+
+ pub fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
+ unsafe {
+ llvm::LLVMAddClause(landing_pad, clause);
+ }
+ }
+
+ pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value {
+ let ret =
+ unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
+ ret.expect("LLVM does not have support for catchret")
+ }
+
+ fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value {
+ let dest_ptr_ty = self.cx.val_ty(ptr);
+ let stored_ty = self.cx.val_ty(val);
+ let stored_ptr_ty = self.cx.type_ptr_to(stored_ty);
+
+ assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer);
+
+ if dest_ptr_ty == stored_ptr_ty {
+ ptr
+ } else {
+ debug!(
+ "type mismatch in store. \
+ Expected {:?}, got {:?}; inserting bitcast",
+ dest_ptr_ty, stored_ptr_ty
+ );
+ self.bitcast(ptr, stored_ptr_ty)
+ }
+ }
+
+ fn check_call<'b>(
+ &mut self,
+ typ: &str,
+ llfn: &'ll Value,
+ args: &'b [&'ll Value],
+ ) -> Cow<'b, [&'ll Value]> {
+ let mut fn_ty = self.cx.val_ty(llfn);
+ // Strip off pointers
+ while self.cx.type_kind(fn_ty) == TypeKind::Pointer {
+ fn_ty = self.cx.element_type(fn_ty);
+ }
+
+ assert!(
+ self.cx.type_kind(fn_ty) == TypeKind::Function,
+ "builder::{} not passed a function, but {:?}",
+ typ,
+ fn_ty
+ );
+
+ let param_tys = self.cx.func_params_types(fn_ty);
+
+ let all_args_match = param_tys
+ .iter()
+ .zip(args.iter().map(|&v| self.val_ty(v)))
+ .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
+
+ if all_args_match {
+ return Cow::Borrowed(args);
+ }
+
+ let casted_args: Vec<_> = param_tys
+ .into_iter()
+ .zip(args.iter())
+ .enumerate()
+ .map(|(i, (expected_ty, &actual_val))| {
+ let actual_ty = self.val_ty(actual_val);
+ if expected_ty != actual_ty {
+ debug!(
+ "type mismatch in function call of {:?}. \
+ Expected {:?} for param {}, got {:?}; injecting bitcast",
+ llfn, expected_ty, i, actual_ty
+ );
+ self.bitcast(actual_val, expected_ty)
+ } else {
+ actual_val
+ }
+ })
+ .collect();
+
+ Cow::Owned(casted_args)
+ }
+
+ pub fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
+ unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
+ }
+
+ fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
+ let size = size.bytes();
+ if size == 0 {
+ return;
+ }
+
+ if !self.cx().sess().emit_lifetime_markers() {
+ return;
+ }
+
+ let lifetime_intrinsic = self.cx.get_intrinsic(intrinsic);
+
+ let ptr = self.pointercast(ptr, self.cx.type_i8p());
+ self.call(lifetime_intrinsic, &[self.cx.const_u64(size), ptr], None);
+ }
+
+ pub(crate) fn phi(
+ &mut self,
+ ty: &'ll Type,
+ vals: &[&'ll Value],
+ bbs: &[&'ll BasicBlock],
+ ) -> &'ll Value {
+ assert_eq!(vals.len(), bbs.len());
+ let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
+ unsafe {
+ llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
+ phi
+ }
+ }
+
+ fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
+ unsafe {
+ llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
+ }
+ }
+
+ fn wasm_and_missing_nontrapping_fptoint(&self) -> bool {
+ self.sess().target.target.arch == "wasm32"
+ && !self.sess().target_features.contains(&sym::nontrapping_dash_fptoint)
+ }
+}
--- /dev/null
- use rustc_codegen_ssa::base::{compare_simd_types, to_immediate, wants_msvc_seh};
+use crate::abi::{Abi, FnAbi, LlvmType, PassMode};
+use crate::builder::Builder;
+use crate::context::CodegenCx;
+use crate::llvm;
+use crate::type_::Type;
+use crate::type_of::LayoutLlvmExt;
+use crate::va_arg::emit_va_arg;
+use crate::value::Value;
+
+use rustc_ast as ast;
- to_immediate(self, load, self.layout_of(tp_ty))
++use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh};
+use rustc_codegen_ssa::common::span_invalid_monomorphization_error;
+use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
+use rustc_codegen_ssa::glue;
+use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
+use rustc_codegen_ssa::mir::place::PlaceRef;
+use rustc_codegen_ssa::traits::*;
+use rustc_codegen_ssa::MemFlags;
+use rustc_hir as hir;
+use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt};
+use rustc_middle::ty::{self, Ty};
+use rustc_middle::{bug, span_bug};
+use rustc_span::{sym, symbol::kw, Span, Symbol};
+use rustc_target::abi::{self, HasDataLayout, LayoutOf, Primitive};
+use rustc_target::spec::PanicStrategy;
+
+use std::cmp::Ordering;
+use std::iter;
+
+fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: Symbol) -> Option<&'ll Value> {
+ let llvm_name = match name {
+ sym::sqrtf32 => "llvm.sqrt.f32",
+ sym::sqrtf64 => "llvm.sqrt.f64",
+ sym::powif32 => "llvm.powi.f32",
+ sym::powif64 => "llvm.powi.f64",
+ sym::sinf32 => "llvm.sin.f32",
+ sym::sinf64 => "llvm.sin.f64",
+ sym::cosf32 => "llvm.cos.f32",
+ sym::cosf64 => "llvm.cos.f64",
+ sym::powf32 => "llvm.pow.f32",
+ sym::powf64 => "llvm.pow.f64",
+ sym::expf32 => "llvm.exp.f32",
+ sym::expf64 => "llvm.exp.f64",
+ sym::exp2f32 => "llvm.exp2.f32",
+ sym::exp2f64 => "llvm.exp2.f64",
+ sym::logf32 => "llvm.log.f32",
+ sym::logf64 => "llvm.log.f64",
+ sym::log10f32 => "llvm.log10.f32",
+ sym::log10f64 => "llvm.log10.f64",
+ sym::log2f32 => "llvm.log2.f32",
+ sym::log2f64 => "llvm.log2.f64",
+ sym::fmaf32 => "llvm.fma.f32",
+ sym::fmaf64 => "llvm.fma.f64",
+ sym::fabsf32 => "llvm.fabs.f32",
+ sym::fabsf64 => "llvm.fabs.f64",
+ sym::minnumf32 => "llvm.minnum.f32",
+ sym::minnumf64 => "llvm.minnum.f64",
+ sym::maxnumf32 => "llvm.maxnum.f32",
+ sym::maxnumf64 => "llvm.maxnum.f64",
+ sym::copysignf32 => "llvm.copysign.f32",
+ sym::copysignf64 => "llvm.copysign.f64",
+ sym::floorf32 => "llvm.floor.f32",
+ sym::floorf64 => "llvm.floor.f64",
+ sym::ceilf32 => "llvm.ceil.f32",
+ sym::ceilf64 => "llvm.ceil.f64",
+ sym::truncf32 => "llvm.trunc.f32",
+ sym::truncf64 => "llvm.trunc.f64",
+ sym::rintf32 => "llvm.rint.f32",
+ sym::rintf64 => "llvm.rint.f64",
+ sym::nearbyintf32 => "llvm.nearbyint.f32",
+ sym::nearbyintf64 => "llvm.nearbyint.f64",
+ sym::roundf32 => "llvm.round.f32",
+ sym::roundf64 => "llvm.round.f64",
+ sym::assume => "llvm.assume",
+ sym::abort => "llvm.trap",
+ _ => return None,
+ };
+ Some(cx.get_intrinsic(&llvm_name))
+}
+
+impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
+ fn codegen_intrinsic_call(
+ &mut self,
+ instance: ty::Instance<'tcx>,
+ fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+ args: &[OperandRef<'tcx, &'ll Value>],
+ llresult: &'ll Value,
+ span: Span,
+ ) {
+ let tcx = self.tcx;
+ let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all());
+
+ let (def_id, substs) = match callee_ty.kind {
+ ty::FnDef(def_id, substs) => (def_id, substs),
+ _ => bug!("expected fn item type, found {}", callee_ty),
+ };
+
+ let sig = callee_ty.fn_sig(tcx);
+ let sig = tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig);
+ let arg_tys = sig.inputs();
+ let ret_ty = sig.output();
+ let name = tcx.item_name(def_id);
+ let name_str = &*name.as_str();
+
+ let llret_ty = self.layout_of(ret_ty).llvm_type(self);
+ let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
+
+ let simple = get_simple_intrinsic(self, name);
+ let llval = match name {
+ _ if simple.is_some() => self.call(
+ simple.unwrap(),
+ &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
+ None,
+ ),
+ sym::unreachable => {
+ return;
+ }
+ sym::likely => {
+ let expect = self.get_intrinsic(&("llvm.expect.i1"));
+ self.call(expect, &[args[0].immediate(), self.const_bool(true)], None)
+ }
+ sym::unlikely => {
+ let expect = self.get_intrinsic(&("llvm.expect.i1"));
+ self.call(expect, &[args[0].immediate(), self.const_bool(false)], None)
+ }
+ kw::Try => {
+ try_intrinsic(
+ self,
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ llresult,
+ );
+ return;
+ }
+ sym::breakpoint => {
+ let llfn = self.get_intrinsic(&("llvm.debugtrap"));
+ self.call(llfn, &[], None)
+ }
+ sym::va_start => self.va_start(args[0].immediate()),
+ sym::va_end => self.va_end(args[0].immediate()),
+ sym::va_copy => {
+ let intrinsic = self.cx().get_intrinsic(&("llvm.va_copy"));
+ self.call(intrinsic, &[args[0].immediate(), args[1].immediate()], None)
+ }
+ sym::va_arg => {
+ match fn_abi.ret.layout.abi {
+ abi::Abi::Scalar(ref scalar) => {
+ match scalar.value {
+ Primitive::Int(..) => {
+ if self.cx().size_of(ret_ty).bytes() < 4 {
+ // `va_arg` should not be called on a integer type
+ // less than 4 bytes in length. If it is, promote
+ // the integer to a `i32` and truncate the result
+ // back to the smaller type.
+ let promoted_result = emit_va_arg(self, args[0], tcx.types.i32);
+ self.trunc(promoted_result, llret_ty)
+ } else {
+ emit_va_arg(self, args[0], ret_ty)
+ }
+ }
+ Primitive::F64 | Primitive::Pointer => {
+ emit_va_arg(self, args[0], ret_ty)
+ }
+ // `va_arg` should never be used with the return type f32.
+ Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"),
+ }
+ }
+ _ => bug!("the va_arg intrinsic does not work with non-scalar types"),
+ }
+ }
+ sym::size_of_val => {
+ let tp_ty = substs.type_at(0);
+ if let OperandValue::Pair(_, meta) = args[0].val {
+ let (llsize, _) = glue::size_and_align_of_dst(self, tp_ty, Some(meta));
+ llsize
+ } else {
+ self.const_usize(self.size_of(tp_ty).bytes())
+ }
+ }
+ sym::min_align_of_val => {
+ let tp_ty = substs.type_at(0);
+ if let OperandValue::Pair(_, meta) = args[0].val {
+ let (_, llalign) = glue::size_and_align_of_dst(self, tp_ty, Some(meta));
+ llalign
+ } else {
+ self.const_usize(self.align_of(tp_ty).bytes())
+ }
+ }
+ sym::size_of
+ | sym::pref_align_of
+ | sym::min_align_of
+ | sym::needs_drop
+ | sym::type_id
+ | sym::type_name
+ | sym::variant_count => {
+ let value = self
+ .tcx
+ .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None)
+ .unwrap();
+ OperandRef::from_const(self, value, ret_ty).immediate_or_packed_pair(self)
+ }
+ // Effectively no-op
+ sym::forget => {
+ return;
+ }
+ sym::offset => {
+ let ptr = args[0].immediate();
+ let offset = args[1].immediate();
+ self.inbounds_gep(ptr, &[offset])
+ }
+ sym::arith_offset => {
+ let ptr = args[0].immediate();
+ let offset = args[1].immediate();
+ self.gep(ptr, &[offset])
+ }
+
+ sym::copy_nonoverlapping => {
+ copy_intrinsic(
+ self,
+ false,
+ false,
+ substs.type_at(0),
+ args[1].immediate(),
+ args[0].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+ sym::copy => {
+ copy_intrinsic(
+ self,
+ true,
+ false,
+ substs.type_at(0),
+ args[1].immediate(),
+ args[0].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+ sym::write_bytes => {
+ memset_intrinsic(
+ self,
+ false,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+
+ sym::volatile_copy_nonoverlapping_memory => {
+ copy_intrinsic(
+ self,
+ false,
+ true,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+ sym::volatile_copy_memory => {
+ copy_intrinsic(
+ self,
+ true,
+ true,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+ sym::volatile_set_memory => {
+ memset_intrinsic(
+ self,
+ true,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+ sym::volatile_load | sym::unaligned_volatile_load => {
+ let tp_ty = substs.type_at(0);
+ let mut ptr = args[0].immediate();
+ if let PassMode::Cast(ty) = fn_abi.ret.mode {
+ ptr = self.pointercast(ptr, self.type_ptr_to(ty.llvm_type(self)));
+ }
+ let load = self.volatile_load(ptr);
+ let align = if name == sym::unaligned_volatile_load {
+ 1
+ } else {
+ self.align_of(tp_ty).bytes() as u32
+ };
+ unsafe {
+ llvm::LLVMSetAlignment(load, align);
+ }
++ self.to_immediate(load, self.layout_of(tp_ty))
+ }
+ sym::volatile_store => {
+ let dst = args[0].deref(self.cx());
+ args[1].val.volatile_store(self, dst);
+ return;
+ }
+ sym::unaligned_volatile_store => {
+ let dst = args[0].deref(self.cx());
+ args[1].val.unaligned_volatile_store(self, dst);
+ return;
+ }
+ sym::prefetch_read_data
+ | sym::prefetch_write_data
+ | sym::prefetch_read_instruction
+ | sym::prefetch_write_instruction => {
+ let expect = self.get_intrinsic(&("llvm.prefetch"));
+ let (rw, cache_type) = match name {
+ sym::prefetch_read_data => (0, 1),
+ sym::prefetch_write_data => (1, 1),
+ sym::prefetch_read_instruction => (0, 0),
+ sym::prefetch_write_instruction => (1, 0),
+ _ => bug!(),
+ };
+ self.call(
+ expect,
+ &[
+ args[0].immediate(),
+ self.const_i32(rw),
+ args[1].immediate(),
+ self.const_i32(cache_type),
+ ],
+ None,
+ )
+ }
+ sym::ctlz
+ | sym::ctlz_nonzero
+ | sym::cttz
+ | sym::cttz_nonzero
+ | sym::ctpop
+ | sym::bswap
+ | sym::bitreverse
+ | sym::add_with_overflow
+ | sym::sub_with_overflow
+ | sym::mul_with_overflow
+ | sym::wrapping_add
+ | sym::wrapping_sub
+ | sym::wrapping_mul
+ | sym::unchecked_div
+ | sym::unchecked_rem
+ | sym::unchecked_shl
+ | sym::unchecked_shr
+ | sym::unchecked_add
+ | sym::unchecked_sub
+ | sym::unchecked_mul
+ | sym::exact_div
+ | sym::rotate_left
+ | sym::rotate_right
+ | sym::saturating_add
+ | sym::saturating_sub => {
+ let ty = arg_tys[0];
+ match int_type_width_signed(ty, self) {
+ Some((width, signed)) => match name {
+ sym::ctlz | sym::cttz => {
+ let y = self.const_bool(false);
+ let llfn = self.get_intrinsic(&format!("llvm.{}.i{}", name, width));
+ self.call(llfn, &[args[0].immediate(), y], None)
+ }
+ sym::ctlz_nonzero | sym::cttz_nonzero => {
+ let y = self.const_bool(true);
+ let llvm_name = &format!("llvm.{}.i{}", &name_str[..4], width);
+ let llfn = self.get_intrinsic(llvm_name);
+ self.call(llfn, &[args[0].immediate(), y], None)
+ }
+ sym::ctpop => self.call(
+ self.get_intrinsic(&format!("llvm.ctpop.i{}", width)),
+ &[args[0].immediate()],
+ None,
+ ),
+ sym::bswap => {
+ if width == 8 {
+ args[0].immediate() // byte swap a u8/i8 is just a no-op
+ } else {
+ self.call(
+ self.get_intrinsic(&format!("llvm.bswap.i{}", width)),
+ &[args[0].immediate()],
+ None,
+ )
+ }
+ }
+ sym::bitreverse => self.call(
+ self.get_intrinsic(&format!("llvm.bitreverse.i{}", width)),
+ &[args[0].immediate()],
+ None,
+ ),
+ sym::add_with_overflow
+ | sym::sub_with_overflow
+ | sym::mul_with_overflow => {
+ let intrinsic = format!(
+ "llvm.{}{}.with.overflow.i{}",
+ if signed { 's' } else { 'u' },
+ &name_str[..3],
+ width
+ );
+ let llfn = self.get_intrinsic(&intrinsic);
+
+ // Convert `i1` to a `bool`, and write it to the out parameter
+ let pair =
+ self.call(llfn, &[args[0].immediate(), args[1].immediate()], None);
+ let val = self.extract_value(pair, 0);
+ let overflow = self.extract_value(pair, 1);
+ let overflow = self.zext(overflow, self.type_bool());
+
+ let dest = result.project_field(self, 0);
+ self.store(val, dest.llval, dest.align);
+ let dest = result.project_field(self, 1);
+ self.store(overflow, dest.llval, dest.align);
+
+ return;
+ }
+ sym::wrapping_add => self.add(args[0].immediate(), args[1].immediate()),
+ sym::wrapping_sub => self.sub(args[0].immediate(), args[1].immediate()),
+ sym::wrapping_mul => self.mul(args[0].immediate(), args[1].immediate()),
+ sym::exact_div => {
+ if signed {
+ self.exactsdiv(args[0].immediate(), args[1].immediate())
+ } else {
+ self.exactudiv(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_div => {
+ if signed {
+ self.sdiv(args[0].immediate(), args[1].immediate())
+ } else {
+ self.udiv(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_rem => {
+ if signed {
+ self.srem(args[0].immediate(), args[1].immediate())
+ } else {
+ self.urem(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_shl => self.shl(args[0].immediate(), args[1].immediate()),
+ sym::unchecked_shr => {
+ if signed {
+ self.ashr(args[0].immediate(), args[1].immediate())
+ } else {
+ self.lshr(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_add => {
+ if signed {
+ self.unchecked_sadd(args[0].immediate(), args[1].immediate())
+ } else {
+ self.unchecked_uadd(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_sub => {
+ if signed {
+ self.unchecked_ssub(args[0].immediate(), args[1].immediate())
+ } else {
+ self.unchecked_usub(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_mul => {
+ if signed {
+ self.unchecked_smul(args[0].immediate(), args[1].immediate())
+ } else {
+ self.unchecked_umul(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::rotate_left | sym::rotate_right => {
+ let is_left = name == sym::rotate_left;
+ let val = args[0].immediate();
+ let raw_shift = args[1].immediate();
+ // rotate = funnel shift with first two args the same
+ let llvm_name =
+ &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width);
+ let llfn = self.get_intrinsic(llvm_name);
+ self.call(llfn, &[val, val, raw_shift], None)
+ }
+ sym::saturating_add | sym::saturating_sub => {
+ let is_add = name == sym::saturating_add;
+ let lhs = args[0].immediate();
+ let rhs = args[1].immediate();
+ let llvm_name = &format!(
+ "llvm.{}{}.sat.i{}",
+ if signed { 's' } else { 'u' },
+ if is_add { "add" } else { "sub" },
+ width
+ );
+ let llfn = self.get_intrinsic(llvm_name);
+ self.call(llfn, &[lhs, rhs], None)
+ }
+ _ => bug!(),
+ },
+ None => {
+ span_invalid_monomorphization_error(
+ tcx.sess,
+ span,
+ &format!(
+ "invalid monomorphization of `{}` intrinsic: \
+ expected basic integer type, found `{}`",
+ name, ty
+ ),
+ );
+ return;
+ }
+ }
+ }
+ sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
+ match float_type_width(arg_tys[0]) {
+ Some(_width) => match name {
+ sym::fadd_fast => self.fadd_fast(args[0].immediate(), args[1].immediate()),
+ sym::fsub_fast => self.fsub_fast(args[0].immediate(), args[1].immediate()),
+ sym::fmul_fast => self.fmul_fast(args[0].immediate(), args[1].immediate()),
+ sym::fdiv_fast => self.fdiv_fast(args[0].immediate(), args[1].immediate()),
+ sym::frem_fast => self.frem_fast(args[0].immediate(), args[1].immediate()),
+ _ => bug!(),
+ },
+ None => {
+ span_invalid_monomorphization_error(
+ tcx.sess,
+ span,
+ &format!(
+ "invalid monomorphization of `{}` intrinsic: \
+ expected basic float type, found `{}`",
+ name, arg_tys[0]
+ ),
+ );
+ return;
+ }
+ }
+ }
+
+ sym::float_to_int_unchecked => {
+ if float_type_width(arg_tys[0]).is_none() {
+ span_invalid_monomorphization_error(
+ tcx.sess,
+ span,
+ &format!(
+ "invalid monomorphization of `float_to_int_unchecked` \
+ intrinsic: expected basic float type, \
+ found `{}`",
+ arg_tys[0]
+ ),
+ );
+ return;
+ }
+ let (width, signed) = match int_type_width_signed(ret_ty, self.cx) {
+ Some(pair) => pair,
+ None => {
+ span_invalid_monomorphization_error(
+ tcx.sess,
+ span,
+ &format!(
+ "invalid monomorphization of `float_to_int_unchecked` \
+ intrinsic: expected basic integer type, \
+ found `{}`",
+ ret_ty
+ ),
+ );
+ return;
+ }
+ };
+ if signed {
+ self.fptosi(args[0].immediate(), self.cx.type_ix(width))
+ } else {
+ self.fptoui(args[0].immediate(), self.cx.type_ix(width))
+ }
+ }
+
+ sym::discriminant_value => {
+ if ret_ty.is_integral() {
+ args[0].deref(self.cx()).codegen_get_discr(self, ret_ty)
+ } else {
+ span_bug!(span, "Invalid discriminant type for `{:?}`", arg_tys[0])
+ }
+ }
+
+ _ if name_str.starts_with("simd_") => {
+ match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) {
+ Ok(llval) => llval,
+ Err(()) => return,
+ }
+ }
+ // This requires that atomic intrinsics follow a specific naming pattern:
+ // "atomic_<operation>[_<ordering>]", and no ordering means SeqCst
+ name if name_str.starts_with("atomic_") => {
+ use rustc_codegen_ssa::common::AtomicOrdering::*;
+ use rustc_codegen_ssa::common::{AtomicRmwBinOp, SynchronizationScope};
+
+ let split: Vec<&str> = name_str.split('_').collect();
+
+ let is_cxchg = split[1] == "cxchg" || split[1] == "cxchgweak";
+ let (order, failorder) = match split.len() {
+ 2 => (SequentiallyConsistent, SequentiallyConsistent),
+ 3 => match split[2] {
+ "unordered" => (Unordered, Unordered),
+ "relaxed" => (Monotonic, Monotonic),
+ "acq" => (Acquire, Acquire),
+ "rel" => (Release, Monotonic),
+ "acqrel" => (AcquireRelease, Acquire),
+ "failrelaxed" if is_cxchg => (SequentiallyConsistent, Monotonic),
+ "failacq" if is_cxchg => (SequentiallyConsistent, Acquire),
+ _ => self.sess().fatal("unknown ordering in atomic intrinsic"),
+ },
+ 4 => match (split[2], split[3]) {
+ ("acq", "failrelaxed") if is_cxchg => (Acquire, Monotonic),
+ ("acqrel", "failrelaxed") if is_cxchg => (AcquireRelease, Monotonic),
+ _ => self.sess().fatal("unknown ordering in atomic intrinsic"),
+ },
+ _ => self.sess().fatal("Atomic intrinsic not in correct format"),
+ };
+
+ let invalid_monomorphization = |ty| {
+ span_invalid_monomorphization_error(
+ tcx.sess,
+ span,
+ &format!(
+ "invalid monomorphization of `{}` intrinsic: \
+ expected basic integer type, found `{}`",
+ name, ty
+ ),
+ );
+ };
+
+ match split[1] {
+ "cxchg" | "cxchgweak" => {
+ let ty = substs.type_at(0);
+ if int_type_width_signed(ty, self).is_some() {
+ let weak = split[1] == "cxchgweak";
+ let pair = self.atomic_cmpxchg(
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ order,
+ failorder,
+ weak,
+ );
+ let val = self.extract_value(pair, 0);
+ let success = self.extract_value(pair, 1);
+ let success = self.zext(success, self.type_bool());
+
+ let dest = result.project_field(self, 0);
+ self.store(val, dest.llval, dest.align);
+ let dest = result.project_field(self, 1);
+ self.store(success, dest.llval, dest.align);
+ return;
+ } else {
+ return invalid_monomorphization(ty);
+ }
+ }
+
+ "load" => {
+ let ty = substs.type_at(0);
+ if int_type_width_signed(ty, self).is_some() {
+ let size = self.size_of(ty);
+ self.atomic_load(args[0].immediate(), order, size)
+ } else {
+ return invalid_monomorphization(ty);
+ }
+ }
+
+ "store" => {
+ let ty = substs.type_at(0);
+ if int_type_width_signed(ty, self).is_some() {
+ let size = self.size_of(ty);
+ self.atomic_store(
+ args[1].immediate(),
+ args[0].immediate(),
+ order,
+ size,
+ );
+ return;
+ } else {
+ return invalid_monomorphization(ty);
+ }
+ }
+
+ "fence" => {
+ self.atomic_fence(order, SynchronizationScope::CrossThread);
+ return;
+ }
+
+ "singlethreadfence" => {
+ self.atomic_fence(order, SynchronizationScope::SingleThread);
+ return;
+ }
+
+ // These are all AtomicRMW ops
+ op => {
+ let atom_op = match op {
+ "xchg" => AtomicRmwBinOp::AtomicXchg,
+ "xadd" => AtomicRmwBinOp::AtomicAdd,
+ "xsub" => AtomicRmwBinOp::AtomicSub,
+ "and" => AtomicRmwBinOp::AtomicAnd,
+ "nand" => AtomicRmwBinOp::AtomicNand,
+ "or" => AtomicRmwBinOp::AtomicOr,
+ "xor" => AtomicRmwBinOp::AtomicXor,
+ "max" => AtomicRmwBinOp::AtomicMax,
+ "min" => AtomicRmwBinOp::AtomicMin,
+ "umax" => AtomicRmwBinOp::AtomicUMax,
+ "umin" => AtomicRmwBinOp::AtomicUMin,
+ _ => self.sess().fatal("unknown atomic operation"),
+ };
+
+ let ty = substs.type_at(0);
+ if int_type_width_signed(ty, self).is_some() {
+ self.atomic_rmw(
+ atom_op,
+ args[0].immediate(),
+ args[1].immediate(),
+ order,
+ )
+ } else {
+ return invalid_monomorphization(ty);
+ }
+ }
+ }
+ }
+
+ sym::nontemporal_store => {
+ let dst = args[0].deref(self.cx());
+ args[1].val.nontemporal_store(self, dst);
+ return;
+ }
+
+ sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => {
+ let a = args[0].immediate();
+ let b = args[1].immediate();
+ if name == sym::ptr_guaranteed_eq {
+ self.icmp(IntPredicate::IntEQ, a, b)
+ } else {
+ self.icmp(IntPredicate::IntNE, a, b)
+ }
+ }
+
+ sym::ptr_offset_from => {
+ let ty = substs.type_at(0);
+ let pointee_size = self.size_of(ty);
+
+ // This is the same sequence that Clang emits for pointer subtraction.
+ // It can be neither `nsw` nor `nuw` because the input is treated as
+ // unsigned but then the output is treated as signed, so neither works.
+ let a = args[0].immediate();
+ let b = args[1].immediate();
+ let a = self.ptrtoint(a, self.type_isize());
+ let b = self.ptrtoint(b, self.type_isize());
+ let d = self.sub(a, b);
+ let pointee_size = self.const_usize(pointee_size.bytes());
+ // this is where the signed magic happens (notice the `s` in `exactsdiv`)
+ self.exactsdiv(d, pointee_size)
+ }
+
+ _ => bug!("unknown intrinsic '{}'", name),
+ };
+
+ if !fn_abi.ret.is_ignore() {
+ if let PassMode::Cast(ty) = fn_abi.ret.mode {
+ let ptr_llty = self.type_ptr_to(ty.llvm_type(self));
+ let ptr = self.pointercast(result.llval, ptr_llty);
+ self.store(llval, ptr, result.align);
+ } else {
+ OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
+ .val
+ .store(self, result);
+ }
+ }
+ }
+
+ fn abort(&mut self) {
+ let fnname = self.get_intrinsic(&("llvm.trap"));
+ self.call(fnname, &[], None);
+ }
+
+ fn assume(&mut self, val: Self::Value) {
+ let assume_intrinsic = self.get_intrinsic("llvm.assume");
+ self.call(assume_intrinsic, &[val], None);
+ }
+
+ fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
+ let expect = self.get_intrinsic(&"llvm.expect.i1");
+ self.call(expect, &[cond, self.const_bool(expected)], None)
+ }
+
+ fn sideeffect(&mut self) {
+ if self.tcx.sess.opts.debugging_opts.insert_sideeffect {
+ let fnname = self.get_intrinsic(&("llvm.sideeffect"));
+ self.call(fnname, &[], None);
+ }
+ }
+
+ fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
+ let intrinsic = self.cx().get_intrinsic("llvm.va_start");
+ self.call(intrinsic, &[va_list], None)
+ }
+
+ fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
+ let intrinsic = self.cx().get_intrinsic("llvm.va_end");
+ self.call(intrinsic, &[va_list], None)
+ }
+}
+
+fn copy_intrinsic(
+ bx: &mut Builder<'a, 'll, 'tcx>,
+ allow_overlap: bool,
+ volatile: bool,
+ ty: Ty<'tcx>,
+ dst: &'ll Value,
+ src: &'ll Value,
+ count: &'ll Value,
+) {
+ let (size, align) = bx.size_and_align_of(ty);
+ let size = bx.mul(bx.const_usize(size.bytes()), count);
+ let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() };
+ if allow_overlap {
+ bx.memmove(dst, align, src, align, size, flags);
+ } else {
+ bx.memcpy(dst, align, src, align, size, flags);
+ }
+}
+
+fn memset_intrinsic(
+ bx: &mut Builder<'a, 'll, 'tcx>,
+ volatile: bool,
+ ty: Ty<'tcx>,
+ dst: &'ll Value,
+ val: &'ll Value,
+ count: &'ll Value,
+) {
+ let (size, align) = bx.size_and_align_of(ty);
+ let size = bx.mul(bx.const_usize(size.bytes()), count);
+ let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() };
+ bx.memset(dst, val, size, align, flags);
+}
+
+fn try_intrinsic(
+ bx: &mut Builder<'a, 'll, 'tcx>,
+ try_func: &'ll Value,
+ data: &'ll Value,
+ catch_func: &'ll Value,
+ dest: &'ll Value,
+) {
+ if bx.sess().panic_strategy() == PanicStrategy::Abort {
+ bx.call(try_func, &[data], None);
+ // Return 0 unconditionally from the intrinsic call;
+ // we can never unwind.
+ let ret_align = bx.tcx().data_layout.i32_align.abi;
+ bx.store(bx.const_i32(0), dest, ret_align);
+ } else if wants_msvc_seh(bx.sess()) {
+ codegen_msvc_try(bx, try_func, data, catch_func, dest);
+ } else if bx.sess().target.target.options.is_like_emscripten {
+ codegen_emcc_try(bx, try_func, data, catch_func, dest);
+ } else {
+ codegen_gnu_try(bx, try_func, data, catch_func, dest);
+ }
+}
+
+// MSVC's definition of the `rust_try` function.
+//
+// This implementation uses the new exception handling instructions in LLVM
+// which have support in LLVM for SEH on MSVC targets. Although these
+// instructions are meant to work for all targets, as of the time of this
+// writing, however, LLVM does not recommend the usage of these new instructions
+// as the old ones are still more optimized.
+fn codegen_msvc_try(
+ bx: &mut Builder<'a, 'll, 'tcx>,
+ try_func: &'ll Value,
+ data: &'ll Value,
+ catch_func: &'ll Value,
+ dest: &'ll Value,
+) {
+ let llfn = get_rust_try_fn(bx, &mut |mut bx| {
+ bx.set_personality_fn(bx.eh_personality());
+ bx.sideeffect();
+
+ let mut normal = bx.build_sibling_block("normal");
+ let mut catchswitch = bx.build_sibling_block("catchswitch");
+ let mut catchpad_rust = bx.build_sibling_block("catchpad_rust");
+ let mut catchpad_foreign = bx.build_sibling_block("catchpad_foreign");
+ let mut caught = bx.build_sibling_block("caught");
+
+ let try_func = llvm::get_param(bx.llfn(), 0);
+ let data = llvm::get_param(bx.llfn(), 1);
+ let catch_func = llvm::get_param(bx.llfn(), 2);
+
+ // We're generating an IR snippet that looks like:
+ //
+ // declare i32 @rust_try(%try_func, %data, %catch_func) {
+ // %slot = alloca i8*
+ // invoke %try_func(%data) to label %normal unwind label %catchswitch
+ //
+ // normal:
+ // ret i32 0
+ //
+ // catchswitch:
+ // %cs = catchswitch within none [%catchpad_rust, %catchpad_foreign] unwind to caller
+ //
+ // catchpad_rust:
+ // %tok = catchpad within %cs [%type_descriptor, 8, %slot]
+ // %ptr = load %slot
+ // call %catch_func(%data, %ptr)
+ // catchret from %tok to label %caught
+ //
+ // catchpad_foreign:
+ // %tok = catchpad within %cs [null, 64, null]
+ // call %catch_func(%data, null)
+ // catchret from %tok to label %caught
+ //
+ // caught:
+ // ret i32 1
+ // }
+ //
+ // This structure follows the basic usage of throw/try/catch in LLVM.
+ // For example, compile this C++ snippet to see what LLVM generates:
+ //
+ // struct rust_panic {
+ // rust_panic(const rust_panic&);
+ // ~rust_panic();
+ //
+ // void* x[2];
+ // };
+ //
+ // int __rust_try(
+ // void (*try_func)(void*),
+ // void *data,
+ // void (*catch_func)(void*, void*) noexcept
+ // ) {
+ // try {
+ // try_func(data);
+ // return 0;
+ // } catch(rust_panic& a) {
+ // catch_func(data, &a);
+ // return 1;
+ // } catch(...) {
+ // catch_func(data, NULL);
+ // return 1;
+ // }
+ // }
+ //
+ // More information can be found in libstd's seh.rs implementation.
+ let ptr_align = bx.tcx().data_layout.pointer_align.abi;
+ let slot = bx.alloca(bx.type_i8p(), ptr_align);
+ bx.invoke(try_func, &[data], normal.llbb(), catchswitch.llbb(), None);
+
+ normal.ret(bx.const_i32(0));
+
+ let cs = catchswitch.catch_switch(None, None, 2);
+ catchswitch.add_handler(cs, catchpad_rust.llbb());
+ catchswitch.add_handler(cs, catchpad_foreign.llbb());
+
+ // We can't use the TypeDescriptor defined in libpanic_unwind because it
+ // might be in another DLL and the SEH encoding only supports specifying
+ // a TypeDescriptor from the current module.
+ //
+ // However this isn't an issue since the MSVC runtime uses string
+ // comparison on the type name to match TypeDescriptors rather than
+ // pointer equality.
+ //
+ // So instead we generate a new TypeDescriptor in each module that uses
+ // `try` and let the linker merge duplicate definitions in the same
+ // module.
+ //
+ // When modifying, make sure that the type_name string exactly matches
+ // the one used in src/libpanic_unwind/seh.rs.
+ let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_i8p());
+ let type_name = bx.const_bytes(b"rust_panic\0");
+ let type_info =
+ bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_i8p()), type_name], false);
+ let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info));
+ unsafe {
+ llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
+ llvm::SetUniqueComdat(bx.llmod, tydesc);
+ llvm::LLVMSetInitializer(tydesc, type_info);
+ }
+
+ // The flag value of 8 indicates that we are catching the exception by
+ // reference instead of by value. We can't use catch by value because
+ // that requires copying the exception object, which we don't support
+ // since our exception object effectively contains a Box.
+ //
+ // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang
+ let flags = bx.const_i32(8);
+ let funclet = catchpad_rust.catch_pad(cs, &[tydesc, flags, slot]);
+ let ptr = catchpad_rust.load(slot, ptr_align);
+ catchpad_rust.call(catch_func, &[data, ptr], Some(&funclet));
+ catchpad_rust.catch_ret(&funclet, caught.llbb());
+
+ // The flag value of 64 indicates a "catch-all".
+ let flags = bx.const_i32(64);
+ let null = bx.const_null(bx.type_i8p());
+ let funclet = catchpad_foreign.catch_pad(cs, &[null, flags, null]);
+ catchpad_foreign.call(catch_func, &[data, null], Some(&funclet));
+ catchpad_foreign.catch_ret(&funclet, caught.llbb());
+
+ caught.ret(bx.const_i32(1));
+ });
+
+ // Note that no invoke is used here because by definition this function
+ // can't panic (that's what it's catching).
+ let ret = bx.call(llfn, &[try_func, data, catch_func], None);
+ let i32_align = bx.tcx().data_layout.i32_align.abi;
+ bx.store(ret, dest, i32_align);
+}
+
+// Definition of the standard `try` function for Rust using the GNU-like model
+// of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke`
+// instructions).
+//
+// This codegen is a little surprising because we always call a shim
+// function instead of inlining the call to `invoke` manually here. This is done
+// because in LLVM we're only allowed to have one personality per function
+// definition. The call to the `try` intrinsic is being inlined into the
+// function calling it, and that function may already have other personality
+// functions in play. By calling a shim we're guaranteed that our shim will have
+// the right personality function.
+fn codegen_gnu_try(
+ bx: &mut Builder<'a, 'll, 'tcx>,
+ try_func: &'ll Value,
+ data: &'ll Value,
+ catch_func: &'ll Value,
+ dest: &'ll Value,
+) {
+ let llfn = get_rust_try_fn(bx, &mut |mut bx| {
+ // Codegens the shims described above:
+ //
+ // bx:
+ // invoke %try_func(%data) normal %normal unwind %catch
+ //
+ // normal:
+ // ret 0
+ //
+ // catch:
+ // (%ptr, _) = landingpad
+ // call %catch_func(%data, %ptr)
+ // ret 1
+
+ bx.sideeffect();
+
+ let mut then = bx.build_sibling_block("then");
+ let mut catch = bx.build_sibling_block("catch");
+
+ let try_func = llvm::get_param(bx.llfn(), 0);
+ let data = llvm::get_param(bx.llfn(), 1);
+ let catch_func = llvm::get_param(bx.llfn(), 2);
+ bx.invoke(try_func, &[data], then.llbb(), catch.llbb(), None);
+ then.ret(bx.const_i32(0));
+
+ // Type indicator for the exception being thrown.
+ //
+ // The first value in this tuple is a pointer to the exception object
+ // being thrown. The second value is a "selector" indicating which of
+ // the landing pad clauses the exception's type had been matched to.
+ // rust_try ignores the selector.
+ let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false);
+ let vals = catch.landing_pad(lpad_ty, bx.eh_personality(), 1);
+ let tydesc = bx.const_null(bx.type_i8p());
+ catch.add_clause(vals, tydesc);
+ let ptr = catch.extract_value(vals, 0);
+ catch.call(catch_func, &[data, ptr], None);
+ catch.ret(bx.const_i32(1));
+ });
+
+ // Note that no invoke is used here because by definition this function
+ // can't panic (that's what it's catching).
+ let ret = bx.call(llfn, &[try_func, data, catch_func], None);
+ let i32_align = bx.tcx().data_layout.i32_align.abi;
+ bx.store(ret, dest, i32_align);
+}
+
+// Variant of codegen_gnu_try used for emscripten where Rust panics are
+// implemented using C++ exceptions. Here we use exceptions of a specific type
+// (`struct rust_panic`) to represent Rust panics.
+fn codegen_emcc_try(
+ bx: &mut Builder<'a, 'll, 'tcx>,
+ try_func: &'ll Value,
+ data: &'ll Value,
+ catch_func: &'ll Value,
+ dest: &'ll Value,
+) {
+ let llfn = get_rust_try_fn(bx, &mut |mut bx| {
+ // Codegens the shims described above:
+ //
+ // bx:
+ // invoke %try_func(%data) normal %normal unwind %catch
+ //
+ // normal:
+ // ret 0
+ //
+ // catch:
+ // (%ptr, %selector) = landingpad
+ // %rust_typeid = @llvm.eh.typeid.for(@_ZTI10rust_panic)
+ // %is_rust_panic = %selector == %rust_typeid
+ // %catch_data = alloca { i8*, i8 }
+ // %catch_data[0] = %ptr
+ // %catch_data[1] = %is_rust_panic
+ // call %catch_func(%data, %catch_data)
+ // ret 1
+
+ bx.sideeffect();
+
+ let mut then = bx.build_sibling_block("then");
+ let mut catch = bx.build_sibling_block("catch");
+
+ let try_func = llvm::get_param(bx.llfn(), 0);
+ let data = llvm::get_param(bx.llfn(), 1);
+ let catch_func = llvm::get_param(bx.llfn(), 2);
+ bx.invoke(try_func, &[data], then.llbb(), catch.llbb(), None);
+ then.ret(bx.const_i32(0));
+
+ // Type indicator for the exception being thrown.
+ //
+ // The first value in this tuple is a pointer to the exception object
+ // being thrown. The second value is a "selector" indicating which of
+ // the landing pad clauses the exception's type had been matched to.
+ let tydesc = bx.eh_catch_typeinfo();
+ let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false);
+ let vals = catch.landing_pad(lpad_ty, bx.eh_personality(), 2);
+ catch.add_clause(vals, tydesc);
+ catch.add_clause(vals, bx.const_null(bx.type_i8p()));
+ let ptr = catch.extract_value(vals, 0);
+ let selector = catch.extract_value(vals, 1);
+
+ // Check if the typeid we got is the one for a Rust panic.
+ let llvm_eh_typeid_for = bx.get_intrinsic("llvm.eh.typeid.for");
+ let rust_typeid = catch.call(llvm_eh_typeid_for, &[tydesc], None);
+ let is_rust_panic = catch.icmp(IntPredicate::IntEQ, selector, rust_typeid);
+ let is_rust_panic = catch.zext(is_rust_panic, bx.type_bool());
+
+ // We need to pass two values to catch_func (ptr and is_rust_panic), so
+ // create an alloca and pass a pointer to that.
+ let ptr_align = bx.tcx().data_layout.pointer_align.abi;
+ let i8_align = bx.tcx().data_layout.i8_align.abi;
+ let catch_data =
+ catch.alloca(bx.type_struct(&[bx.type_i8p(), bx.type_bool()], false), ptr_align);
+ let catch_data_0 = catch.inbounds_gep(catch_data, &[bx.const_usize(0), bx.const_usize(0)]);
+ catch.store(ptr, catch_data_0, ptr_align);
+ let catch_data_1 = catch.inbounds_gep(catch_data, &[bx.const_usize(0), bx.const_usize(1)]);
+ catch.store(is_rust_panic, catch_data_1, i8_align);
+ let catch_data = catch.bitcast(catch_data, bx.type_i8p());
+
+ catch.call(catch_func, &[data, catch_data], None);
+ catch.ret(bx.const_i32(1));
+ });
+
+ // Note that no invoke is used here because by definition this function
+ // can't panic (that's what it's catching).
+ let ret = bx.call(llfn, &[try_func, data, catch_func], None);
+ let i32_align = bx.tcx().data_layout.i32_align.abi;
+ bx.store(ret, dest, i32_align);
+}
+
+// Helper function to give a Block to a closure to codegen a shim function.
+// This is currently primarily used for the `try` intrinsic functions above.
+fn gen_fn<'ll, 'tcx>(
+ cx: &CodegenCx<'ll, 'tcx>,
+ name: &str,
+ inputs: Vec<Ty<'tcx>>,
+ output: Ty<'tcx>,
+ codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>),
+) -> &'ll Value {
+ let rust_fn_sig = ty::Binder::bind(cx.tcx.mk_fn_sig(
+ inputs.into_iter(),
+ output,
+ false,
+ hir::Unsafety::Unsafe,
+ Abi::Rust,
+ ));
+ let fn_abi = FnAbi::of_fn_ptr(cx, rust_fn_sig, &[]);
+ let llfn = cx.declare_fn(name, &fn_abi);
+ cx.set_frame_pointer_elimination(llfn);
+ cx.apply_target_cpu_attr(llfn);
+ // FIXME(eddyb) find a nicer way to do this.
+ unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) };
+ let bx = Builder::new_block(cx, llfn, "entry-block");
+ codegen(bx);
+ llfn
+}
+
+// Helper function used to get a handle to the `__rust_try` function used to
+// catch exceptions.
+//
+// This function is only generated once and is then cached.
+fn get_rust_try_fn<'ll, 'tcx>(
+ cx: &CodegenCx<'ll, 'tcx>,
+ codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>),
+) -> &'ll Value {
+ if let Some(llfn) = cx.rust_try_fn.get() {
+ return llfn;
+ }
+
+ // Define the type up front for the signature of the rust_try function.
+ let tcx = cx.tcx;
+ let i8p = tcx.mk_mut_ptr(tcx.types.i8);
+ let try_fn_ty = tcx.mk_fn_ptr(ty::Binder::bind(tcx.mk_fn_sig(
+ iter::once(i8p),
+ tcx.mk_unit(),
+ false,
+ hir::Unsafety::Unsafe,
+ Abi::Rust,
+ )));
+ let catch_fn_ty = tcx.mk_fn_ptr(ty::Binder::bind(tcx.mk_fn_sig(
+ [i8p, i8p].iter().cloned(),
+ tcx.mk_unit(),
+ false,
+ hir::Unsafety::Unsafe,
+ Abi::Rust,
+ )));
+ let output = tcx.types.i32;
+ let rust_try = gen_fn(cx, "__rust_try", vec![try_fn_ty, i8p, catch_fn_ty], output, codegen);
+ cx.rust_try_fn.set(Some(rust_try));
+ rust_try
+}
+
+fn generic_simd_intrinsic(
+ bx: &mut Builder<'a, 'll, 'tcx>,
+ name: Symbol,
+ callee_ty: Ty<'tcx>,
+ args: &[OperandRef<'tcx, &'ll Value>],
+ ret_ty: Ty<'tcx>,
+ llret_ty: &'ll Type,
+ span: Span,
+) -> Result<&'ll Value, ()> {
+ // macros for error handling:
+ macro_rules! emit_error {
+ ($msg: tt) => {
+ emit_error!($msg, )
+ };
+ ($msg: tt, $($fmt: tt)*) => {
+ span_invalid_monomorphization_error(
+ bx.sess(), span,
+ &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg),
+ name, $($fmt)*));
+ }
+ }
+
+ macro_rules! return_error {
+ ($($fmt: tt)*) => {
+ {
+ emit_error!($($fmt)*);
+ return Err(());
+ }
+ }
+ }
+
+ macro_rules! require {
+ ($cond: expr, $($fmt: tt)*) => {
+ if !$cond {
+ return_error!($($fmt)*);
+ }
+ };
+ }
+
+ macro_rules! require_simd {
+ ($ty: expr, $position: expr) => {
+ require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty)
+ };
+ }
+
+ let tcx = bx.tcx();
+ let sig = tcx
+ .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &callee_ty.fn_sig(tcx));
+ let arg_tys = sig.inputs();
+ let name_str = &*name.as_str();
+
+ if name == sym::simd_select_bitmask {
+ let in_ty = arg_tys[0];
+ let m_len = match in_ty.kind {
+ // Note that this `.unwrap()` crashes for isize/usize, that's sort
+ // of intentional as there's not currently a use case for that.
+ ty::Int(i) => i.bit_width().unwrap(),
+ ty::Uint(i) => i.bit_width().unwrap(),
+ _ => return_error!("`{}` is not an integral type", in_ty),
+ };
+ require_simd!(arg_tys[1], "argument");
+ let v_len = arg_tys[1].simd_size(tcx);
+ require!(
+ m_len == v_len,
+ "mismatched lengths: mask length `{}` != other vector length `{}`",
+ m_len,
+ v_len
+ );
+ let i1 = bx.type_i1();
+ let i1xn = bx.type_vector(i1, m_len);
+ let m_i1s = bx.bitcast(args[0].immediate(), i1xn);
+ return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
+ }
+
+ // every intrinsic below takes a SIMD vector as its first argument
+ require_simd!(arg_tys[0], "input");
+ let in_ty = arg_tys[0];
+ let in_elem = arg_tys[0].simd_type(tcx);
+ let in_len = arg_tys[0].simd_size(tcx);
+
+ let comparison = match name {
+ sym::simd_eq => Some(hir::BinOpKind::Eq),
+ sym::simd_ne => Some(hir::BinOpKind::Ne),
+ sym::simd_lt => Some(hir::BinOpKind::Lt),
+ sym::simd_le => Some(hir::BinOpKind::Le),
+ sym::simd_gt => Some(hir::BinOpKind::Gt),
+ sym::simd_ge => Some(hir::BinOpKind::Ge),
+ _ => None,
+ };
+
+ if let Some(cmp_op) = comparison {
+ require_simd!(ret_ty, "return");
+
+ let out_len = ret_ty.simd_size(tcx);
+ require!(
+ in_len == out_len,
+ "expected return type with length {} (same as input type `{}`), \
+ found `{}` with length {}",
+ in_len,
+ in_ty,
+ ret_ty,
+ out_len
+ );
+ require!(
+ bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
+ "expected return type with integer elements, found `{}` with non-integer `{}`",
+ ret_ty,
+ ret_ty.simd_type(tcx)
+ );
+
+ return Ok(compare_simd_types(
+ bx,
+ args[0].immediate(),
+ args[1].immediate(),
+ in_elem,
+ llret_ty,
+ cmp_op,
+ ));
+ }
+
+ if name_str.starts_with("simd_shuffle") {
+ let n: u64 = name_str["simd_shuffle".len()..].parse().unwrap_or_else(|_| {
+ span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")
+ });
+
+ require_simd!(ret_ty, "return");
+
+ let out_len = ret_ty.simd_size(tcx);
+ require!(
+ out_len == n,
+ "expected return type of length {}, found `{}` with length {}",
+ n,
+ ret_ty,
+ out_len
+ );
+ require!(
+ in_elem == ret_ty.simd_type(tcx),
+ "expected return element type `{}` (element of input `{}`), \
+ found `{}` with element type `{}`",
+ in_elem,
+ in_ty,
+ ret_ty,
+ ret_ty.simd_type(tcx)
+ );
+
+ let total_len = u128::from(in_len) * 2;
+
+ let vector = args[2].immediate();
+
+ let indices: Option<Vec<_>> = (0..n)
+ .map(|i| {
+ let arg_idx = i;
+ let val = bx.const_get_elt(vector, i as u64);
+ match bx.const_to_opt_u128(val, true) {
+ None => {
+ emit_error!("shuffle index #{} is not a constant", arg_idx);
+ None
+ }
+ Some(idx) if idx >= total_len => {
+ emit_error!(
+ "shuffle index #{} is out of bounds (limit {})",
+ arg_idx,
+ total_len
+ );
+ None
+ }
+ Some(idx) => Some(bx.const_i32(idx as i32)),
+ }
+ })
+ .collect();
+ let indices = match indices {
+ Some(i) => i,
+ None => return Ok(bx.const_null(llret_ty)),
+ };
+
+ return Ok(bx.shuffle_vector(
+ args[0].immediate(),
+ args[1].immediate(),
+ bx.const_vector(&indices),
+ ));
+ }
+
+ if name == sym::simd_insert {
+ require!(
+ in_elem == arg_tys[2],
+ "expected inserted type `{}` (element of input `{}`), found `{}`",
+ in_elem,
+ in_ty,
+ arg_tys[2]
+ );
+ return Ok(bx.insert_element(
+ args[0].immediate(),
+ args[2].immediate(),
+ args[1].immediate(),
+ ));
+ }
+ if name == sym::simd_extract {
+ require!(
+ ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem,
+ in_ty,
+ ret_ty
+ );
+ return Ok(bx.extract_element(args[0].immediate(), args[1].immediate()));
+ }
+
+ if name == sym::simd_select {
+ let m_elem_ty = in_elem;
+ let m_len = in_len;
+ require_simd!(arg_tys[1], "argument");
+ let v_len = arg_tys[1].simd_size(tcx);
+ require!(
+ m_len == v_len,
+ "mismatched lengths: mask length `{}` != other vector length `{}`",
+ m_len,
+ v_len
+ );
+ match m_elem_ty.kind {
+ ty::Int(_) => {}
+ _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty),
+ }
+ // truncate the mask to a vector of i1s
+ let i1 = bx.type_i1();
+ let i1xn = bx.type_vector(i1, m_len as u64);
+ let m_i1s = bx.trunc(args[0].immediate(), i1xn);
+ return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
+ }
+
+ if name == sym::simd_bitmask {
+ // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a
+ // vector mask and returns an unsigned integer containing the most
+ // significant bit (MSB) of each lane.
+
+ // If the vector has less than 8 lanes, an u8 is returned with zeroed
+ // trailing bits.
+ let expected_int_bits = in_len.max(8);
+ match ret_ty.kind {
+ ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => (),
+ _ => return_error!("bitmask `{}`, expected `u{}`", ret_ty, expected_int_bits),
+ }
+
+ // Integer vector <i{in_bitwidth} x in_len>:
+ let (i_xn, in_elem_bitwidth) = match in_elem.kind {
+ ty::Int(i) => {
+ (args[0].immediate(), i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits()))
+ }
+ ty::Uint(i) => {
+ (args[0].immediate(), i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits()))
+ }
+ _ => return_error!(
+ "vector argument `{}`'s element type `{}`, expected integer element type",
+ in_ty,
+ in_elem
+ ),
+ };
+
+ // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position.
+ let shift_indices =
+ vec![
+ bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
+ in_len as _
+ ];
+ let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
+ // Truncate vector to an <i1 x N>
+ let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len));
+ // Bitcast <i1 x N> to iN:
+ let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
+ // Zero-extend iN to the bitmask type:
+ return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
+ }
+
+ fn simd_simple_float_intrinsic(
+ name: &str,
+ in_elem: &::rustc_middle::ty::TyS<'_>,
+ in_ty: &::rustc_middle::ty::TyS<'_>,
+ in_len: u64,
+ bx: &mut Builder<'a, 'll, 'tcx>,
+ span: Span,
+ args: &[OperandRef<'tcx, &'ll Value>],
+ ) -> Result<&'ll Value, ()> {
+ macro_rules! emit_error {
+ ($msg: tt) => {
+ emit_error!($msg, )
+ };
+ ($msg: tt, $($fmt: tt)*) => {
+ span_invalid_monomorphization_error(
+ bx.sess(), span,
+ &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg),
+ name, $($fmt)*));
+ }
+ }
+ macro_rules! return_error {
+ ($($fmt: tt)*) => {
+ {
+ emit_error!($($fmt)*);
+ return Err(());
+ }
+ }
+ }
+ let ety = match in_elem.kind {
+ ty::Float(f) if f.bit_width() == 32 => {
+ if in_len < 2 || in_len > 16 {
+ return_error!(
+ "unsupported floating-point vector `{}` with length `{}` \
+ out-of-range [2, 16]",
+ in_ty,
+ in_len
+ );
+ }
+ "f32"
+ }
+ ty::Float(f) if f.bit_width() == 64 => {
+ if in_len < 2 || in_len > 8 {
+ return_error!(
+ "unsupported floating-point vector `{}` with length `{}` \
+ out-of-range [2, 8]",
+ in_ty,
+ in_len
+ );
+ }
+ "f64"
+ }
+ ty::Float(f) => {
+ return_error!(
+ "unsupported element type `{}` of floating-point vector `{}`",
+ f.name_str(),
+ in_ty
+ );
+ }
+ _ => {
+ return_error!("`{}` is not a floating-point type", in_ty);
+ }
+ };
+
+ let llvm_name = &format!("llvm.{0}.v{1}{2}", name, in_len, ety);
+ let intrinsic = bx.get_intrinsic(&llvm_name);
+ let c =
+ bx.call(intrinsic, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
+ unsafe { llvm::LLVMRustSetHasUnsafeAlgebra(c) };
+ Ok(c)
+ }
+
+ match name {
+ sym::simd_fsqrt => {
+ return simd_simple_float_intrinsic("sqrt", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_fsin => {
+ return simd_simple_float_intrinsic("sin", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_fcos => {
+ return simd_simple_float_intrinsic("cos", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_fabs => {
+ return simd_simple_float_intrinsic("fabs", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_floor => {
+ return simd_simple_float_intrinsic("floor", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_ceil => {
+ return simd_simple_float_intrinsic("ceil", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_fexp => {
+ return simd_simple_float_intrinsic("exp", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_fexp2 => {
+ return simd_simple_float_intrinsic("exp2", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_flog10 => {
+ return simd_simple_float_intrinsic("log10", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_flog2 => {
+ return simd_simple_float_intrinsic("log2", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_flog => {
+ return simd_simple_float_intrinsic("log", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_fpowi => {
+ return simd_simple_float_intrinsic("powi", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_fpow => {
+ return simd_simple_float_intrinsic("pow", in_elem, in_ty, in_len, bx, span, args);
+ }
+ sym::simd_fma => {
+ return simd_simple_float_intrinsic("fma", in_elem, in_ty, in_len, bx, span, args);
+ }
+ _ => { /* fallthrough */ }
+ }
+
+ // FIXME: use:
+ // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182
+ // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81
+ fn llvm_vector_str(elem_ty: Ty<'_>, vec_len: u64, no_pointers: usize) -> String {
+ let p0s: String = "p0".repeat(no_pointers);
+ match elem_ty.kind {
+ ty::Int(v) => format!("v{}{}i{}", vec_len, p0s, v.bit_width().unwrap()),
+ ty::Uint(v) => format!("v{}{}i{}", vec_len, p0s, v.bit_width().unwrap()),
+ ty::Float(v) => format!("v{}{}f{}", vec_len, p0s, v.bit_width()),
+ _ => unreachable!(),
+ }
+ }
+
+ fn llvm_vector_ty(
+ cx: &CodegenCx<'ll, '_>,
+ elem_ty: Ty<'_>,
+ vec_len: u64,
+ mut no_pointers: usize,
+ ) -> &'ll Type {
+ // FIXME: use cx.layout_of(ty).llvm_type() ?
+ let mut elem_ty = match elem_ty.kind {
+ ty::Int(v) => cx.type_int_from_ty(v),
+ ty::Uint(v) => cx.type_uint_from_ty(v),
+ ty::Float(v) => cx.type_float_from_ty(v),
+ _ => unreachable!(),
+ };
+ while no_pointers > 0 {
+ elem_ty = cx.type_ptr_to(elem_ty);
+ no_pointers -= 1;
+ }
+ cx.type_vector(elem_ty, vec_len)
+ }
+
+ if name == sym::simd_gather {
+ // simd_gather(values: <N x T>, pointers: <N x *_ T>,
+ // mask: <N x i{M}>) -> <N x T>
+ // * N: number of elements in the input vectors
+ // * T: type of the element to load
+ // * M: any integer width is supported, will be truncated to i1
+
+ // All types must be simd vector types
+ require_simd!(in_ty, "first");
+ require_simd!(arg_tys[1], "second");
+ require_simd!(arg_tys[2], "third");
+ require_simd!(ret_ty, "return");
+
+ // Of the same length:
+ require!(
+ in_len == arg_tys[1].simd_size(tcx),
+ "expected {} argument with length {} (same as input type `{}`), \
+ found `{}` with length {}",
+ "second",
+ in_len,
+ in_ty,
+ arg_tys[1],
+ arg_tys[1].simd_size(tcx)
+ );
+ require!(
+ in_len == arg_tys[2].simd_size(tcx),
+ "expected {} argument with length {} (same as input type `{}`), \
+ found `{}` with length {}",
+ "third",
+ in_len,
+ in_ty,
+ arg_tys[2],
+ arg_tys[2].simd_size(tcx)
+ );
+
+ // The return type must match the first argument type
+ require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty);
+
+ // This counts how many pointers
+ fn ptr_count(t: Ty<'_>) -> usize {
+ match t.kind {
+ ty::RawPtr(p) => 1 + ptr_count(p.ty),
+ _ => 0,
+ }
+ }
+
+ // Non-ptr type
+ fn non_ptr(t: Ty<'_>) -> Ty<'_> {
+ match t.kind {
+ ty::RawPtr(p) => non_ptr(p.ty),
+ _ => t,
+ }
+ }
+
+ // The second argument must be a simd vector with an element type that's a pointer
+ // to the element type of the first argument
+ let (pointer_count, underlying_ty) = match arg_tys[1].simd_type(tcx).kind {
+ ty::RawPtr(p) if p.ty == in_elem => {
+ (ptr_count(arg_tys[1].simd_type(tcx)), non_ptr(arg_tys[1].simd_type(tcx)))
+ }
+ _ => {
+ require!(
+ false,
+ "expected element type `{}` of second argument `{}` \
+ to be a pointer to the element type `{}` of the first \
+ argument `{}`, found `{}` != `*_ {}`",
+ arg_tys[1].simd_type(tcx),
+ arg_tys[1],
+ in_elem,
+ in_ty,
+ arg_tys[1].simd_type(tcx),
+ in_elem
+ );
+ unreachable!();
+ }
+ };
+ assert!(pointer_count > 0);
+ assert_eq!(pointer_count - 1, ptr_count(arg_tys[0].simd_type(tcx)));
+ assert_eq!(underlying_ty, non_ptr(arg_tys[0].simd_type(tcx)));
+
+ // The element type of the third argument must be a signed integer type of any width:
+ match arg_tys[2].simd_type(tcx).kind {
+ ty::Int(_) => (),
+ _ => {
+ require!(
+ false,
+ "expected element type `{}` of third argument `{}` \
+ to be a signed integer type",
+ arg_tys[2].simd_type(tcx),
+ arg_tys[2]
+ );
+ }
+ }
+
+ // Alignment of T, must be a constant integer value:
+ let alignment_ty = bx.type_i32();
+ let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
+
+ // Truncate the mask vector to a vector of i1s:
+ let (mask, mask_ty) = {
+ let i1 = bx.type_i1();
+ let i1xn = bx.type_vector(i1, in_len);
+ (bx.trunc(args[2].immediate(), i1xn), i1xn)
+ };
+
+ // Type of the vector of pointers:
+ let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count);
+ let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count);
+
+ // Type of the vector of elements:
+ let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1);
+ let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1);
+
+ let llvm_intrinsic =
+ format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str);
+ let f = bx.declare_cfn(
+ &llvm_intrinsic,
+ bx.type_func(
+ &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty],
+ llvm_elem_vec_ty,
+ ),
+ );
+ llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No);
+ let v = bx.call(f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None);
+ return Ok(v);
+ }
+
+ if name == sym::simd_scatter {
+ // simd_scatter(values: <N x T>, pointers: <N x *mut T>,
+ // mask: <N x i{M}>) -> ()
+ // * N: number of elements in the input vectors
+ // * T: type of the element to load
+ // * M: any integer width is supported, will be truncated to i1
+
+ // All types must be simd vector types
+ require_simd!(in_ty, "first");
+ require_simd!(arg_tys[1], "second");
+ require_simd!(arg_tys[2], "third");
+
+ // Of the same length:
+ require!(
+ in_len == arg_tys[1].simd_size(tcx),
+ "expected {} argument with length {} (same as input type `{}`), \
+ found `{}` with length {}",
+ "second",
+ in_len,
+ in_ty,
+ arg_tys[1],
+ arg_tys[1].simd_size(tcx)
+ );
+ require!(
+ in_len == arg_tys[2].simd_size(tcx),
+ "expected {} argument with length {} (same as input type `{}`), \
+ found `{}` with length {}",
+ "third",
+ in_len,
+ in_ty,
+ arg_tys[2],
+ arg_tys[2].simd_size(tcx)
+ );
+
+ // This counts how many pointers
+ fn ptr_count(t: Ty<'_>) -> usize {
+ match t.kind {
+ ty::RawPtr(p) => 1 + ptr_count(p.ty),
+ _ => 0,
+ }
+ }
+
+ // Non-ptr type
+ fn non_ptr(t: Ty<'_>) -> Ty<'_> {
+ match t.kind {
+ ty::RawPtr(p) => non_ptr(p.ty),
+ _ => t,
+ }
+ }
+
+ // The second argument must be a simd vector with an element type that's a pointer
+ // to the element type of the first argument
+ let (pointer_count, underlying_ty) = match arg_tys[1].simd_type(tcx).kind {
+ ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => {
+ (ptr_count(arg_tys[1].simd_type(tcx)), non_ptr(arg_tys[1].simd_type(tcx)))
+ }
+ _ => {
+ require!(
+ false,
+ "expected element type `{}` of second argument `{}` \
+ to be a pointer to the element type `{}` of the first \
+ argument `{}`, found `{}` != `*mut {}`",
+ arg_tys[1].simd_type(tcx),
+ arg_tys[1],
+ in_elem,
+ in_ty,
+ arg_tys[1].simd_type(tcx),
+ in_elem
+ );
+ unreachable!();
+ }
+ };
+ assert!(pointer_count > 0);
+ assert_eq!(pointer_count - 1, ptr_count(arg_tys[0].simd_type(tcx)));
+ assert_eq!(underlying_ty, non_ptr(arg_tys[0].simd_type(tcx)));
+
+ // The element type of the third argument must be a signed integer type of any width:
+ match arg_tys[2].simd_type(tcx).kind {
+ ty::Int(_) => (),
+ _ => {
+ require!(
+ false,
+ "expected element type `{}` of third argument `{}` \
+ to be a signed integer type",
+ arg_tys[2].simd_type(tcx),
+ arg_tys[2]
+ );
+ }
+ }
+
+ // Alignment of T, must be a constant integer value:
+ let alignment_ty = bx.type_i32();
+ let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
+
+ // Truncate the mask vector to a vector of i1s:
+ let (mask, mask_ty) = {
+ let i1 = bx.type_i1();
+ let i1xn = bx.type_vector(i1, in_len);
+ (bx.trunc(args[2].immediate(), i1xn), i1xn)
+ };
+
+ let ret_t = bx.type_void();
+
+ // Type of the vector of pointers:
+ let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count);
+ let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count);
+
+ // Type of the vector of elements:
+ let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1);
+ let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1);
+
+ let llvm_intrinsic =
+ format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str);
+ let f = bx.declare_cfn(
+ &llvm_intrinsic,
+ bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t),
+ );
+ llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No);
+ let v = bx.call(f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None);
+ return Ok(v);
+ }
+
+ macro_rules! arith_red {
+ ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
+ $identity:expr) => {
+ if name == sym::$name {
+ require!(
+ ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem,
+ in_ty,
+ ret_ty
+ );
+ return match in_elem.kind {
+ ty::Int(_) | ty::Uint(_) => {
+ let r = bx.$integer_reduce(args[0].immediate());
+ if $ordered {
+ // if overflow occurs, the result is the
+ // mathematical result modulo 2^n:
+ Ok(bx.$op(args[1].immediate(), r))
+ } else {
+ Ok(bx.$integer_reduce(args[0].immediate()))
+ }
+ }
+ ty::Float(f) => {
+ let acc = if $ordered {
+ // ordered arithmetic reductions take an accumulator
+ args[1].immediate()
+ } else {
+ // unordered arithmetic reductions use the identity accumulator
+ match f.bit_width() {
+ 32 => bx.const_real(bx.type_f32(), $identity),
+ 64 => bx.const_real(bx.type_f64(), $identity),
+ v => return_error!(
+ r#"
+unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#,
+ sym::$name,
+ in_ty,
+ in_elem,
+ v,
+ ret_ty
+ ),
+ }
+ };
+ Ok(bx.$float_reduce(acc, args[0].immediate()))
+ }
+ _ => return_error!(
+ "unsupported {} from `{}` with element `{}` to `{}`",
+ sym::$name,
+ in_ty,
+ in_elem,
+ ret_ty
+ ),
+ };
+ }
+ };
+ }
+
+ arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, 0.0);
+ arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
+ arith_red!(
+ simd_reduce_add_unordered: vector_reduce_add,
+ vector_reduce_fadd_fast,
+ false,
+ add,
+ 0.0
+ );
+ arith_red!(
+ simd_reduce_mul_unordered: vector_reduce_mul,
+ vector_reduce_fmul_fast,
+ false,
+ mul,
+ 1.0
+ );
+
+ macro_rules! minmax_red {
+ ($name:ident: $int_red:ident, $float_red:ident) => {
+ if name == sym::$name {
+ require!(
+ ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem,
+ in_ty,
+ ret_ty
+ );
+ return match in_elem.kind {
+ ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
+ ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
+ ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())),
+ _ => return_error!(
+ "unsupported {} from `{}` with element `{}` to `{}`",
+ sym::$name,
+ in_ty,
+ in_elem,
+ ret_ty
+ ),
+ };
+ }
+ };
+ }
+
+ minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
+ minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
+
+ minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast);
+ minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast);
+
+ macro_rules! bitwise_red {
+ ($name:ident : $red:ident, $boolean:expr) => {
+ if name == sym::$name {
+ let input = if !$boolean {
+ require!(
+ ret_ty == in_elem,
+ "expected return type `{}` (element of input `{}`), found `{}`",
+ in_elem,
+ in_ty,
+ ret_ty
+ );
+ args[0].immediate()
+ } else {
+ match in_elem.kind {
+ ty::Int(_) | ty::Uint(_) => {}
+ _ => return_error!(
+ "unsupported {} from `{}` with element `{}` to `{}`",
+ sym::$name,
+ in_ty,
+ in_elem,
+ ret_ty
+ ),
+ }
+
+ // boolean reductions operate on vectors of i1s:
+ let i1 = bx.type_i1();
+ let i1xn = bx.type_vector(i1, in_len as u64);
+ bx.trunc(args[0].immediate(), i1xn)
+ };
+ return match in_elem.kind {
+ ty::Int(_) | ty::Uint(_) => {
+ let r = bx.$red(input);
+ Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
+ }
+ _ => return_error!(
+ "unsupported {} from `{}` with element `{}` to `{}`",
+ sym::$name,
+ in_ty,
+ in_elem,
+ ret_ty
+ ),
+ };
+ }
+ };
+ }
+
+ bitwise_red!(simd_reduce_and: vector_reduce_and, false);
+ bitwise_red!(simd_reduce_or: vector_reduce_or, false);
+ bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
+ bitwise_red!(simd_reduce_all: vector_reduce_and, true);
+ bitwise_red!(simd_reduce_any: vector_reduce_or, true);
+
+ if name == sym::simd_cast {
+ require_simd!(ret_ty, "return");
+ let out_len = ret_ty.simd_size(tcx);
+ require!(
+ in_len == out_len,
+ "expected return type with length {} (same as input type `{}`), \
+ found `{}` with length {}",
+ in_len,
+ in_ty,
+ ret_ty,
+ out_len
+ );
+ // casting cares about nominal type, not just structural type
+ let out_elem = ret_ty.simd_type(tcx);
+
+ if in_elem == out_elem {
+ return Ok(args[0].immediate());
+ }
+
+ enum Style {
+ Float,
+ Int(/* is signed? */ bool),
+ Unsupported,
+ }
+
+ let (in_style, in_width) = match in_elem.kind {
+ // vectors of pointer-sized integers should've been
+ // disallowed before here, so this unwrap is safe.
+ ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()),
+ ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()),
+ ty::Float(f) => (Style::Float, f.bit_width()),
+ _ => (Style::Unsupported, 0),
+ };
+ let (out_style, out_width) = match out_elem.kind {
+ ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()),
+ ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()),
+ ty::Float(f) => (Style::Float, f.bit_width()),
+ _ => (Style::Unsupported, 0),
+ };
+
+ match (in_style, out_style) {
+ (Style::Int(in_is_signed), Style::Int(_)) => {
+ return Ok(match in_width.cmp(&out_width) {
+ Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
+ Ordering::Equal => args[0].immediate(),
+ Ordering::Less => {
+ if in_is_signed {
+ bx.sext(args[0].immediate(), llret_ty)
+ } else {
+ bx.zext(args[0].immediate(), llret_ty)
+ }
+ }
+ });
+ }
+ (Style::Int(in_is_signed), Style::Float) => {
+ return Ok(if in_is_signed {
+ bx.sitofp(args[0].immediate(), llret_ty)
+ } else {
+ bx.uitofp(args[0].immediate(), llret_ty)
+ });
+ }
+ (Style::Float, Style::Int(out_is_signed)) => {
+ return Ok(if out_is_signed {
+ bx.fptosi(args[0].immediate(), llret_ty)
+ } else {
+ bx.fptoui(args[0].immediate(), llret_ty)
+ });
+ }
+ (Style::Float, Style::Float) => {
+ return Ok(match in_width.cmp(&out_width) {
+ Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
+ Ordering::Equal => args[0].immediate(),
+ Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
+ });
+ }
+ _ => { /* Unsupported. Fallthrough. */ }
+ }
+ require!(
+ false,
+ "unsupported cast from `{}` with element `{}` to `{}` with element `{}`",
+ in_ty,
+ in_elem,
+ ret_ty,
+ out_elem
+ );
+ }
+ macro_rules! arith {
+ ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
+ $(if name == sym::$name {
+ match in_elem.kind {
+ $($(ty::$p(_))|* => {
+ return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
+ })*
+ _ => {},
+ }
+ require!(false,
+ "unsupported operation on `{}` with element `{}`",
+ in_ty,
+ in_elem)
+ })*
+ }
+ }
+ arith! {
+ simd_add: Uint, Int => add, Float => fadd;
+ simd_sub: Uint, Int => sub, Float => fsub;
+ simd_mul: Uint, Int => mul, Float => fmul;
+ simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
+ simd_rem: Uint => urem, Int => srem, Float => frem;
+ simd_shl: Uint, Int => shl;
+ simd_shr: Uint => lshr, Int => ashr;
+ simd_and: Uint, Int => and;
+ simd_or: Uint, Int => or;
+ simd_xor: Uint, Int => xor;
+ simd_fmax: Float => maxnum;
+ simd_fmin: Float => minnum;
+
+ }
+
+ if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
+ let lhs = args[0].immediate();
+ let rhs = args[1].immediate();
+ let is_add = name == sym::simd_saturating_add;
+ let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
+ let (signed, elem_width, elem_ty) = match in_elem.kind {
+ ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)),
+ ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)),
+ _ => {
+ return_error!(
+ "expected element type `{}` of vector type `{}` \
+ to be a signed or unsigned integer type",
+ arg_tys[0].simd_type(tcx),
+ arg_tys[0]
+ );
+ }
+ };
+ let llvm_intrinsic = &format!(
+ "llvm.{}{}.sat.v{}i{}",
+ if signed { 's' } else { 'u' },
+ if is_add { "add" } else { "sub" },
+ in_len,
+ elem_width
+ );
+ let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
+
+ let f = bx.declare_cfn(&llvm_intrinsic, bx.type_func(&[vec_ty, vec_ty], vec_ty));
+ llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No);
+ let v = bx.call(f, &[lhs, rhs], None);
+ return Ok(v);
+ }
+
+ span_bug!(span, "unknown SIMD intrinsic");
+}
+
+// Returns the width of an int Ty, and if it's signed or not
+// Returns None if the type is not an integer
+// FIXME: there’s multiple of this functions, investigate using some of the already existing
+// stuffs.
+fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> {
+ match ty.kind {
+ ty::Int(t) => Some((
+ match t {
+ ast::IntTy::Isize => u64::from(cx.tcx.sess.target.ptr_width),
+ ast::IntTy::I8 => 8,
+ ast::IntTy::I16 => 16,
+ ast::IntTy::I32 => 32,
+ ast::IntTy::I64 => 64,
+ ast::IntTy::I128 => 128,
+ },
+ true,
+ )),
+ ty::Uint(t) => Some((
+ match t {
+ ast::UintTy::Usize => u64::from(cx.tcx.sess.target.ptr_width),
+ ast::UintTy::U8 => 8,
+ ast::UintTy::U16 => 16,
+ ast::UintTy::U32 => 32,
+ ast::UintTy::U64 => 64,
+ ast::UintTy::U128 => 128,
+ },
+ false,
+ )),
+ _ => None,
+ }
+}
+
+// Returns the width of a float Ty
+// Returns None if the type is not a float
+fn float_type_width(ty: Ty<'_>) -> Option<u64> {
+ match ty.kind {
+ ty::Float(t) => Some(t.bit_width()),
+ _ => None,
+ }
+}
--- /dev/null
- use rustc_middle::ty::layout::{self, HasTyCtxt, TyAndLayout};
+//! Codegen the completed AST to the LLVM IR.
+//!
+//! Some functions here, such as `codegen_block` and `codegen_expr`, return a value --
+//! the result of the codegen to LLVM -- while others, such as `codegen_fn`
+//! and `mono_item`, are called only for the side effect of adding a
+//! particular definition to the LLVM IR output we're producing.
+//!
+//! Hopefully useful general knowledge about codegen:
+//!
+//! * There's no way to find out the `Ty` type of a `Value`. Doing so
+//! would be "trying to get the eggs out of an omelette" (credit:
+//! pcwalton). You can, instead, find out its `llvm::Type` by calling `val_ty`,
+//! but one `llvm::Type` corresponds to many `Ty`s; for instance, `tup(int, int,
+//! int)` and `rec(x=int, y=int, z=int)` will have the same `llvm::Type`.
+
+use crate::back::write::{
+ compute_per_cgu_lto_type, start_async_codegen, submit_codegened_module_to_llvm,
+ submit_post_lto_module_to_llvm, submit_pre_lto_module_to_llvm, ComputedLtoType, OngoingCodegen,
+};
+use crate::common::{IntPredicate, RealPredicate, TypeKind};
+use crate::meth;
+use crate::mir;
+use crate::mir::operand::OperandValue;
+use crate::mir::place::PlaceRef;
+use crate::traits::*;
+use crate::{CachedModuleCodegen, CrateInfo, MemFlags, ModuleCodegen, ModuleKind};
+
+use rustc_attr as attr;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::profiling::print_time_passes_entry;
+use rustc_data_structures::sync::{par_iter, Lock, ParallelIterator};
+use rustc_hir as hir;
+use rustc_hir::def_id::{LocalDefId, LOCAL_CRATE};
+use rustc_hir::lang_items::LangItem;
+use rustc_index::vec::Idx;
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
+use rustc_middle::middle::cstore::EncodedMetadata;
+use rustc_middle::middle::cstore::{self, LinkagePreference};
+use rustc_middle::middle::lang_items;
+use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem};
- use rustc_target::abi::{Abi, Align, LayoutOf, Scalar, VariantIdx};
++use rustc_middle::ty::layout::{HasTyCtxt, TyAndLayout};
+use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
+use rustc_middle::ty::query::Providers;
+use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
+use rustc_session::cgu_reuse_tracker::CguReuse;
+use rustc_session::config::{self, EntryFnType};
+use rustc_session::utils::NativeLibKind;
+use rustc_session::Session;
+use rustc_span::Span;
+use rustc_symbol_mangling::test as symbol_names_test;
- pub fn from_immediate<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
- bx: &mut Bx,
- val: Bx::Value,
- ) -> Bx::Value {
- if bx.cx().val_ty(val) == bx.cx().type_i1() { bx.zext(val, bx.cx().type_i8()) } else { val }
- }
-
- pub fn to_immediate<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
- bx: &mut Bx,
- val: Bx::Value,
- layout: layout::TyAndLayout<'_>,
- ) -> Bx::Value {
- if let Abi::Scalar(ref scalar) = layout.abi {
- return to_immediate_scalar(bx, val, scalar);
- }
- val
- }
-
- pub fn to_immediate_scalar<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
- bx: &mut Bx,
- val: Bx::Value,
- scalar: &Scalar,
- ) -> Bx::Value {
- if scalar.is_bool() {
- return bx.trunc(val, bx.cx().type_i1());
- }
- val
- }
-
++use rustc_target::abi::{Align, LayoutOf, VariantIdx};
+
+use std::cmp;
+use std::ops::{Deref, DerefMut};
+use std::time::{Duration, Instant};
+
+pub fn bin_op_to_icmp_predicate(op: hir::BinOpKind, signed: bool) -> IntPredicate {
+ match op {
+ hir::BinOpKind::Eq => IntPredicate::IntEQ,
+ hir::BinOpKind::Ne => IntPredicate::IntNE,
+ hir::BinOpKind::Lt => {
+ if signed {
+ IntPredicate::IntSLT
+ } else {
+ IntPredicate::IntULT
+ }
+ }
+ hir::BinOpKind::Le => {
+ if signed {
+ IntPredicate::IntSLE
+ } else {
+ IntPredicate::IntULE
+ }
+ }
+ hir::BinOpKind::Gt => {
+ if signed {
+ IntPredicate::IntSGT
+ } else {
+ IntPredicate::IntUGT
+ }
+ }
+ hir::BinOpKind::Ge => {
+ if signed {
+ IntPredicate::IntSGE
+ } else {
+ IntPredicate::IntUGE
+ }
+ }
+ op => bug!(
+ "comparison_op_to_icmp_predicate: expected comparison operator, \
+ found {:?}",
+ op
+ ),
+ }
+}
+
+pub fn bin_op_to_fcmp_predicate(op: hir::BinOpKind) -> RealPredicate {
+ match op {
+ hir::BinOpKind::Eq => RealPredicate::RealOEQ,
+ hir::BinOpKind::Ne => RealPredicate::RealUNE,
+ hir::BinOpKind::Lt => RealPredicate::RealOLT,
+ hir::BinOpKind::Le => RealPredicate::RealOLE,
+ hir::BinOpKind::Gt => RealPredicate::RealOGT,
+ hir::BinOpKind::Ge => RealPredicate::RealOGE,
+ op => {
+ bug!(
+ "comparison_op_to_fcmp_predicate: expected comparison operator, \
+ found {:?}",
+ op
+ );
+ }
+ }
+}
+
+pub fn compare_simd_types<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ lhs: Bx::Value,
+ rhs: Bx::Value,
+ t: Ty<'tcx>,
+ ret_ty: Bx::Type,
+ op: hir::BinOpKind,
+) -> Bx::Value {
+ let signed = match t.kind {
+ ty::Float(_) => {
+ let cmp = bin_op_to_fcmp_predicate(op);
+ let cmp = bx.fcmp(cmp, lhs, rhs);
+ return bx.sext(cmp, ret_ty);
+ }
+ ty::Uint(_) => false,
+ ty::Int(_) => true,
+ _ => bug!("compare_simd_types: invalid SIMD type"),
+ };
+
+ let cmp = bin_op_to_icmp_predicate(op, signed);
+ let cmp = bx.icmp(cmp, lhs, rhs);
+ // LLVM outputs an `< size x i1 >`, so we need to perform a sign extension
+ // to get the correctly sized type. This will compile to a single instruction
+ // once the IR is converted to assembly if the SIMD instruction is supported
+ // by the target architecture.
+ bx.sext(cmp, ret_ty)
+}
+
+/// Retrieves the information we are losing (making dynamic) in an unsizing
+/// adjustment.
+///
+/// The `old_info` argument is a bit odd. It is intended for use in an upcast,
+/// where the new vtable for an object will be derived from the old one.
+pub fn unsized_info<'tcx, Cx: CodegenMethods<'tcx>>(
+ cx: &Cx,
+ source: Ty<'tcx>,
+ target: Ty<'tcx>,
+ old_info: Option<Cx::Value>,
+) -> Cx::Value {
+ let (source, target) =
+ cx.tcx().struct_lockstep_tails_erasing_lifetimes(source, target, cx.param_env());
+ match (&source.kind, &target.kind) {
+ (&ty::Array(_, len), &ty::Slice(_)) => {
+ cx.const_usize(len.eval_usize(cx.tcx(), ty::ParamEnv::reveal_all()))
+ }
+ (&ty::Dynamic(..), &ty::Dynamic(..)) => {
+ // For now, upcasts are limited to changes in marker
+ // traits, and hence never actually require an actual
+ // change to the vtable.
+ old_info.expect("unsized_info: missing old info for trait upcast")
+ }
+ (_, &ty::Dynamic(ref data, ..)) => {
+ let vtable_ptr = cx.layout_of(cx.tcx().mk_mut_ptr(target)).field(cx, FAT_PTR_EXTRA);
+ cx.const_ptrcast(
+ meth::get_vtable(cx, source, data.principal()),
+ cx.backend_type(vtable_ptr),
+ )
+ }
+ _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target),
+ }
+}
+
+/// Coerces `src` to `dst_ty`. `src_ty` must be a thin pointer.
+pub fn unsize_thin_ptr<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ src: Bx::Value,
+ src_ty: Ty<'tcx>,
+ dst_ty: Ty<'tcx>,
+) -> (Bx::Value, Bx::Value) {
+ debug!("unsize_thin_ptr: {:?} => {:?}", src_ty, dst_ty);
+ match (&src_ty.kind, &dst_ty.kind) {
+ (&ty::Ref(_, a, _), &ty::Ref(_, b, _) | &ty::RawPtr(ty::TypeAndMut { ty: b, .. }))
+ | (&ty::RawPtr(ty::TypeAndMut { ty: a, .. }), &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) => {
+ assert!(bx.cx().type_is_sized(a));
+ let ptr_ty = bx.cx().type_ptr_to(bx.cx().backend_type(bx.cx().layout_of(b)));
+ (bx.pointercast(src, ptr_ty), unsized_info(bx.cx(), a, b, None))
+ }
+ (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {
+ assert_eq!(def_a, def_b);
+
+ let src_layout = bx.cx().layout_of(src_ty);
+ let dst_layout = bx.cx().layout_of(dst_ty);
+ let mut result = None;
+ for i in 0..src_layout.fields.count() {
+ let src_f = src_layout.field(bx.cx(), i);
+ assert_eq!(src_layout.fields.offset(i).bytes(), 0);
+ assert_eq!(dst_layout.fields.offset(i).bytes(), 0);
+ if src_f.is_zst() {
+ continue;
+ }
+ assert_eq!(src_layout.size, src_f.size);
+
+ let dst_f = dst_layout.field(bx.cx(), i);
+ assert_ne!(src_f.ty, dst_f.ty);
+ assert_eq!(result, None);
+ result = Some(unsize_thin_ptr(bx, src, src_f.ty, dst_f.ty));
+ }
+ let (lldata, llextra) = result.unwrap();
+ // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
+ // FIXME(eddyb) move these out of this `match` arm, so they're always
+ // applied, uniformly, no matter the source/destination types.
+ (
+ bx.bitcast(lldata, bx.cx().scalar_pair_element_backend_type(dst_layout, 0, true)),
+ bx.bitcast(llextra, bx.cx().scalar_pair_element_backend_type(dst_layout, 1, true)),
+ )
+ }
+ _ => bug!("unsize_thin_ptr: called on bad types"),
+ }
+}
+
+/// Coerces `src`, which is a reference to a value of type `src_ty`,
+/// to a value of type `dst_ty`, and stores the result in `dst`.
+pub fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ src: PlaceRef<'tcx, Bx::Value>,
+ dst: PlaceRef<'tcx, Bx::Value>,
+) {
+ let src_ty = src.layout.ty;
+ let dst_ty = dst.layout.ty;
+ match (&src_ty.kind, &dst_ty.kind) {
+ (&ty::Ref(..), &ty::Ref(..) | &ty::RawPtr(..)) | (&ty::RawPtr(..), &ty::RawPtr(..)) => {
+ let (base, info) = match bx.load_operand(src).val {
+ OperandValue::Pair(base, info) => {
+ // fat-ptr to fat-ptr unsize preserves the vtable
+ // i.e., &'a fmt::Debug+Send => &'a fmt::Debug
+ // So we need to pointercast the base to ensure
+ // the types match up.
+ // FIXME(eddyb) use `scalar_pair_element_backend_type` here,
+ // like `unsize_thin_ptr` does.
+ let thin_ptr = dst.layout.field(bx.cx(), FAT_PTR_ADDR);
+ (bx.pointercast(base, bx.cx().backend_type(thin_ptr)), info)
+ }
+ OperandValue::Immediate(base) => unsize_thin_ptr(bx, base, src_ty, dst_ty),
+ OperandValue::Ref(..) => bug!(),
+ };
+ OperandValue::Pair(base, info).store(bx, dst);
+ }
+
+ (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {
+ assert_eq!(def_a, def_b);
+
+ for i in 0..def_a.variants[VariantIdx::new(0)].fields.len() {
+ let src_f = src.project_field(bx, i);
+ let dst_f = dst.project_field(bx, i);
+
+ if dst_f.layout.is_zst() {
+ continue;
+ }
+
+ if src_f.layout.ty == dst_f.layout.ty {
+ memcpy_ty(
+ bx,
+ dst_f.llval,
+ dst_f.align,
+ src_f.llval,
+ src_f.align,
+ src_f.layout,
+ MemFlags::empty(),
+ );
+ } else {
+ coerce_unsized_into(bx, src_f, dst_f);
+ }
+ }
+ }
+ _ => bug!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty, dst_ty,),
+ }
+}
+
+pub fn cast_shift_expr_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ op: hir::BinOpKind,
+ lhs: Bx::Value,
+ rhs: Bx::Value,
+) -> Bx::Value {
+ cast_shift_rhs(bx, op, lhs, rhs)
+}
+
+fn cast_shift_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ op: hir::BinOpKind,
+ lhs: Bx::Value,
+ rhs: Bx::Value,
+) -> Bx::Value {
+ // Shifts may have any size int on the rhs
+ if op.is_shift() {
+ let mut rhs_llty = bx.cx().val_ty(rhs);
+ let mut lhs_llty = bx.cx().val_ty(lhs);
+ if bx.cx().type_kind(rhs_llty) == TypeKind::Vector {
+ rhs_llty = bx.cx().element_type(rhs_llty)
+ }
+ if bx.cx().type_kind(lhs_llty) == TypeKind::Vector {
+ lhs_llty = bx.cx().element_type(lhs_llty)
+ }
+ let rhs_sz = bx.cx().int_width(rhs_llty);
+ let lhs_sz = bx.cx().int_width(lhs_llty);
+ if lhs_sz < rhs_sz {
+ bx.trunc(rhs, lhs_llty)
+ } else if lhs_sz > rhs_sz {
+ // FIXME (#1877: If in the future shifting by negative
+ // values is no longer undefined then this is wrong.
+ bx.zext(rhs, lhs_llty)
+ } else {
+ rhs
+ }
+ } else {
+ rhs
+ }
+}
+
+/// Returns `true` if this session's target will use SEH-based unwinding.
+///
+/// This is only true for MSVC targets, and even then the 64-bit MSVC target
+/// currently uses SEH-ish unwinding with DWARF info tables to the side (same as
+/// 64-bit MinGW) instead of "full SEH".
+pub fn wants_msvc_seh(sess: &Session) -> bool {
+ sess.target.target.options.is_like_msvc
+}
+
+pub fn memcpy_ty<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ dst: Bx::Value,
+ dst_align: Align,
+ src: Bx::Value,
+ src_align: Align,
+ layout: TyAndLayout<'tcx>,
+ flags: MemFlags,
+) {
+ let size = layout.size.bytes();
+ if size == 0 {
+ return;
+ }
+
+ bx.memcpy(dst, dst_align, src, src_align, bx.cx().const_usize(size), flags);
+}
+
+pub fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
+ cx: &'a Bx::CodegenCx,
+ instance: Instance<'tcx>,
+) {
+ // this is an info! to allow collecting monomorphization statistics
+ // and to allow finding the last function before LLVM aborts from
+ // release builds.
+ info!("codegen_instance({})", instance);
+
+ mir::codegen_mir::<Bx>(cx, instance);
+}
+
+/// Creates the `main` function which will initialize the rust runtime and call
+/// users main function.
+pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ cx: &'a Bx::CodegenCx,
+) -> Option<Bx::Function> {
+ let (main_def_id, span) = match cx.tcx().entry_fn(LOCAL_CRATE) {
+ Some((def_id, _)) => (def_id, cx.tcx().def_span(def_id)),
+ None => return None,
+ };
+
+ let instance = Instance::mono(cx.tcx(), main_def_id.to_def_id());
+
+ if !cx.codegen_unit().contains_item(&MonoItem::Fn(instance)) {
+ // We want to create the wrapper in the same codegen unit as Rust's main
+ // function.
+ return None;
+ }
+
+ let main_llfn = cx.get_fn_addr(instance);
+
+ return cx.tcx().entry_fn(LOCAL_CRATE).map(|(_, et)| {
+ let use_start_lang_item = EntryFnType::Start != et;
+ create_entry_fn::<Bx>(cx, span, main_llfn, main_def_id, use_start_lang_item)
+ });
+
+ fn create_entry_fn<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ cx: &'a Bx::CodegenCx,
+ sp: Span,
+ rust_main: Bx::Value,
+ rust_main_def_id: LocalDefId,
+ use_start_lang_item: bool,
+ ) -> Bx::Function {
+ // The entry function is either `int main(void)` or `int main(int argc, char **argv)`,
+ // depending on whether the target needs `argc` and `argv` to be passed in.
+ let llfty = if cx.sess().target.target.options.main_needs_argc_argv {
+ cx.type_func(&[cx.type_int(), cx.type_ptr_to(cx.type_i8p())], cx.type_int())
+ } else {
+ cx.type_func(&[], cx.type_int())
+ };
+
+ let main_ret_ty = cx.tcx().fn_sig(rust_main_def_id).output();
+ // Given that `main()` has no arguments,
+ // then its return type cannot have
+ // late-bound regions, since late-bound
+ // regions must appear in the argument
+ // listing.
+ let main_ret_ty = cx.tcx().erase_regions(&main_ret_ty.no_bound_vars().unwrap());
+
+ if cx.get_declared_value("main").is_some() {
+ // FIXME: We should be smart and show a better diagnostic here.
+ cx.sess()
+ .struct_span_err(sp, "entry symbol `main` declared multiple times")
+ .help("did you use `#[no_mangle]` on `fn main`? Use `#[start]` instead")
+ .emit();
+ cx.sess().abort_if_errors();
+ bug!();
+ }
+ let llfn = cx.declare_cfn("main", llfty);
+
+ // `main` should respect same config for frame pointer elimination as rest of code
+ cx.set_frame_pointer_elimination(llfn);
+ cx.apply_target_cpu_attr(llfn);
+
+ let mut bx = Bx::new_block(&cx, llfn, "top");
+
+ bx.insert_reference_to_gdb_debug_scripts_section_global();
+
+ let (arg_argc, arg_argv) = get_argc_argv(cx, &mut bx);
+
+ let (start_fn, args) = if use_start_lang_item {
+ let start_def_id = cx.tcx().require_lang_item(LangItem::Start, None);
+ let start_fn = cx.get_fn_addr(
+ ty::Instance::resolve(
+ cx.tcx(),
+ ty::ParamEnv::reveal_all(),
+ start_def_id,
+ cx.tcx().intern_substs(&[main_ret_ty.into()]),
+ )
+ .unwrap()
+ .unwrap(),
+ );
+ (
+ start_fn,
+ vec![bx.pointercast(rust_main, cx.type_ptr_to(cx.type_i8p())), arg_argc, arg_argv],
+ )
+ } else {
+ debug!("using user-defined start fn");
+ (rust_main, vec![arg_argc, arg_argv])
+ };
+
+ let result = bx.call(start_fn, &args, None);
+ let cast = bx.intcast(result, cx.type_int(), true);
+ bx.ret(cast);
+
+ llfn
+ }
+}
+
+/// Obtain the `argc` and `argv` values to pass to the rust start function.
+fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ cx: &'a Bx::CodegenCx,
+ bx: &mut Bx,
+) -> (Bx::Value, Bx::Value) {
+ if cx.sess().target.target.options.main_needs_argc_argv {
+ // Params from native `main()` used as args for rust start function
+ let param_argc = bx.get_param(0);
+ let param_argv = bx.get_param(1);
+ let arg_argc = bx.intcast(param_argc, cx.type_isize(), true);
+ let arg_argv = param_argv;
+ (arg_argc, arg_argv)
+ } else {
+ // The Rust start function doesn't need `argc` and `argv`, so just pass zeros.
+ let arg_argc = bx.const_int(cx.type_int(), 0);
+ let arg_argv = bx.const_null(cx.type_ptr_to(cx.type_i8p()));
+ (arg_argc, arg_argv)
+ }
+}
+
+pub const CODEGEN_WORKER_ID: usize = usize::MAX;
+
+pub fn codegen_crate<B: ExtraBackendMethods>(
+ backend: B,
+ tcx: TyCtxt<'tcx>,
+ metadata: EncodedMetadata,
+ need_metadata_module: bool,
+) -> OngoingCodegen<B> {
+ // Skip crate items and just output metadata in -Z no-codegen mode.
+ if tcx.sess.opts.debugging_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
+ let ongoing_codegen = start_async_codegen(backend, tcx, metadata, 1);
+
+ ongoing_codegen.codegen_finished(tcx);
+
+ finalize_tcx(tcx);
+
+ ongoing_codegen.check_for_errors(tcx.sess);
+
+ return ongoing_codegen;
+ }
+
+ let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
+
+ // Run the monomorphization collector and partition the collected items into
+ // codegen units.
+ let codegen_units = tcx.collect_and_partition_mono_items(LOCAL_CRATE).1;
+
+ // Force all codegen_unit queries so they are already either red or green
+ // when compile_codegen_unit accesses them. We are not able to re-execute
+ // the codegen_unit query from just the DepNode, so an unknown color would
+ // lead to having to re-execute compile_codegen_unit, possibly
+ // unnecessarily.
+ if tcx.dep_graph.is_fully_enabled() {
+ for cgu in codegen_units {
+ tcx.ensure().codegen_unit(cgu.name());
+ }
+ }
+
+ let ongoing_codegen = start_async_codegen(backend.clone(), tcx, metadata, codegen_units.len());
+ let ongoing_codegen = AbortCodegenOnDrop::<B>(Some(ongoing_codegen));
+
+ // Codegen an allocator shim, if necessary.
+ //
+ // If the crate doesn't have an `allocator_kind` set then there's definitely
+ // no shim to generate. Otherwise we also check our dependency graph for all
+ // our output crate types. If anything there looks like its a `Dynamic`
+ // linkage, then it's already got an allocator shim and we'll be using that
+ // one instead. If nothing exists then it's our job to generate the
+ // allocator!
+ let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| {
+ use rustc_middle::middle::dependency_format::Linkage;
+ list.iter().any(|&linkage| linkage == Linkage::Dynamic)
+ });
+ let allocator_module = if any_dynamic_crate {
+ None
+ } else if let Some(kind) = tcx.allocator_kind() {
+ let llmod_id =
+ cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("allocator")).to_string();
+ let mut modules = backend.new_metadata(tcx, &llmod_id);
+ tcx.sess
+ .time("write_allocator_module", || backend.codegen_allocator(tcx, &mut modules, kind));
+
+ Some(ModuleCodegen { name: llmod_id, module_llvm: modules, kind: ModuleKind::Allocator })
+ } else {
+ None
+ };
+
+ if let Some(allocator_module) = allocator_module {
+ ongoing_codegen.submit_pre_codegened_module_to_llvm(tcx, allocator_module);
+ }
+
+ if need_metadata_module {
+ // Codegen the encoded metadata.
+ let metadata_cgu_name =
+ cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("metadata")).to_string();
+ let mut metadata_llvm_module = backend.new_metadata(tcx, &metadata_cgu_name);
+ tcx.sess.time("write_compressed_metadata", || {
+ backend.write_compressed_metadata(
+ tcx,
+ &ongoing_codegen.metadata,
+ &mut metadata_llvm_module,
+ );
+ });
+
+ let metadata_module = ModuleCodegen {
+ name: metadata_cgu_name,
+ module_llvm: metadata_llvm_module,
+ kind: ModuleKind::Metadata,
+ };
+ ongoing_codegen.submit_pre_codegened_module_to_llvm(tcx, metadata_module);
+ }
+
+ // We sort the codegen units by size. This way we can schedule work for LLVM
+ // a bit more efficiently.
+ let codegen_units = {
+ let mut codegen_units = codegen_units.iter().collect::<Vec<_>>();
+ codegen_units.sort_by_cached_key(|cgu| cmp::Reverse(cgu.size_estimate()));
+ codegen_units
+ };
+
+ let total_codegen_time = Lock::new(Duration::new(0, 0));
+
+ // The non-parallel compiler can only translate codegen units to LLVM IR
+ // on a single thread, leading to a staircase effect where the N LLVM
+ // threads have to wait on the single codegen threads to generate work
+ // for them. The parallel compiler does not have this restriction, so
+ // we can pre-load the LLVM queue in parallel before handing off
+ // coordination to the OnGoingCodegen scheduler.
+ //
+ // This likely is a temporary measure. Once we don't have to support the
+ // non-parallel compiler anymore, we can compile CGUs end-to-end in
+ // parallel and get rid of the complicated scheduling logic.
+ let pre_compile_cgus = |cgu_reuse: &[CguReuse]| {
+ if cfg!(parallel_compiler) {
+ tcx.sess.time("compile_first_CGU_batch", || {
+ // Try to find one CGU to compile per thread.
+ let cgus: Vec<_> = cgu_reuse
+ .iter()
+ .enumerate()
+ .filter(|&(_, reuse)| reuse == &CguReuse::No)
+ .take(tcx.sess.threads())
+ .collect();
+
+ // Compile the found CGUs in parallel.
+ par_iter(cgus)
+ .map(|(i, _)| {
+ let start_time = Instant::now();
+ let module = backend.compile_codegen_unit(tcx, codegen_units[i].name());
+ let mut time = total_codegen_time.lock();
+ *time += start_time.elapsed();
+ (i, module)
+ })
+ .collect()
+ })
+ } else {
+ FxHashMap::default()
+ }
+ };
+
+ let mut cgu_reuse = Vec::new();
+ let mut pre_compiled_cgus: Option<FxHashMap<usize, _>> = None;
+
+ for (i, cgu) in codegen_units.iter().enumerate() {
+ ongoing_codegen.wait_for_signal_to_codegen_item();
+ ongoing_codegen.check_for_errors(tcx.sess);
+
+ // Do some setup work in the first iteration
+ if pre_compiled_cgus.is_none() {
+ // Calculate the CGU reuse
+ cgu_reuse = tcx.sess.time("find_cgu_reuse", || {
+ codegen_units.iter().map(|cgu| determine_cgu_reuse(tcx, &cgu)).collect()
+ });
+ // Pre compile some CGUs
+ pre_compiled_cgus = Some(pre_compile_cgus(&cgu_reuse));
+ }
+
+ let cgu_reuse = cgu_reuse[i];
+ tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
+
+ match cgu_reuse {
+ CguReuse::No => {
+ let (module, cost) =
+ if let Some(cgu) = pre_compiled_cgus.as_mut().unwrap().remove(&i) {
+ cgu
+ } else {
+ let start_time = Instant::now();
+ let module = backend.compile_codegen_unit(tcx, cgu.name());
+ let mut time = total_codegen_time.lock();
+ *time += start_time.elapsed();
+ module
+ };
+ submit_codegened_module_to_llvm(
+ &backend,
+ &ongoing_codegen.coordinator_send,
+ module,
+ cost,
+ );
+ false
+ }
+ CguReuse::PreLto => {
+ submit_pre_lto_module_to_llvm(
+ &backend,
+ tcx,
+ &ongoing_codegen.coordinator_send,
+ CachedModuleCodegen {
+ name: cgu.name().to_string(),
+ source: cgu.work_product(tcx),
+ },
+ );
+ true
+ }
+ CguReuse::PostLto => {
+ submit_post_lto_module_to_llvm(
+ &backend,
+ &ongoing_codegen.coordinator_send,
+ CachedModuleCodegen {
+ name: cgu.name().to_string(),
+ source: cgu.work_product(tcx),
+ },
+ );
+ true
+ }
+ };
+ }
+
+ ongoing_codegen.codegen_finished(tcx);
+
+ // Since the main thread is sometimes blocked during codegen, we keep track
+ // -Ztime-passes output manually.
+ print_time_passes_entry(
+ tcx.sess.time_passes(),
+ "codegen_to_LLVM_IR",
+ total_codegen_time.into_inner(),
+ );
+
+ ::rustc_incremental::assert_module_sources::assert_module_sources(tcx);
+
+ symbol_names_test::report_symbol_names(tcx);
+
+ ongoing_codegen.check_for_errors(tcx.sess);
+
+ finalize_tcx(tcx);
+
+ ongoing_codegen.into_inner()
+}
+
+/// A curious wrapper structure whose only purpose is to call `codegen_aborted`
+/// when it's dropped abnormally.
+///
+/// In the process of working on rust-lang/rust#55238 a mysterious segfault was
+/// stumbled upon. The segfault was never reproduced locally, but it was
+/// suspected to be related to the fact that codegen worker threads were
+/// sticking around by the time the main thread was exiting, causing issues.
+///
+/// This structure is an attempt to fix that issue where the `codegen_aborted`
+/// message will block until all workers have finished. This should ensure that
+/// even if the main codegen thread panics we'll wait for pending work to
+/// complete before returning from the main thread, hopefully avoiding
+/// segfaults.
+///
+/// If you see this comment in the code, then it means that this workaround
+/// worked! We may yet one day track down the mysterious cause of that
+/// segfault...
+struct AbortCodegenOnDrop<B: ExtraBackendMethods>(Option<OngoingCodegen<B>>);
+
+impl<B: ExtraBackendMethods> AbortCodegenOnDrop<B> {
+ fn into_inner(mut self) -> OngoingCodegen<B> {
+ self.0.take().unwrap()
+ }
+}
+
+impl<B: ExtraBackendMethods> Deref for AbortCodegenOnDrop<B> {
+ type Target = OngoingCodegen<B>;
+
+ fn deref(&self) -> &OngoingCodegen<B> {
+ self.0.as_ref().unwrap()
+ }
+}
+
+impl<B: ExtraBackendMethods> DerefMut for AbortCodegenOnDrop<B> {
+ fn deref_mut(&mut self) -> &mut OngoingCodegen<B> {
+ self.0.as_mut().unwrap()
+ }
+}
+
+impl<B: ExtraBackendMethods> Drop for AbortCodegenOnDrop<B> {
+ fn drop(&mut self) {
+ if let Some(codegen) = self.0.take() {
+ codegen.codegen_aborted();
+ }
+ }
+}
+
+fn finalize_tcx(tcx: TyCtxt<'_>) {
+ tcx.sess.time("assert_dep_graph", || ::rustc_incremental::assert_dep_graph(tcx));
+ tcx.sess.time("serialize_dep_graph", || ::rustc_incremental::save_dep_graph(tcx));
+
+ // We assume that no queries are run past here. If there are new queries
+ // after this point, they'll show up as "<unknown>" in self-profiling data.
+ {
+ let _prof_timer = tcx.prof.generic_activity("self_profile_alloc_query_strings");
+ tcx.alloc_self_profile_query_strings();
+ }
+}
+
+impl CrateInfo {
+ pub fn new(tcx: TyCtxt<'_>) -> CrateInfo {
+ let mut info = CrateInfo {
+ panic_runtime: None,
+ compiler_builtins: None,
+ profiler_runtime: None,
+ is_no_builtins: Default::default(),
+ native_libraries: Default::default(),
+ used_libraries: tcx.native_libraries(LOCAL_CRATE),
+ link_args: tcx.link_args(LOCAL_CRATE),
+ crate_name: Default::default(),
+ used_crates_dynamic: cstore::used_crates(tcx, LinkagePreference::RequireDynamic),
+ used_crates_static: cstore::used_crates(tcx, LinkagePreference::RequireStatic),
+ used_crate_source: Default::default(),
+ lang_item_to_crate: Default::default(),
+ missing_lang_items: Default::default(),
+ dependency_formats: tcx.dependency_formats(LOCAL_CRATE),
+ };
+ let lang_items = tcx.lang_items();
+
+ let crates = tcx.crates();
+
+ let n_crates = crates.len();
+ info.native_libraries.reserve(n_crates);
+ info.crate_name.reserve(n_crates);
+ info.used_crate_source.reserve(n_crates);
+ info.missing_lang_items.reserve(n_crates);
+
+ for &cnum in crates.iter() {
+ info.native_libraries.insert(cnum, tcx.native_libraries(cnum));
+ info.crate_name.insert(cnum, tcx.crate_name(cnum).to_string());
+ info.used_crate_source.insert(cnum, tcx.used_crate_source(cnum));
+ if tcx.is_panic_runtime(cnum) {
+ info.panic_runtime = Some(cnum);
+ }
+ if tcx.is_compiler_builtins(cnum) {
+ info.compiler_builtins = Some(cnum);
+ }
+ if tcx.is_profiler_runtime(cnum) {
+ info.profiler_runtime = Some(cnum);
+ }
+ if tcx.is_no_builtins(cnum) {
+ info.is_no_builtins.insert(cnum);
+ }
+ let missing = tcx.missing_lang_items(cnum);
+ for &item in missing.iter() {
+ if let Ok(id) = lang_items.require(item) {
+ info.lang_item_to_crate.insert(item, id.krate);
+ }
+ }
+
+ // No need to look for lang items that don't actually need to exist.
+ let missing =
+ missing.iter().cloned().filter(|&l| lang_items::required(tcx, l)).collect();
+ info.missing_lang_items.insert(cnum, missing);
+ }
+
+ info
+ }
+}
+
+pub fn provide_both(providers: &mut Providers) {
+ providers.backend_optimization_level = |tcx, cratenum| {
+ let for_speed = match tcx.sess.opts.optimize {
+ // If globally no optimisation is done, #[optimize] has no effect.
+ //
+ // This is done because if we ended up "upgrading" to `-O2` here, we’d populate the
+ // pass manager and it is likely that some module-wide passes (such as inliner or
+ // cross-function constant propagation) would ignore the `optnone` annotation we put
+ // on the functions, thus necessarily involving these functions into optimisations.
+ config::OptLevel::No => return config::OptLevel::No,
+ // If globally optimise-speed is already specified, just use that level.
+ config::OptLevel::Less => return config::OptLevel::Less,
+ config::OptLevel::Default => return config::OptLevel::Default,
+ config::OptLevel::Aggressive => return config::OptLevel::Aggressive,
+ // If globally optimize-for-size has been requested, use -O2 instead (if optimize(size)
+ // are present).
+ config::OptLevel::Size => config::OptLevel::Default,
+ config::OptLevel::SizeMin => config::OptLevel::Default,
+ };
+
+ let (defids, _) = tcx.collect_and_partition_mono_items(cratenum);
+ for id in &*defids {
+ let CodegenFnAttrs { optimize, .. } = tcx.codegen_fn_attrs(*id);
+ match optimize {
+ attr::OptimizeAttr::None => continue,
+ attr::OptimizeAttr::Size => continue,
+ attr::OptimizeAttr::Speed => {
+ return for_speed;
+ }
+ }
+ }
+ tcx.sess.opts.optimize
+ };
+
+ providers.dllimport_foreign_items = |tcx, krate| {
+ let module_map = tcx.foreign_modules(krate);
+ let module_map =
+ module_map.iter().map(|lib| (lib.def_id, lib)).collect::<FxHashMap<_, _>>();
+
+ let dllimports = tcx
+ .native_libraries(krate)
+ .iter()
+ .filter(|lib| {
+ if !matches!(lib.kind, NativeLibKind::Dylib | NativeLibKind::Unspecified) {
+ return false;
+ }
+ let cfg = match lib.cfg {
+ Some(ref cfg) => cfg,
+ None => return true,
+ };
+ attr::cfg_matches(cfg, &tcx.sess.parse_sess, None)
+ })
+ .filter_map(|lib| lib.foreign_module)
+ .map(|id| &module_map[&id])
+ .flat_map(|module| module.foreign_items.iter().cloned())
+ .collect();
+ dllimports
+ };
+
+ providers.is_dllimport_foreign_item =
+ |tcx, def_id| tcx.dllimport_foreign_items(def_id.krate).contains(&def_id);
+}
+
+fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguReuse {
+ if !tcx.dep_graph.is_fully_enabled() {
+ return CguReuse::No;
+ }
+
+ let work_product_id = &cgu.work_product_id();
+ if tcx.dep_graph.previous_work_product(work_product_id).is_none() {
+ // We don't have anything cached for this CGU. This can happen
+ // if the CGU did not exist in the previous session.
+ return CguReuse::No;
+ }
+
+ // Try to mark the CGU as green. If it we can do so, it means that nothing
+ // affecting the LLVM module has changed and we can re-use a cached version.
+ // If we compile with any kind of LTO, this means we can re-use the bitcode
+ // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only
+ // know that later). If we are not doing LTO, there is only one optimized
+ // version of each module, so we re-use that.
+ let dep_node = cgu.codegen_dep_node(tcx);
+ assert!(
+ !tcx.dep_graph.dep_node_exists(&dep_node),
+ "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.",
+ cgu.name()
+ );
+
+ if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() {
+ // We can re-use either the pre- or the post-thinlto state. If no LTO is
+ // being performed then we can use post-LTO artifacts, otherwise we must
+ // reuse pre-LTO artifacts
+ match compute_per_cgu_lto_type(
+ &tcx.sess.lto(),
+ &tcx.sess.opts,
+ &tcx.sess.crate_types(),
+ ModuleKind::Regular,
+ ) {
+ ComputedLtoType::No => CguReuse::PostLto,
+ _ => CguReuse::PreLto,
+ }
+ } else {
+ CguReuse::No
+ }
+}
--- /dev/null
- llval = base::to_immediate(bx, llval, arg.layout);
+use super::operand::OperandRef;
+use super::operand::OperandValue::{Immediate, Pair, Ref};
+use super::place::PlaceRef;
+use super::{FunctionCx, LocalRef};
+
+use crate::base;
+use crate::common::{self, IntPredicate};
+use crate::meth;
+use crate::traits::*;
+use crate::MemFlags;
+
+use rustc_ast as ast;
+use rustc_hir::lang_items::LangItem;
+use rustc_index::vec::Idx;
+use rustc_middle::mir;
+use rustc_middle::mir::interpret::{AllocId, ConstValue, Pointer, Scalar};
+use rustc_middle::mir::AssertKind;
+use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt};
+use rustc_middle::ty::{self, Instance, Ty, TypeFoldable};
+use rustc_span::source_map::Span;
+use rustc_span::{sym, Symbol};
+use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode};
+use rustc_target::abi::{self, LayoutOf};
+use rustc_target::spec::abi::Abi;
+
+use std::borrow::Cow;
+
+/// Used by `FunctionCx::codegen_terminator` for emitting common patterns
+/// e.g., creating a basic block, calling a function, etc.
+struct TerminatorCodegenHelper<'tcx> {
+ bb: mir::BasicBlock,
+ terminator: &'tcx mir::Terminator<'tcx>,
+ funclet_bb: Option<mir::BasicBlock>,
+}
+
+impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
+ /// Returns the associated funclet from `FunctionCx::funclets` for the
+ /// `funclet_bb` member if it is not `None`.
+ fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &'b mut FunctionCx<'a, 'tcx, Bx>,
+ ) -> Option<&'b Bx::Funclet> {
+ match self.funclet_bb {
+ Some(funcl) => fx.funclets[funcl].as_ref(),
+ None => None,
+ }
+ }
+
+ fn lltarget<Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &mut FunctionCx<'a, 'tcx, Bx>,
+ target: mir::BasicBlock,
+ ) -> (Bx::BasicBlock, bool) {
+ let span = self.terminator.source_info.span;
+ let lltarget = fx.blocks[target];
+ let target_funclet = fx.cleanup_kinds[target].funclet_bb(target);
+ match (self.funclet_bb, target_funclet) {
+ (None, None) => (lltarget, false),
+ (Some(f), Some(t_f)) if f == t_f || !base::wants_msvc_seh(fx.cx.tcx().sess) => {
+ (lltarget, false)
+ }
+ // jump *into* cleanup - need a landing pad if GNU
+ (None, Some(_)) => (fx.landing_pad_to(target), false),
+ (Some(_), None) => span_bug!(span, "{:?} - jump out of cleanup?", self.terminator),
+ (Some(_), Some(_)) => (fx.landing_pad_to(target), true),
+ }
+ }
+
+ /// Create a basic block.
+ fn llblock<Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &mut FunctionCx<'a, 'tcx, Bx>,
+ target: mir::BasicBlock,
+ ) -> Bx::BasicBlock {
+ let (lltarget, is_cleanupret) = self.lltarget(fx, target);
+ if is_cleanupret {
+ // MSVC cross-funclet jump - need a trampoline
+
+ debug!("llblock: creating cleanup trampoline for {:?}", target);
+ let name = &format!("{:?}_cleanup_trampoline_{:?}", self.bb, target);
+ let mut trampoline = fx.new_block(name);
+ trampoline.cleanup_ret(self.funclet(fx).unwrap(), Some(lltarget));
+ trampoline.llbb()
+ } else {
+ lltarget
+ }
+ }
+
+ fn funclet_br<Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &mut FunctionCx<'a, 'tcx, Bx>,
+ bx: &mut Bx,
+ target: mir::BasicBlock,
+ ) {
+ let (lltarget, is_cleanupret) = self.lltarget(fx, target);
+ if is_cleanupret {
+ // micro-optimization: generate a `ret` rather than a jump
+ // to a trampoline.
+ bx.cleanup_ret(self.funclet(fx).unwrap(), Some(lltarget));
+ } else {
+ bx.br(lltarget);
+ }
+ }
+
+ /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional
+ /// return destination `destination` and the cleanup function `cleanup`.
+ fn do_call<Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &mut FunctionCx<'a, 'tcx, Bx>,
+ bx: &mut Bx,
+ fn_abi: FnAbi<'tcx, Ty<'tcx>>,
+ fn_ptr: Bx::Value,
+ llargs: &[Bx::Value],
+ destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>,
+ cleanup: Option<mir::BasicBlock>,
+ ) {
+ // If there is a cleanup block and the function we're calling can unwind, then
+ // do an invoke, otherwise do a call.
+ if let Some(cleanup) = cleanup.filter(|_| fn_abi.can_unwind) {
+ let ret_bx = if let Some((_, target)) = destination {
+ fx.blocks[target]
+ } else {
+ fx.unreachable_block()
+ };
+ let invokeret =
+ bx.invoke(fn_ptr, &llargs, ret_bx, self.llblock(fx, cleanup), self.funclet(fx));
+ bx.apply_attrs_callsite(&fn_abi, invokeret);
+
+ if let Some((ret_dest, target)) = destination {
+ let mut ret_bx = fx.build_block(target);
+ fx.set_debug_loc(&mut ret_bx, self.terminator.source_info);
+ fx.store_return(&mut ret_bx, ret_dest, &fn_abi.ret, invokeret);
+ }
+ } else {
+ let llret = bx.call(fn_ptr, &llargs, self.funclet(fx));
+ bx.apply_attrs_callsite(&fn_abi, llret);
+ if fx.mir[self.bb].is_cleanup {
+ // Cleanup is always the cold path. Don't inline
+ // drop glue. Also, when there is a deeply-nested
+ // struct, there are "symmetry" issues that cause
+ // exponential inlining - see issue #41696.
+ bx.do_not_inline(llret);
+ }
+
+ if let Some((ret_dest, target)) = destination {
+ fx.store_return(bx, ret_dest, &fn_abi.ret, llret);
+ self.funclet_br(fx, bx, target);
+ } else {
+ bx.unreachable();
+ }
+ }
+ }
+
+ // Generate sideeffect intrinsic if jumping to any of the targets can form
+ // a loop.
+ fn maybe_sideeffect<Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ mir: &'tcx mir::Body<'tcx>,
+ bx: &mut Bx,
+ targets: &[mir::BasicBlock],
+ ) {
+ if bx.tcx().sess.opts.debugging_opts.insert_sideeffect {
+ if targets.iter().any(|&target| {
+ target <= self.bb
+ && target.start_location().is_predecessor_of(self.bb.start_location(), mir)
+ }) {
+ bx.sideeffect();
+ }
+ }
+ }
+}
+
+/// Codegen implementations for some terminator variants.
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ /// Generates code for a `Resume` terminator.
+ fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, mut bx: Bx) {
+ if let Some(funclet) = helper.funclet(self) {
+ bx.cleanup_ret(funclet, None);
+ } else {
+ let slot = self.get_personality_slot(&mut bx);
+ let lp0 = slot.project_field(&mut bx, 0);
+ let lp0 = bx.load_operand(lp0).immediate();
+ let lp1 = slot.project_field(&mut bx, 1);
+ let lp1 = bx.load_operand(lp1).immediate();
+ slot.storage_dead(&mut bx);
+
+ let mut lp = bx.const_undef(self.landing_pad_type());
+ lp = bx.insert_value(lp, lp0, 0);
+ lp = bx.insert_value(lp, lp1, 1);
+ bx.resume(lp);
+ }
+ }
+
+ fn codegen_switchint_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ discr: &mir::Operand<'tcx>,
+ switch_ty: Ty<'tcx>,
+ values: &Cow<'tcx, [u128]>,
+ targets: &Vec<mir::BasicBlock>,
+ ) {
+ let discr = self.codegen_operand(&mut bx, &discr);
+ // `switch_ty` is redundant, sanity-check that.
+ assert_eq!(discr.layout.ty, switch_ty);
+ if targets.len() == 2 {
+ // If there are two targets, emit br instead of switch
+ let lltrue = helper.llblock(self, targets[0]);
+ let llfalse = helper.llblock(self, targets[1]);
+ if switch_ty == bx.tcx().types.bool {
+ helper.maybe_sideeffect(self.mir, &mut bx, targets.as_slice());
+ // Don't generate trivial icmps when switching on bool
+ if let [0] = values[..] {
+ bx.cond_br(discr.immediate(), llfalse, lltrue);
+ } else {
+ assert_eq!(&values[..], &[1]);
+ bx.cond_br(discr.immediate(), lltrue, llfalse);
+ }
+ } else {
+ let switch_llty = bx.immediate_backend_type(bx.layout_of(switch_ty));
+ let llval = bx.const_uint_big(switch_llty, values[0]);
+ let cmp = bx.icmp(IntPredicate::IntEQ, discr.immediate(), llval);
+ helper.maybe_sideeffect(self.mir, &mut bx, targets.as_slice());
+ bx.cond_br(cmp, lltrue, llfalse);
+ }
+ } else {
+ helper.maybe_sideeffect(self.mir, &mut bx, targets.as_slice());
+ let (otherwise, targets) = targets.split_last().unwrap();
+ bx.switch(
+ discr.immediate(),
+ helper.llblock(self, *otherwise),
+ values
+ .iter()
+ .zip(targets)
+ .map(|(&value, target)| (value, helper.llblock(self, *target))),
+ );
+ }
+ }
+
+ fn codegen_return_terminator(&mut self, mut bx: Bx) {
+ // Call `va_end` if this is the definition of a C-variadic function.
+ if self.fn_abi.c_variadic {
+ // The `VaList` "spoofed" argument is just after all the real arguments.
+ let va_list_arg_idx = self.fn_abi.args.len();
+ match self.locals[mir::Local::new(1 + va_list_arg_idx)] {
+ LocalRef::Place(va_list) => {
+ bx.va_end(va_list.llval);
+ }
+ _ => bug!("C-variadic function must have a `VaList` place"),
+ }
+ }
+ if self.fn_abi.ret.layout.abi.is_uninhabited() {
+ // Functions with uninhabited return values are marked `noreturn`,
+ // so we should make sure that we never actually do.
+ // We play it safe by using a well-defined `abort`, but we could go for immediate UB
+ // if that turns out to be helpful.
+ bx.abort();
+ // `abort` does not terminate the block, so we still need to generate
+ // an `unreachable` terminator after it.
+ bx.unreachable();
+ return;
+ }
+ let llval = match self.fn_abi.ret.mode {
+ PassMode::Ignore | PassMode::Indirect(..) => {
+ bx.ret_void();
+ return;
+ }
+
+ PassMode::Direct(_) | PassMode::Pair(..) => {
+ let op = self.codegen_consume(&mut bx, mir::Place::return_place().as_ref());
+ if let Ref(llval, _, align) = op.val {
+ bx.load(llval, align)
+ } else {
+ op.immediate_or_packed_pair(&mut bx)
+ }
+ }
+
+ PassMode::Cast(cast_ty) => {
+ let op = match self.locals[mir::RETURN_PLACE] {
+ LocalRef::Operand(Some(op)) => op,
+ LocalRef::Operand(None) => bug!("use of return before def"),
+ LocalRef::Place(cg_place) => OperandRef {
+ val: Ref(cg_place.llval, None, cg_place.align),
+ layout: cg_place.layout,
+ },
+ LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
+ };
+ let llslot = match op.val {
+ Immediate(_) | Pair(..) => {
+ let scratch = PlaceRef::alloca(&mut bx, self.fn_abi.ret.layout);
+ op.val.store(&mut bx, scratch);
+ scratch.llval
+ }
+ Ref(llval, _, align) => {
+ assert_eq!(align, op.layout.align.abi, "return place is unaligned!");
+ llval
+ }
+ };
+ let addr = bx.pointercast(llslot, bx.type_ptr_to(bx.cast_backend_type(&cast_ty)));
+ bx.load(addr, self.fn_abi.ret.layout.align.abi)
+ }
+ };
+ bx.ret(llval);
+ }
+
+ fn codegen_drop_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ location: mir::Place<'tcx>,
+ target: mir::BasicBlock,
+ unwind: Option<mir::BasicBlock>,
+ ) {
+ let ty = location.ty(self.mir, bx.tcx()).ty;
+ let ty = self.monomorphize(&ty);
+ let drop_fn = Instance::resolve_drop_in_place(bx.tcx(), ty);
+
+ if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def {
+ // we don't actually need to drop anything.
+ helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+ helper.funclet_br(self, &mut bx, target);
+ return;
+ }
+
+ let place = self.codegen_place(&mut bx, location.as_ref());
+ let (args1, args2);
+ let mut args = if let Some(llextra) = place.llextra {
+ args2 = [place.llval, llextra];
+ &args2[..]
+ } else {
+ args1 = [place.llval];
+ &args1[..]
+ };
+ let (drop_fn, fn_abi) = match ty.kind {
+ // FIXME(eddyb) perhaps move some of this logic into
+ // `Instance::resolve_drop_in_place`?
+ ty::Dynamic(..) => {
+ let virtual_drop = Instance {
+ def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
+ substs: drop_fn.substs,
+ };
+ let fn_abi = FnAbi::of_instance(&bx, virtual_drop, &[]);
+ let vtable = args[1];
+ args = &args[..1];
+ (meth::DESTRUCTOR.get_fn(&mut bx, vtable, &fn_abi), fn_abi)
+ }
+ _ => (bx.get_fn_addr(drop_fn), FnAbi::of_instance(&bx, drop_fn, &[])),
+ };
+ helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+ helper.do_call(
+ self,
+ &mut bx,
+ fn_abi,
+ drop_fn,
+ args,
+ Some((ReturnDest::Nothing, target)),
+ unwind,
+ );
+ }
+
+ fn codegen_assert_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ terminator: &mir::Terminator<'tcx>,
+ cond: &mir::Operand<'tcx>,
+ expected: bool,
+ msg: &mir::AssertMessage<'tcx>,
+ target: mir::BasicBlock,
+ cleanup: Option<mir::BasicBlock>,
+ ) {
+ let span = terminator.source_info.span;
+ let cond = self.codegen_operand(&mut bx, cond).immediate();
+ let mut const_cond = bx.const_to_opt_u128(cond, false).map(|c| c == 1);
+
+ // This case can currently arise only from functions marked
+ // with #[rustc_inherit_overflow_checks] and inlined from
+ // another crate (mostly core::num generic/#[inline] fns),
+ // while the current crate doesn't use overflow checks.
+ // NOTE: Unlike binops, negation doesn't have its own
+ // checked operation, just a comparison with the minimum
+ // value, so we have to check for the assert message.
+ if !bx.check_overflow() {
+ if let AssertKind::OverflowNeg(_) = *msg {
+ const_cond = Some(expected);
+ }
+ }
+
+ // Don't codegen the panic block if success if known.
+ if const_cond == Some(expected) {
+ helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+ helper.funclet_br(self, &mut bx, target);
+ return;
+ }
+
+ // Pass the condition through llvm.expect for branch hinting.
+ let cond = bx.expect(cond, expected);
+
+ // Create the failure block and the conditional branch to it.
+ let lltarget = helper.llblock(self, target);
+ let panic_block = self.new_block("panic");
+ helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+ if expected {
+ bx.cond_br(cond, lltarget, panic_block.llbb());
+ } else {
+ bx.cond_br(cond, panic_block.llbb(), lltarget);
+ }
+
+ // After this point, bx is the block for the call to panic.
+ bx = panic_block;
+ self.set_debug_loc(&mut bx, terminator.source_info);
+
+ // Get the location information.
+ let location = self.get_caller_location(&mut bx, span).immediate();
+
+ // Put together the arguments to the panic entry point.
+ let (lang_item, args) = match msg {
+ AssertKind::BoundsCheck { ref len, ref index } => {
+ let len = self.codegen_operand(&mut bx, len).immediate();
+ let index = self.codegen_operand(&mut bx, index).immediate();
+ // It's `fn panic_bounds_check(index: usize, len: usize)`,
+ // and `#[track_caller]` adds an implicit third argument.
+ (LangItem::PanicBoundsCheck, vec![index, len, location])
+ }
+ _ => {
+ let msg_str = Symbol::intern(msg.description());
+ let msg = bx.const_str(msg_str);
+ // It's `pub fn panic(expr: &str)`, with the wide reference being passed
+ // as two arguments, and `#[track_caller]` adds an implicit third argument.
+ (LangItem::Panic, vec![msg.0, msg.1, location])
+ }
+ };
+
+ // Obtain the panic entry point.
+ let def_id = common::langcall(bx.tcx(), Some(span), "", lang_item);
+ let instance = ty::Instance::mono(bx.tcx(), def_id);
+ let fn_abi = FnAbi::of_instance(&bx, instance, &[]);
+ let llfn = bx.get_fn_addr(instance);
+
+ // Codegen the actual panic invoke/call.
+ helper.do_call(self, &mut bx, fn_abi, llfn, &args, None, cleanup);
+ }
+
+ /// Returns `true` if this is indeed a panic intrinsic and codegen is done.
+ fn codegen_panic_intrinsic(
+ &mut self,
+ helper: &TerminatorCodegenHelper<'tcx>,
+ bx: &mut Bx,
+ intrinsic: Option<Symbol>,
+ instance: Option<Instance<'tcx>>,
+ span: Span,
+ destination: &Option<(mir::Place<'tcx>, mir::BasicBlock)>,
+ cleanup: Option<mir::BasicBlock>,
+ ) -> bool {
+ // Emit a panic or a no-op for `assert_*` intrinsics.
+ // These are intrinsics that compile to panics so that we can get a message
+ // which mentions the offending type, even from a const context.
+ #[derive(Debug, PartialEq)]
+ enum AssertIntrinsic {
+ Inhabited,
+ ZeroValid,
+ UninitValid,
+ };
+ let panic_intrinsic = intrinsic.and_then(|i| match i {
+ sym::assert_inhabited => Some(AssertIntrinsic::Inhabited),
+ sym::assert_zero_valid => Some(AssertIntrinsic::ZeroValid),
+ sym::assert_uninit_valid => Some(AssertIntrinsic::UninitValid),
+ _ => None,
+ });
+ if let Some(intrinsic) = panic_intrinsic {
+ use AssertIntrinsic::*;
+ let ty = instance.unwrap().substs.type_at(0);
+ let layout = bx.layout_of(ty);
+ let do_panic = match intrinsic {
+ Inhabited => layout.abi.is_uninhabited(),
+ // We unwrap as the error type is `!`.
+ ZeroValid => !layout.might_permit_raw_init(bx, /*zero:*/ true).unwrap(),
+ // We unwrap as the error type is `!`.
+ UninitValid => !layout.might_permit_raw_init(bx, /*zero:*/ false).unwrap(),
+ };
+ if do_panic {
+ let msg_str = if layout.abi.is_uninhabited() {
+ // Use this error even for the other intrinsics as it is more precise.
+ format!("attempted to instantiate uninhabited type `{}`", ty)
+ } else if intrinsic == ZeroValid {
+ format!("attempted to zero-initialize type `{}`, which is invalid", ty)
+ } else {
+ format!("attempted to leave type `{}` uninitialized, which is invalid", ty)
+ };
+ let msg = bx.const_str(Symbol::intern(&msg_str));
+ let location = self.get_caller_location(bx, span).immediate();
+
+ // Obtain the panic entry point.
+ // FIXME: dedup this with `codegen_assert_terminator` above.
+ let def_id = common::langcall(bx.tcx(), Some(span), "", LangItem::Panic);
+ let instance = ty::Instance::mono(bx.tcx(), def_id);
+ let fn_abi = FnAbi::of_instance(bx, instance, &[]);
+ let llfn = bx.get_fn_addr(instance);
+
+ if let Some((_, target)) = destination.as_ref() {
+ helper.maybe_sideeffect(self.mir, bx, &[*target]);
+ }
+ // Codegen the actual panic invoke/call.
+ helper.do_call(
+ self,
+ bx,
+ fn_abi,
+ llfn,
+ &[msg.0, msg.1, location],
+ destination.as_ref().map(|(_, bb)| (ReturnDest::Nothing, *bb)),
+ cleanup,
+ );
+ } else {
+ // a NOP
+ let target = destination.as_ref().unwrap().1;
+ helper.maybe_sideeffect(self.mir, bx, &[target]);
+ helper.funclet_br(self, bx, target)
+ }
+ true
+ } else {
+ false
+ }
+ }
+
+ fn codegen_call_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ terminator: &mir::Terminator<'tcx>,
+ func: &mir::Operand<'tcx>,
+ args: &Vec<mir::Operand<'tcx>>,
+ destination: &Option<(mir::Place<'tcx>, mir::BasicBlock)>,
+ cleanup: Option<mir::BasicBlock>,
+ fn_span: Span,
+ ) {
+ let span = terminator.source_info.span;
+ // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar.
+ let callee = self.codegen_operand(&mut bx, func);
+
+ let (instance, mut llfn) = match callee.layout.ty.kind {
+ ty::FnDef(def_id, substs) => (
+ Some(
+ ty::Instance::resolve(bx.tcx(), ty::ParamEnv::reveal_all(), def_id, substs)
+ .unwrap()
+ .unwrap()
+ .polymorphize(bx.tcx()),
+ ),
+ None,
+ ),
+ ty::FnPtr(_) => (None, Some(callee.immediate())),
+ _ => bug!("{} is not callable", callee.layout.ty),
+ };
+ let def = instance.map(|i| i.def);
+
+ if let Some(ty::InstanceDef::DropGlue(_, None)) = def {
+ // Empty drop glue; a no-op.
+ let &(_, target) = destination.as_ref().unwrap();
+ helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+ helper.funclet_br(self, &mut bx, target);
+ return;
+ }
+
+ // FIXME(eddyb) avoid computing this if possible, when `instance` is
+ // available - right now `sig` is only needed for getting the `abi`
+ // and figuring out how many extra args were passed to a C-variadic `fn`.
+ let sig = callee.layout.ty.fn_sig(bx.tcx());
+ let abi = sig.abi();
+
+ // Handle intrinsics old codegen wants Expr's for, ourselves.
+ let intrinsic = match def {
+ Some(ty::InstanceDef::Intrinsic(def_id)) => Some(bx.tcx().item_name(def_id)),
+ _ => None,
+ };
+
+ let extra_args = &args[sig.inputs().skip_binder().len()..];
+ let extra_args = extra_args
+ .iter()
+ .map(|op_arg| {
+ let op_ty = op_arg.ty(self.mir, bx.tcx());
+ self.monomorphize(&op_ty)
+ })
+ .collect::<Vec<_>>();
+
+ let fn_abi = match instance {
+ Some(instance) => FnAbi::of_instance(&bx, instance, &extra_args),
+ None => FnAbi::of_fn_ptr(&bx, sig, &extra_args),
+ };
+
+ if intrinsic == Some(sym::transmute) {
+ if let Some(destination_ref) = destination.as_ref() {
+ let &(dest, target) = destination_ref;
+ self.codegen_transmute(&mut bx, &args[0], dest);
+ helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+ helper.funclet_br(self, &mut bx, target);
+ } else {
+ // If we are trying to transmute to an uninhabited type,
+ // it is likely there is no allotted destination. In fact,
+ // transmuting to an uninhabited type is UB, which means
+ // we can do what we like. Here, we declare that transmuting
+ // into an uninhabited type is impossible, so anything following
+ // it must be unreachable.
+ assert_eq!(fn_abi.ret.layout.abi, abi::Abi::Uninhabited);
+ bx.unreachable();
+ }
+ return;
+ }
+
+ if self.codegen_panic_intrinsic(
+ &helper,
+ &mut bx,
+ intrinsic,
+ instance,
+ span,
+ destination,
+ cleanup,
+ ) {
+ return;
+ }
+
+ // The arguments we'll be passing. Plus one to account for outptr, if used.
+ let arg_count = fn_abi.args.len() + fn_abi.ret.is_indirect() as usize;
+ let mut llargs = Vec::with_capacity(arg_count);
+
+ // Prepare the return value destination
+ let ret_dest = if let Some((dest, _)) = *destination {
+ let is_intrinsic = intrinsic.is_some();
+ self.make_return_dest(&mut bx, dest, &fn_abi.ret, &mut llargs, is_intrinsic)
+ } else {
+ ReturnDest::Nothing
+ };
+
+ if intrinsic == Some(sym::caller_location) {
+ if let Some((_, target)) = destination.as_ref() {
+ let location = self.get_caller_location(&mut bx, fn_span);
+
+ if let ReturnDest::IndirectOperand(tmp, _) = ret_dest {
+ location.val.store(&mut bx, tmp);
+ }
+ self.store_return(&mut bx, ret_dest, &fn_abi.ret, location.immediate());
+
+ helper.maybe_sideeffect(self.mir, &mut bx, &[*target]);
+ helper.funclet_br(self, &mut bx, *target);
+ }
+ return;
+ }
+
+ if intrinsic.is_some() && intrinsic != Some(sym::drop_in_place) {
+ let intrinsic = intrinsic.unwrap();
+ let dest = match ret_dest {
+ _ if fn_abi.ret.is_indirect() => llargs[0],
+ ReturnDest::Nothing => {
+ bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret)))
+ }
+ ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval,
+ ReturnDest::DirectOperand(_) => {
+ bug!("Cannot use direct operand with an intrinsic call")
+ }
+ };
+
+ let args: Vec<_> = args
+ .iter()
+ .enumerate()
+ .map(|(i, arg)| {
+ // The indices passed to simd_shuffle* in the
+ // third argument must be constant. This is
+ // checked by const-qualification, which also
+ // promotes any complex rvalues to constants.
+ if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
+ if let mir::Operand::Constant(constant) = arg {
+ let c = self.eval_mir_constant(constant);
+ let (llval, ty) = self.simd_shuffle_indices(
+ &bx,
+ constant.span,
+ constant.literal.ty,
+ c,
+ );
+ return OperandRef { val: Immediate(llval), layout: bx.layout_of(ty) };
+ } else {
+ span_bug!(span, "shuffle indices must be constant");
+ }
+ }
+
+ self.codegen_operand(&mut bx, arg)
+ })
+ .collect();
+
+ bx.codegen_intrinsic_call(
+ *instance.as_ref().unwrap(),
+ &fn_abi,
+ &args,
+ dest,
+ terminator.source_info.span,
+ );
+
+ if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
+ self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval);
+ }
+
+ if let Some((_, target)) = *destination {
+ helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+ helper.funclet_br(self, &mut bx, target);
+ } else {
+ bx.unreachable();
+ }
+
+ return;
+ }
+
+ // Split the rust-call tupled arguments off.
+ let (first_args, untuple) = if abi == Abi::RustCall && !args.is_empty() {
+ let (tup, args) = args.split_last().unwrap();
+ (args, Some(tup))
+ } else {
+ (&args[..], None)
+ };
+
+ 'make_args: for (i, arg) in first_args.iter().enumerate() {
+ let mut op = self.codegen_operand(&mut bx, arg);
+
+ if let (0, Some(ty::InstanceDef::Virtual(_, idx))) = (i, def) {
+ if let Pair(..) = op.val {
+ // In the case of Rc<Self>, we need to explicitly pass a
+ // *mut RcBox<Self> with a Scalar (not ScalarPair) ABI. This is a hack
+ // that is understood elsewhere in the compiler as a method on
+ // `dyn Trait`.
+ // To get a `*mut RcBox<Self>`, we just keep unwrapping newtypes until
+ // we get a value of a built-in pointer type
+ 'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
+ && !op.layout.ty.is_region_ptr()
+ {
+ for i in 0..op.layout.fields.count() {
+ let field = op.extract_field(&mut bx, i);
+ if !field.layout.is_zst() {
+ // we found the one non-zero-sized field that is allowed
+ // now find *its* non-zero-sized field, or stop if it's a
+ // pointer
+ op = field;
+ continue 'descend_newtypes;
+ }
+ }
+
+ span_bug!(span, "receiver has no non-zero-sized fields {:?}", op);
+ }
+
+ // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
+ // data pointer and vtable. Look up the method in the vtable, and pass
+ // the data pointer as the first argument
+ match op.val {
+ Pair(data_ptr, meta) => {
+ llfn = Some(
+ meth::VirtualIndex::from_index(idx).get_fn(&mut bx, meta, &fn_abi),
+ );
+ llargs.push(data_ptr);
+ continue 'make_args;
+ }
+ other => bug!("expected a Pair, got {:?}", other),
+ }
+ } else if let Ref(data_ptr, Some(meta), _) = op.val {
+ // by-value dynamic dispatch
+ llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(&mut bx, meta, &fn_abi));
+ llargs.push(data_ptr);
+ continue;
+ } else {
+ span_bug!(span, "can't codegen a virtual call on {:?}", op);
+ }
+ }
+
+ // The callee needs to own the argument memory if we pass it
+ // by-ref, so make a local copy of non-immediate constants.
+ match (arg, op.val) {
+ (&mir::Operand::Copy(_), Ref(_, None, _))
+ | (&mir::Operand::Constant(_), Ref(_, None, _)) => {
+ let tmp = PlaceRef::alloca(&mut bx, op.layout);
+ op.val.store(&mut bx, tmp);
+ op.val = Ref(tmp.llval, None, tmp.align);
+ }
+ _ => {}
+ }
+
+ self.codegen_argument(&mut bx, op, &mut llargs, &fn_abi.args[i]);
+ }
+ if let Some(tup) = untuple {
+ self.codegen_arguments_untupled(
+ &mut bx,
+ tup,
+ &mut llargs,
+ &fn_abi.args[first_args.len()..],
+ )
+ }
+
+ let needs_location =
+ instance.map_or(false, |i| i.def.requires_caller_location(self.cx.tcx()));
+ if needs_location {
+ assert_eq!(
+ fn_abi.args.len(),
+ args.len() + 1,
+ "#[track_caller] fn's must have 1 more argument in their ABI than in their MIR",
+ );
+ let location = self.get_caller_location(&mut bx, fn_span);
+ debug!(
+ "codegen_call_terminator({:?}): location={:?} (fn_span {:?})",
+ terminator, location, fn_span
+ );
+
+ let last_arg = fn_abi.args.last().unwrap();
+ self.codegen_argument(&mut bx, location, &mut llargs, last_arg);
+ }
+
+ let fn_ptr = match (llfn, instance) {
+ (Some(llfn), _) => llfn,
+ (None, Some(instance)) => bx.get_fn_addr(instance),
+ _ => span_bug!(span, "no llfn for call"),
+ };
+
+ if let Some((_, target)) = destination.as_ref() {
+ helper.maybe_sideeffect(self.mir, &mut bx, &[*target]);
+ }
+ helper.do_call(
+ self,
+ &mut bx,
+ fn_abi,
+ fn_ptr,
+ &llargs,
+ destination.as_ref().map(|&(_, target)| (ret_dest, target)),
+ cleanup,
+ );
+ }
+
+ fn codegen_asm_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ terminator: &mir::Terminator<'tcx>,
+ template: &[ast::InlineAsmTemplatePiece],
+ operands: &[mir::InlineAsmOperand<'tcx>],
+ options: ast::InlineAsmOptions,
+ line_spans: &[Span],
+ destination: Option<mir::BasicBlock>,
+ ) {
+ let span = terminator.source_info.span;
+
+ let operands: Vec<_> = operands
+ .iter()
+ .map(|op| match *op {
+ mir::InlineAsmOperand::In { reg, ref value } => {
+ let value = self.codegen_operand(&mut bx, value);
+ InlineAsmOperandRef::In { reg, value }
+ }
+ mir::InlineAsmOperand::Out { reg, late, ref place } => {
+ let place = place.map(|place| self.codegen_place(&mut bx, place.as_ref()));
+ InlineAsmOperandRef::Out { reg, late, place }
+ }
+ mir::InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => {
+ let in_value = self.codegen_operand(&mut bx, in_value);
+ let out_place =
+ out_place.map(|out_place| self.codegen_place(&mut bx, out_place.as_ref()));
+ InlineAsmOperandRef::InOut { reg, late, in_value, out_place }
+ }
+ mir::InlineAsmOperand::Const { ref value } => {
+ if let mir::Operand::Constant(constant) = value {
+ let const_value = self
+ .eval_mir_constant(constant)
+ .unwrap_or_else(|_| span_bug!(span, "asm const cannot be resolved"));
+ let ty = constant.literal.ty;
+ let size = bx.layout_of(ty).size;
+ let scalar = match const_value {
+ // Promoted constants are evaluated into a ByRef instead of a Scalar,
+ // but we want the scalar value here.
+ ConstValue::ByRef { alloc, offset } => {
+ let ptr = Pointer::new(AllocId(0), offset);
+ alloc
+ .read_scalar(&bx, ptr, size)
+ .and_then(|s| s.check_init())
+ .unwrap_or_else(|e| {
+ bx.tcx().sess.span_err(
+ span,
+ &format!("Could not evaluate asm const: {}", e),
+ );
+
+ // We are erroring out, just emit a dummy constant.
+ Scalar::from_u64(0)
+ })
+ }
+ _ => span_bug!(span, "expected ByRef for promoted asm const"),
+ };
+ let value = scalar.assert_bits(size);
+ let string = match ty.kind {
+ ty::Uint(_) => value.to_string(),
+ ty::Int(int_ty) => {
+ match int_ty.normalize(bx.tcx().sess.target.ptr_width) {
+ ast::IntTy::I8 => (value as i8).to_string(),
+ ast::IntTy::I16 => (value as i16).to_string(),
+ ast::IntTy::I32 => (value as i32).to_string(),
+ ast::IntTy::I64 => (value as i64).to_string(),
+ ast::IntTy::I128 => (value as i128).to_string(),
+ ast::IntTy::Isize => unreachable!(),
+ }
+ }
+ ty::Float(ast::FloatTy::F32) => {
+ f32::from_bits(value as u32).to_string()
+ }
+ ty::Float(ast::FloatTy::F64) => {
+ f64::from_bits(value as u64).to_string()
+ }
+ _ => span_bug!(span, "asm const has bad type {}", ty),
+ };
+ InlineAsmOperandRef::Const { string }
+ } else {
+ span_bug!(span, "asm const is not a constant");
+ }
+ }
+ mir::InlineAsmOperand::SymFn { ref value } => {
+ let literal = self.monomorphize(&value.literal);
+ if let ty::FnDef(def_id, substs) = literal.ty.kind {
+ let instance = ty::Instance::resolve_for_fn_ptr(
+ bx.tcx(),
+ ty::ParamEnv::reveal_all(),
+ def_id,
+ substs,
+ )
+ .unwrap();
+ InlineAsmOperandRef::SymFn { instance }
+ } else {
+ span_bug!(span, "invalid type for asm sym (fn)");
+ }
+ }
+ mir::InlineAsmOperand::SymStatic { def_id } => {
+ InlineAsmOperandRef::SymStatic { def_id }
+ }
+ })
+ .collect();
+
+ bx.codegen_inline_asm(template, &operands, options, line_spans);
+
+ if let Some(target) = destination {
+ helper.funclet_br(self, &mut bx, target);
+ } else {
+ bx.unreachable();
+ }
+ }
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ pub fn codegen_block(&mut self, bb: mir::BasicBlock) {
+ let mut bx = self.build_block(bb);
+ let mir = self.mir;
+ let data = &mir[bb];
+
+ debug!("codegen_block({:?}={:?})", bb, data);
+
+ for statement in &data.statements {
+ bx = self.codegen_statement(bx, statement);
+ }
+
+ self.codegen_terminator(bx, bb, data.terminator());
+ }
+
+ fn codegen_terminator(
+ &mut self,
+ mut bx: Bx,
+ bb: mir::BasicBlock,
+ terminator: &'tcx mir::Terminator<'tcx>,
+ ) {
+ debug!("codegen_terminator: {:?}", terminator);
+
+ // Create the cleanup bundle, if needed.
+ let funclet_bb = self.cleanup_kinds[bb].funclet_bb(bb);
+ let helper = TerminatorCodegenHelper { bb, terminator, funclet_bb };
+
+ self.set_debug_loc(&mut bx, terminator.source_info);
+ match terminator.kind {
+ mir::TerminatorKind::Resume => self.codegen_resume_terminator(helper, bx),
+
+ mir::TerminatorKind::Abort => {
+ bx.abort();
+ // `abort` does not terminate the block, so we still need to generate
+ // an `unreachable` terminator after it.
+ bx.unreachable();
+ }
+
+ mir::TerminatorKind::Goto { target } => {
+ helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+ helper.funclet_br(self, &mut bx, target);
+ }
+
+ mir::TerminatorKind::SwitchInt { ref discr, switch_ty, ref values, ref targets } => {
+ self.codegen_switchint_terminator(helper, bx, discr, switch_ty, values, targets);
+ }
+
+ mir::TerminatorKind::Return => {
+ self.codegen_return_terminator(bx);
+ }
+
+ mir::TerminatorKind::Unreachable => {
+ bx.unreachable();
+ }
+
+ mir::TerminatorKind::Drop { place, target, unwind } => {
+ self.codegen_drop_terminator(helper, bx, place, target, unwind);
+ }
+
+ mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, cleanup } => {
+ self.codegen_assert_terminator(
+ helper, bx, terminator, cond, expected, msg, target, cleanup,
+ );
+ }
+
+ mir::TerminatorKind::DropAndReplace { .. } => {
+ bug!("undesugared DropAndReplace in codegen: {:?}", terminator);
+ }
+
+ mir::TerminatorKind::Call {
+ ref func,
+ ref args,
+ ref destination,
+ cleanup,
+ from_hir_call: _,
+ fn_span,
+ } => {
+ self.codegen_call_terminator(
+ helper,
+ bx,
+ terminator,
+ func,
+ args,
+ destination,
+ cleanup,
+ fn_span,
+ );
+ }
+ mir::TerminatorKind::GeneratorDrop | mir::TerminatorKind::Yield { .. } => {
+ bug!("generator ops in codegen")
+ }
+ mir::TerminatorKind::FalseEdge { .. } | mir::TerminatorKind::FalseUnwind { .. } => {
+ bug!("borrowck false edges in codegen")
+ }
+
+ mir::TerminatorKind::InlineAsm {
+ template,
+ ref operands,
+ options,
+ line_spans,
+ destination,
+ } => {
+ self.codegen_asm_terminator(
+ helper,
+ bx,
+ terminator,
+ template,
+ operands,
+ options,
+ line_spans,
+ destination,
+ );
+ }
+ }
+ }
+
+ fn codegen_argument(
+ &mut self,
+ bx: &mut Bx,
+ op: OperandRef<'tcx, Bx::Value>,
+ llargs: &mut Vec<Bx::Value>,
+ arg: &ArgAbi<'tcx, Ty<'tcx>>,
+ ) {
+ // Fill padding with undef value, where applicable.
+ if let Some(ty) = arg.pad {
+ llargs.push(bx.const_undef(bx.reg_backend_type(&ty)))
+ }
+
+ if arg.is_ignore() {
+ return;
+ }
+
+ if let PassMode::Pair(..) = arg.mode {
+ match op.val {
+ Pair(a, b) => {
+ llargs.push(a);
+ llargs.push(b);
+ return;
+ }
+ _ => bug!("codegen_argument: {:?} invalid for pair argument", op),
+ }
+ } else if arg.is_unsized_indirect() {
+ match op.val {
+ Ref(a, Some(b), _) => {
+ llargs.push(a);
+ llargs.push(b);
+ return;
+ }
+ _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
+ }
+ }
+
+ // Force by-ref if we have to load through a cast pointer.
+ let (mut llval, align, by_ref) = match op.val {
+ Immediate(_) | Pair(..) => match arg.mode {
+ PassMode::Indirect(..) | PassMode::Cast(_) => {
+ let scratch = PlaceRef::alloca(bx, arg.layout);
+ op.val.store(bx, scratch);
+ (scratch.llval, scratch.align, true)
+ }
+ _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi, false),
+ },
+ Ref(llval, _, align) => {
+ if arg.is_indirect() && align < arg.layout.align.abi {
+ // `foo(packed.large_field)`. We can't pass the (unaligned) field directly. I
+ // think that ATM (Rust 1.16) we only pass temporaries, but we shouldn't
+ // have scary latent bugs around.
+
+ let scratch = PlaceRef::alloca(bx, arg.layout);
+ base::memcpy_ty(
+ bx,
+ scratch.llval,
+ scratch.align,
+ llval,
+ align,
+ op.layout,
+ MemFlags::empty(),
+ );
+ (scratch.llval, scratch.align, true)
+ } else {
+ (llval, align, true)
+ }
+ }
+ };
+
+ if by_ref && !arg.is_indirect() {
+ // Have to load the argument, maybe while casting it.
+ if let PassMode::Cast(ty) = arg.mode {
+ let addr = bx.pointercast(llval, bx.type_ptr_to(bx.cast_backend_type(&ty)));
+ llval = bx.load(addr, align.min(arg.layout.align.abi));
+ } else {
+ // We can't use `PlaceRef::load` here because the argument
+ // may have a type we don't treat as immediate, but the ABI
+ // used for this call is passing it by-value. In that case,
+ // the load would just produce `OperandValue::Ref` instead
+ // of the `OperandValue::Immediate` we need for the call.
+ llval = bx.load(llval, align);
+ if let abi::Abi::Scalar(ref scalar) = arg.layout.abi {
+ if scalar.is_bool() {
+ bx.range_metadata(llval, 0..2);
+ }
+ }
+ // We store bools as `i8` so we need to truncate to `i1`.
++ llval = bx.to_immediate(llval, arg.layout);
+ }
+ }
+
+ llargs.push(llval);
+ }
+
+ fn codegen_arguments_untupled(
+ &mut self,
+ bx: &mut Bx,
+ operand: &mir::Operand<'tcx>,
+ llargs: &mut Vec<Bx::Value>,
+ args: &[ArgAbi<'tcx, Ty<'tcx>>],
+ ) {
+ let tuple = self.codegen_operand(bx, operand);
+
+ // Handle both by-ref and immediate tuples.
+ if let Ref(llval, None, align) = tuple.val {
+ let tuple_ptr = PlaceRef::new_sized_aligned(llval, tuple.layout, align);
+ for i in 0..tuple.layout.fields.count() {
+ let field_ptr = tuple_ptr.project_field(bx, i);
+ let field = bx.load_operand(field_ptr);
+ self.codegen_argument(bx, field, llargs, &args[i]);
+ }
+ } else if let Ref(_, Some(_), _) = tuple.val {
+ bug!("closure arguments must be sized")
+ } else {
+ // If the tuple is immediate, the elements are as well.
+ for i in 0..tuple.layout.fields.count() {
+ let op = tuple.extract_field(bx, i);
+ self.codegen_argument(bx, op, llargs, &args[i]);
+ }
+ }
+ }
+
+ fn get_caller_location(&mut self, bx: &mut Bx, span: Span) -> OperandRef<'tcx, Bx::Value> {
+ self.caller_location.unwrap_or_else(|| {
+ let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
+ let caller = bx.tcx().sess.source_map().lookup_char_pos(topmost.lo());
+ let const_loc = bx.tcx().const_caller_location((
+ Symbol::intern(&caller.file.name.to_string()),
+ caller.line as u32,
+ caller.col_display as u32 + 1,
+ ));
+ OperandRef::from_const(bx, const_loc, bx.tcx().caller_location_ty())
+ })
+ }
+
+ fn get_personality_slot(&mut self, bx: &mut Bx) -> PlaceRef<'tcx, Bx::Value> {
+ let cx = bx.cx();
+ if let Some(slot) = self.personality_slot {
+ slot
+ } else {
+ let layout = cx.layout_of(
+ cx.tcx().intern_tup(&[cx.tcx().mk_mut_ptr(cx.tcx().types.u8), cx.tcx().types.i32]),
+ );
+ let slot = PlaceRef::alloca(bx, layout);
+ self.personality_slot = Some(slot);
+ slot
+ }
+ }
+
+ /// Returns the landing-pad wrapper around the given basic block.
+ ///
+ /// No-op in MSVC SEH scheme.
+ fn landing_pad_to(&mut self, target_bb: mir::BasicBlock) -> Bx::BasicBlock {
+ if let Some(block) = self.landing_pads[target_bb] {
+ return block;
+ }
+
+ let block = self.blocks[target_bb];
+ let landing_pad = self.landing_pad_uncached(block);
+ self.landing_pads[target_bb] = Some(landing_pad);
+ landing_pad
+ }
+
+ fn landing_pad_uncached(&mut self, target_bb: Bx::BasicBlock) -> Bx::BasicBlock {
+ if base::wants_msvc_seh(self.cx.sess()) {
+ span_bug!(self.mir.span, "landing pad was not inserted?")
+ }
+
+ let mut bx = self.new_block("cleanup");
+
+ let llpersonality = self.cx.eh_personality();
+ let llretty = self.landing_pad_type();
+ let lp = bx.landing_pad(llretty, llpersonality, 1);
+ bx.set_cleanup(lp);
+
+ let slot = self.get_personality_slot(&mut bx);
+ slot.storage_live(&mut bx);
+ Pair(bx.extract_value(lp, 0), bx.extract_value(lp, 1)).store(&mut bx, slot);
+
+ bx.br(target_bb);
+ bx.llbb()
+ }
+
+ fn landing_pad_type(&self) -> Bx::Type {
+ let cx = self.cx;
+ cx.type_struct(&[cx.type_i8p(), cx.type_i32()], false)
+ }
+
+ fn unreachable_block(&mut self) -> Bx::BasicBlock {
+ self.unreachable_block.unwrap_or_else(|| {
+ let mut bx = self.new_block("unreachable");
+ bx.unreachable();
+ self.unreachable_block = Some(bx.llbb());
+ bx.llbb()
+ })
+ }
+
+ pub fn new_block(&self, name: &str) -> Bx {
+ Bx::new_block(self.cx, self.llfn, name)
+ }
+
+ pub fn build_block(&self, bb: mir::BasicBlock) -> Bx {
+ let mut bx = Bx::with_cx(self.cx);
+ bx.position_at_end(self.blocks[bb]);
+ bx
+ }
+
+ fn make_return_dest(
+ &mut self,
+ bx: &mut Bx,
+ dest: mir::Place<'tcx>,
+ fn_ret: &ArgAbi<'tcx, Ty<'tcx>>,
+ llargs: &mut Vec<Bx::Value>,
+ is_intrinsic: bool,
+ ) -> ReturnDest<'tcx, Bx::Value> {
+ // If the return is ignored, we can just return a do-nothing `ReturnDest`.
+ if fn_ret.is_ignore() {
+ return ReturnDest::Nothing;
+ }
+ let dest = if let Some(index) = dest.as_local() {
+ match self.locals[index] {
+ LocalRef::Place(dest) => dest,
+ LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
+ LocalRef::Operand(None) => {
+ // Handle temporary places, specifically `Operand` ones, as
+ // they don't have `alloca`s.
+ return if fn_ret.is_indirect() {
+ // Odd, but possible, case, we have an operand temporary,
+ // but the calling convention has an indirect return.
+ let tmp = PlaceRef::alloca(bx, fn_ret.layout);
+ tmp.storage_live(bx);
+ llargs.push(tmp.llval);
+ ReturnDest::IndirectOperand(tmp, index)
+ } else if is_intrinsic {
+ // Currently, intrinsics always need a location to store
+ // the result, so we create a temporary `alloca` for the
+ // result.
+ let tmp = PlaceRef::alloca(bx, fn_ret.layout);
+ tmp.storage_live(bx);
+ ReturnDest::IndirectOperand(tmp, index)
+ } else {
+ ReturnDest::DirectOperand(index)
+ };
+ }
+ LocalRef::Operand(Some(_)) => {
+ bug!("place local already assigned to");
+ }
+ }
+ } else {
+ self.codegen_place(
+ bx,
+ mir::PlaceRef { local: dest.local, projection: &dest.projection },
+ )
+ };
+ if fn_ret.is_indirect() {
+ if dest.align < dest.layout.align.abi {
+ // Currently, MIR code generation does not create calls
+ // that store directly to fields of packed structs (in
+ // fact, the calls it creates write only to temps).
+ //
+ // If someone changes that, please update this code path
+ // to create a temporary.
+ span_bug!(self.mir.span, "can't directly store to unaligned value");
+ }
+ llargs.push(dest.llval);
+ ReturnDest::Nothing
+ } else {
+ ReturnDest::Store(dest)
+ }
+ }
+
+ fn codegen_transmute(&mut self, bx: &mut Bx, src: &mir::Operand<'tcx>, dst: mir::Place<'tcx>) {
+ if let Some(index) = dst.as_local() {
+ match self.locals[index] {
+ LocalRef::Place(place) => self.codegen_transmute_into(bx, src, place),
+ LocalRef::UnsizedPlace(_) => bug!("transmute must not involve unsized locals"),
+ LocalRef::Operand(None) => {
+ let dst_layout = bx.layout_of(self.monomorphized_place_ty(dst.as_ref()));
+ assert!(!dst_layout.ty.has_erasable_regions());
+ let place = PlaceRef::alloca(bx, dst_layout);
+ place.storage_live(bx);
+ self.codegen_transmute_into(bx, src, place);
+ let op = bx.load_operand(place);
+ place.storage_dead(bx);
+ self.locals[index] = LocalRef::Operand(Some(op));
+ self.debug_introduce_local(bx, index);
+ }
+ LocalRef::Operand(Some(op)) => {
+ assert!(op.layout.is_zst(), "assigning to initialized SSAtemp");
+ }
+ }
+ } else {
+ let dst = self.codegen_place(bx, dst.as_ref());
+ self.codegen_transmute_into(bx, src, dst);
+ }
+ }
+
+ fn codegen_transmute_into(
+ &mut self,
+ bx: &mut Bx,
+ src: &mir::Operand<'tcx>,
+ dst: PlaceRef<'tcx, Bx::Value>,
+ ) {
+ let src = self.codegen_operand(bx, src);
+ let llty = bx.backend_type(src.layout);
+ let cast_ptr = bx.pointercast(dst.llval, bx.type_ptr_to(llty));
+ let align = src.layout.align.abi.min(dst.align);
+ src.val.store(bx, PlaceRef::new_sized_aligned(cast_ptr, src.layout, align));
+ }
+
+ // Stores the return value of a function call into it's final location.
+ fn store_return(
+ &mut self,
+ bx: &mut Bx,
+ dest: ReturnDest<'tcx, Bx::Value>,
+ ret_abi: &ArgAbi<'tcx, Ty<'tcx>>,
+ llval: Bx::Value,
+ ) {
+ use self::ReturnDest::*;
+
+ match dest {
+ Nothing => (),
+ Store(dst) => bx.store_arg(&ret_abi, llval, dst),
+ IndirectOperand(tmp, index) => {
+ let op = bx.load_operand(tmp);
+ tmp.storage_dead(bx);
+ self.locals[index] = LocalRef::Operand(Some(op));
+ self.debug_introduce_local(bx, index);
+ }
+ DirectOperand(index) => {
+ // If there is a cast, we have to store and reload.
+ let op = if let PassMode::Cast(_) = ret_abi.mode {
+ let tmp = PlaceRef::alloca(bx, ret_abi.layout);
+ tmp.storage_live(bx);
+ bx.store_arg(&ret_abi, llval, tmp);
+ let op = bx.load_operand(tmp);
+ tmp.storage_dead(bx);
+ op
+ } else {
+ OperandRef::from_immediate_or_packed_pair(bx, llval, ret_abi.layout)
+ };
+ self.locals[index] = LocalRef::Operand(Some(op));
+ self.debug_introduce_local(bx, index);
+ }
+ }
+ }
+}
+
+enum ReturnDest<'tcx, V> {
+ // Do nothing; the return value is indirect or ignored.
+ Nothing,
+ // Store the return value to the pointer.
+ Store(PlaceRef<'tcx, V>),
+ // Store an indirect return value to an operand local place.
+ IndirectOperand(PlaceRef<'tcx, V>, mir::Local),
+ // Store a direct return value to an operand local place.
+ DirectOperand(mir::Local),
+}
--- /dev/null
- let imm_a = base::from_immediate(bx, a);
- let imm_b = base::from_immediate(bx, b);
+use super::place::PlaceRef;
+use super::{FunctionCx, LocalRef};
+
+use crate::base;
+use crate::glue;
+use crate::traits::*;
+use crate::MemFlags;
+
+use rustc_errors::ErrorReported;
+use rustc_middle::mir;
+use rustc_middle::mir::interpret::{ConstValue, ErrorHandled, Pointer, Scalar};
+use rustc_middle::ty::layout::TyAndLayout;
+use rustc_middle::ty::Ty;
+use rustc_target::abi::{Abi, Align, LayoutOf, Size};
+
+use std::fmt;
+
+/// The representation of a Rust value. The enum variant is in fact
+/// uniquely determined by the value's type, but is kept as a
+/// safety check.
+#[derive(Copy, Clone, Debug)]
+pub enum OperandValue<V> {
+ /// A reference to the actual operand. The data is guaranteed
+ /// to be valid for the operand's lifetime.
+ /// The second value, if any, is the extra data (vtable or length)
+ /// which indicates that it refers to an unsized rvalue.
+ Ref(V, Option<V>, Align),
+ /// A single LLVM value.
+ Immediate(V),
+ /// A pair of immediate LLVM values. Used by fat pointers too.
+ Pair(V, V),
+}
+
+/// An `OperandRef` is an "SSA" reference to a Rust value, along with
+/// its type.
+///
+/// NOTE: unless you know a value's type exactly, you should not
+/// generate LLVM opcodes acting on it and instead act via methods,
+/// to avoid nasty edge cases. In particular, using `Builder::store`
+/// directly is sure to cause problems -- use `OperandRef::store`
+/// instead.
+#[derive(Copy, Clone)]
+pub struct OperandRef<'tcx, V> {
+ // The value.
+ pub val: OperandValue<V>,
+
+ // The layout of value, based on its Rust type.
+ pub layout: TyAndLayout<'tcx>,
+}
+
+impl<V: CodegenObject> fmt::Debug for OperandRef<'tcx, V> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout)
+ }
+}
+
+impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
+ pub fn new_zst<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ bx: &mut Bx,
+ layout: TyAndLayout<'tcx>,
+ ) -> OperandRef<'tcx, V> {
+ assert!(layout.is_zst());
+ OperandRef {
+ val: OperandValue::Immediate(bx.const_undef(bx.immediate_backend_type(layout))),
+ layout,
+ }
+ }
+
+ pub fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ bx: &mut Bx,
+ val: ConstValue<'tcx>,
+ ty: Ty<'tcx>,
+ ) -> Self {
+ let layout = bx.layout_of(ty);
+
+ if layout.is_zst() {
+ return OperandRef::new_zst(bx, layout);
+ }
+
+ let val = match val {
+ ConstValue::Scalar(x) => {
+ let scalar = match layout.abi {
+ Abi::Scalar(ref x) => x,
+ _ => bug!("from_const: invalid ByVal layout: {:#?}", layout),
+ };
+ let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
+ OperandValue::Immediate(llval)
+ }
+ ConstValue::Slice { data, start, end } => {
+ let a_scalar = match layout.abi {
+ Abi::ScalarPair(ref a, _) => a,
+ _ => bug!("from_const: invalid ScalarPair layout: {:#?}", layout),
+ };
+ let a = Scalar::from(Pointer::new(
+ bx.tcx().create_memory_alloc(data),
+ Size::from_bytes(start),
+ ));
+ let a_llval = bx.scalar_to_backend(
+ a,
+ a_scalar,
+ bx.scalar_pair_element_backend_type(layout, 0, true),
+ );
+ let b_llval = bx.const_usize((end - start) as u64);
+ OperandValue::Pair(a_llval, b_llval)
+ }
+ ConstValue::ByRef { alloc, offset } => {
+ return bx.load_operand(bx.from_const_alloc(layout, alloc, offset));
+ }
+ };
+
+ OperandRef { val, layout }
+ }
+
+ /// Asserts that this operand refers to a scalar and returns
+ /// a reference to its value.
+ pub fn immediate(self) -> V {
+ match self.val {
+ OperandValue::Immediate(s) => s,
+ _ => bug!("not immediate: {:?}", self),
+ }
+ }
+
+ pub fn deref<Cx: LayoutTypeMethods<'tcx>>(self, cx: &Cx) -> PlaceRef<'tcx, V> {
+ let projected_ty = self
+ .layout
+ .ty
+ .builtin_deref(true)
+ .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self))
+ .ty;
+ let (llptr, llextra) = match self.val {
+ OperandValue::Immediate(llptr) => (llptr, None),
+ OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)),
+ OperandValue::Ref(..) => bug!("Deref of by-Ref operand {:?}", self),
+ };
+ let layout = cx.layout_of(projected_ty);
+ PlaceRef { llval: llptr, llextra, layout, align: layout.align.abi }
+ }
+
+ /// If this operand is a `Pair`, we return an aggregate with the two values.
+ /// For other cases, see `immediate`.
+ pub fn immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ ) -> V {
+ if let OperandValue::Pair(a, b) = self.val {
+ let llty = bx.cx().backend_type(self.layout);
+ debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty);
+ // Reconstruct the immediate aggregate.
+ let mut llpair = bx.cx().const_undef(llty);
- let a_llval = base::to_immediate_scalar(bx, a_llval, a);
++ let imm_a = bx.from_immediate(a);
++ let imm_b = bx.from_immediate(b);
+ llpair = bx.insert_value(llpair, imm_a, 0);
+ llpair = bx.insert_value(llpair, imm_b, 1);
+ llpair
+ } else {
+ self.immediate()
+ }
+ }
+
+ /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`.
+ pub fn from_immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ bx: &mut Bx,
+ llval: V,
+ layout: TyAndLayout<'tcx>,
+ ) -> Self {
+ let val = if let Abi::ScalarPair(ref a, ref b) = layout.abi {
+ debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval, layout);
+
+ // Deconstruct the immediate aggregate.
+ let a_llval = bx.extract_value(llval, 0);
- let b_llval = base::to_immediate_scalar(bx, b_llval, b);
++ let a_llval = bx.to_immediate_scalar(a_llval, a);
+ let b_llval = bx.extract_value(llval, 1);
- // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
- // Bools in union fields needs to be truncated.
- let to_immediate_or_cast = |bx: &mut Bx, val, ty| {
- if ty == bx.cx().type_i1() { bx.trunc(val, ty) } else { bx.bitcast(val, ty) }
- };
-
- match val {
- OperandValue::Immediate(ref mut llval) => {
- *llval = to_immediate_or_cast(bx, *llval, bx.cx().immediate_backend_type(field));
++ let b_llval = bx.to_immediate_scalar(b_llval, b);
+ OperandValue::Pair(a_llval, b_llval)
+ } else {
+ OperandValue::Immediate(llval)
+ };
+ OperandRef { val, layout }
+ }
+
+ pub fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ &self,
+ bx: &mut Bx,
+ i: usize,
+ ) -> Self {
+ let field = self.layout.field(bx.cx(), i);
+ let offset = self.layout.fields.offset(i);
+
+ let mut val = match (self.val, &self.layout.abi) {
+ // If the field is ZST, it has no data.
+ _ if field.is_zst() => {
+ return OperandRef::new_zst(bx, field);
+ }
+
+ // Newtype of a scalar, scalar pair or vector.
+ (OperandValue::Immediate(_) | OperandValue::Pair(..), _)
+ if field.size == self.layout.size =>
+ {
+ assert_eq!(offset.bytes(), 0);
+ self.val
+ }
+
+ // Extract a scalar component from a pair.
+ (OperandValue::Pair(a_llval, b_llval), &Abi::ScalarPair(ref a, ref b)) => {
+ if offset.bytes() == 0 {
+ assert_eq!(field.size, a.value.size(bx.cx()));
+ OperandValue::Immediate(a_llval)
+ } else {
+ assert_eq!(offset, a.value.size(bx.cx()).align_to(b.value.align(bx.cx()).abi));
+ assert_eq!(field.size, b.value.size(bx.cx()));
+ OperandValue::Immediate(b_llval)
+ }
+ }
+
+ // `#[repr(simd)]` types are also immediate.
+ (OperandValue::Immediate(llval), &Abi::Vector { .. }) => {
+ OperandValue::Immediate(bx.extract_element(llval, bx.cx().const_usize(i as u64)))
+ }
+
+ _ => bug!("OperandRef::extract_field({:?}): not applicable", self),
+ };
+
- OperandValue::Pair(ref mut a, ref mut b) => {
- *a = to_immediate_or_cast(
- bx,
- *a,
- bx.cx().scalar_pair_element_backend_type(field, 0, true),
- );
- *b = to_immediate_or_cast(
- bx,
- *b,
- bx.cx().scalar_pair_element_backend_type(field, 1, true),
- );
++ match (&mut val, &field.abi) {
++ (OperandValue::Immediate(llval), _) => {
++ // Bools in union fields needs to be truncated.
++ *llval = bx.to_immediate(*llval, field);
++ // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
++ *llval = bx.bitcast(*llval, bx.cx().immediate_backend_type(field));
+ }
- OperandValue::Ref(..) => bug!(),
++ (OperandValue::Pair(a, b), Abi::ScalarPair(a_abi, b_abi)) => {
++ // Bools in union fields needs to be truncated.
++ *a = bx.to_immediate_scalar(*a, a_abi);
++ *b = bx.to_immediate_scalar(*b, b_abi);
++ // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
++ *a = bx.bitcast(*a, bx.cx().scalar_pair_element_backend_type(field, 0, true));
++ *b = bx.bitcast(*b, bx.cx().scalar_pair_element_backend_type(field, 1, true));
+ }
- let val = base::from_immediate(bx, s);
++ (OperandValue::Pair(..), _) => bug!(),
++ (OperandValue::Ref(..), _) => bug!(),
+ }
+
+ OperandRef { val, layout: field }
+ }
+}
+
+impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
+ pub fn store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ dest: PlaceRef<'tcx, V>,
+ ) {
+ self.store_with_flags(bx, dest, MemFlags::empty());
+ }
+
+ pub fn volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ dest: PlaceRef<'tcx, V>,
+ ) {
+ self.store_with_flags(bx, dest, MemFlags::VOLATILE);
+ }
+
+ pub fn unaligned_volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ dest: PlaceRef<'tcx, V>,
+ ) {
+ self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED);
+ }
+
+ pub fn nontemporal_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ dest: PlaceRef<'tcx, V>,
+ ) {
+ self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL);
+ }
+
+ fn store_with_flags<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ dest: PlaceRef<'tcx, V>,
+ flags: MemFlags,
+ ) {
+ debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest);
+ // Avoid generating stores of zero-sized values, because the only way to have a zero-sized
+ // value is through `undef`, and store itself is useless.
+ if dest.layout.is_zst() {
+ return;
+ }
+ match self {
+ OperandValue::Ref(r, None, source_align) => {
+ base::memcpy_ty(bx, dest.llval, dest.align, r, source_align, dest.layout, flags)
+ }
+ OperandValue::Ref(_, Some(_), _) => {
+ bug!("cannot directly store unsized values");
+ }
+ OperandValue::Immediate(s) => {
- let val = base::from_immediate(bx, a);
++ let val = bx.from_immediate(s);
+ bx.store_with_flags(val, dest.llval, dest.align, flags);
+ }
+ OperandValue::Pair(a, b) => {
+ let (a_scalar, b_scalar) = match dest.layout.abi {
+ Abi::ScalarPair(ref a, ref b) => (a, b),
+ _ => bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout),
+ };
+ let b_offset = a_scalar.value.size(bx).align_to(b_scalar.value.align(bx).abi);
+
+ let llptr = bx.struct_gep(dest.llval, 0);
- let val = base::from_immediate(bx, b);
++ let val = bx.from_immediate(a);
+ let align = dest.align;
+ bx.store_with_flags(val, llptr, align, flags);
+
+ let llptr = bx.struct_gep(dest.llval, 1);
++ let val = bx.from_immediate(b);
+ let align = dest.align.restrict_for_offset(b_offset);
+ bx.store_with_flags(val, llptr, align, flags);
+ }
+ }
+ }
+
+ pub fn store_unsized<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ indirect_dest: PlaceRef<'tcx, V>,
+ ) {
+ debug!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest);
+ let flags = MemFlags::empty();
+
+ // `indirect_dest` must have `*mut T` type. We extract `T` out of it.
+ let unsized_ty = indirect_dest
+ .layout
+ .ty
+ .builtin_deref(true)
+ .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest))
+ .ty;
+
+ let (llptr, llextra) = if let OperandValue::Ref(llptr, Some(llextra), _) = self {
+ (llptr, llextra)
+ } else {
+ bug!("store_unsized called with a sized value")
+ };
+
+ // FIXME: choose an appropriate alignment, or use dynamic align somehow
+ let max_align = Align::from_bits(128).unwrap();
+ let min_align = Align::from_bits(8).unwrap();
+
+ // Allocate an appropriate region on the stack, and copy the value into it
+ let (llsize, _) = glue::size_and_align_of_dst(bx, unsized_ty, Some(llextra));
+ let lldst = bx.array_alloca(bx.cx().type_i8(), llsize, max_align);
+ bx.memcpy(lldst, max_align, llptr, min_align, llsize, flags);
+
+ // Store the allocated region and the extra to the indirect place.
+ let indirect_operand = OperandValue::Pair(lldst, llextra);
+ indirect_operand.store(bx, indirect_dest);
+ }
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ fn maybe_codegen_consume_direct(
+ &mut self,
+ bx: &mut Bx,
+ place_ref: mir::PlaceRef<'tcx>,
+ ) -> Option<OperandRef<'tcx, Bx::Value>> {
+ debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref);
+
+ match self.locals[place_ref.local] {
+ LocalRef::Operand(Some(mut o)) => {
+ // Moves out of scalar and scalar pair fields are trivial.
+ for elem in place_ref.projection.iter() {
+ match elem {
+ mir::ProjectionElem::Field(ref f, _) => {
+ o = o.extract_field(bx, f.index());
+ }
+ mir::ProjectionElem::Index(_)
+ | mir::ProjectionElem::ConstantIndex { .. } => {
+ // ZSTs don't require any actual memory access.
+ // FIXME(eddyb) deduplicate this with the identical
+ // checks in `codegen_consume` and `extract_field`.
+ let elem = o.layout.field(bx.cx(), 0);
+ if elem.is_zst() {
+ o = OperandRef::new_zst(bx, elem);
+ } else {
+ return None;
+ }
+ }
+ _ => return None,
+ }
+ }
+
+ Some(o)
+ }
+ LocalRef::Operand(None) => {
+ bug!("use of {:?} before def", place_ref);
+ }
+ LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => {
+ // watch out for locals that do not have an
+ // alloca; they are handled somewhat differently
+ None
+ }
+ }
+ }
+
+ pub fn codegen_consume(
+ &mut self,
+ bx: &mut Bx,
+ place_ref: mir::PlaceRef<'tcx>,
+ ) -> OperandRef<'tcx, Bx::Value> {
+ debug!("codegen_consume(place_ref={:?})", place_ref);
+
+ let ty = self.monomorphized_place_ty(place_ref);
+ let layout = bx.cx().layout_of(ty);
+
+ // ZSTs don't require any actual memory access.
+ if layout.is_zst() {
+ return OperandRef::new_zst(bx, layout);
+ }
+
+ if let Some(o) = self.maybe_codegen_consume_direct(bx, place_ref) {
+ return o;
+ }
+
+ // for most places, to consume them we just load them
+ // out from their home
+ let place = self.codegen_place(bx, place_ref);
+ bx.load_operand(place)
+ }
+
+ pub fn codegen_operand(
+ &mut self,
+ bx: &mut Bx,
+ operand: &mir::Operand<'tcx>,
+ ) -> OperandRef<'tcx, Bx::Value> {
+ debug!("codegen_operand(operand={:?})", operand);
+
+ match *operand {
+ mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
+ self.codegen_consume(bx, place.as_ref())
+ }
+
+ mir::Operand::Constant(ref constant) => {
+ self.eval_mir_constant_to_operand(bx, constant).unwrap_or_else(|err| {
+ match err {
+ // errored or at least linted
+ ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted => {}
+ ErrorHandled::TooGeneric => {
+ bug!("codegen encountered polymorphic constant")
+ }
+ }
+ // Allow RalfJ to sleep soundly knowing that even refactorings that remove
+ // the above error (or silence it under some conditions) will not cause UB.
+ bx.abort();
+ // We still have to return an operand but it doesn't matter,
+ // this code is unreachable.
+ let ty = self.monomorphize(&constant.literal.ty);
+ let layout = bx.cx().layout_of(ty);
+ bx.load_operand(PlaceRef::new_sized(
+ bx.cx().const_undef(bx.cx().type_ptr_to(bx.cx().backend_type(layout))),
+ layout,
+ ))
+ })
+ }
+ }
+ }
+}
--- /dev/null
- let v = base::from_immediate(&mut bx, v);
+use super::operand::{OperandRef, OperandValue};
+use super::place::PlaceRef;
+use super::{FunctionCx, LocalRef};
+
+use crate::base;
+use crate::common::{self, IntPredicate, RealPredicate};
+use crate::traits::*;
+use crate::MemFlags;
+
+use rustc_apfloat::{ieee, Float, Round, Status};
+use rustc_hir::lang_items::LangItem;
+use rustc_middle::mir;
+use rustc_middle::ty::cast::{CastTy, IntTy};
+use rustc_middle::ty::layout::{HasTyCtxt, TyAndLayout};
+use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt};
+use rustc_span::source_map::{Span, DUMMY_SP};
+use rustc_span::symbol::sym;
+use rustc_target::abi::{Abi, Int, LayoutOf, Variants};
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ pub fn codegen_rvalue(
+ &mut self,
+ mut bx: Bx,
+ dest: PlaceRef<'tcx, Bx::Value>,
+ rvalue: &mir::Rvalue<'tcx>,
+ ) -> Bx {
+ debug!("codegen_rvalue(dest.llval={:?}, rvalue={:?})", dest.llval, rvalue);
+
+ match *rvalue {
+ mir::Rvalue::Use(ref operand) => {
+ let cg_operand = self.codegen_operand(&mut bx, operand);
+ // FIXME: consider not copying constants through stack. (Fixable by codegen'ing
+ // constants into `OperandValue::Ref`; why don’t we do that yet if we don’t?)
+ cg_operand.val.store(&mut bx, dest);
+ bx
+ }
+
+ mir::Rvalue::Cast(mir::CastKind::Pointer(PointerCast::Unsize), ref source, _) => {
+ // The destination necessarily contains a fat pointer, so if
+ // it's a scalar pair, it's a fat pointer or newtype thereof.
+ if bx.cx().is_backend_scalar_pair(dest.layout) {
+ // Into-coerce of a thin pointer to a fat pointer -- just
+ // use the operand path.
+ let (mut bx, temp) = self.codegen_rvalue_operand(bx, rvalue);
+ temp.val.store(&mut bx, dest);
+ return bx;
+ }
+
+ // Unsize of a nontrivial struct. I would prefer for
+ // this to be eliminated by MIR building, but
+ // `CoerceUnsized` can be passed by a where-clause,
+ // so the (generic) MIR may not be able to expand it.
+ let operand = self.codegen_operand(&mut bx, source);
+ match operand.val {
+ OperandValue::Pair(..) | OperandValue::Immediate(_) => {
+ // Unsize from an immediate structure. We don't
+ // really need a temporary alloca here, but
+ // avoiding it would require us to have
+ // `coerce_unsized_into` use `extractvalue` to
+ // index into the struct, and this case isn't
+ // important enough for it.
+ debug!("codegen_rvalue: creating ugly alloca");
+ let scratch = PlaceRef::alloca(&mut bx, operand.layout);
+ scratch.storage_live(&mut bx);
+ operand.val.store(&mut bx, scratch);
+ base::coerce_unsized_into(&mut bx, scratch, dest);
+ scratch.storage_dead(&mut bx);
+ }
+ OperandValue::Ref(llref, None, align) => {
+ let source = PlaceRef::new_sized_aligned(llref, operand.layout, align);
+ base::coerce_unsized_into(&mut bx, source, dest);
+ }
+ OperandValue::Ref(_, Some(_), _) => {
+ bug!("unsized coercion on an unsized rvalue");
+ }
+ }
+ bx
+ }
+
+ mir::Rvalue::Repeat(ref elem, count) => {
+ let cg_elem = self.codegen_operand(&mut bx, elem);
+
+ // Do not generate the loop for zero-sized elements or empty arrays.
+ if dest.layout.is_zst() {
+ return bx;
+ }
+
+ if let OperandValue::Immediate(v) = cg_elem.val {
+ let zero = bx.const_usize(0);
+ let start = dest.project_index(&mut bx, zero).llval;
+ let size = bx.const_usize(dest.layout.size.bytes());
+
+ // Use llvm.memset.p0i8.* to initialize all zero arrays
+ if bx.cx().const_to_opt_uint(v) == Some(0) {
+ let fill = bx.cx().const_u8(0);
+ bx.memset(start, fill, size, dest.align, MemFlags::empty());
+ return bx;
+ }
+
+ // Use llvm.memset.p0i8.* to initialize byte arrays
++ let v = bx.from_immediate(v);
+ if bx.cx().val_ty(v) == bx.cx().type_i8() {
+ bx.memset(start, v, size, dest.align, MemFlags::empty());
+ return bx;
+ }
+ }
+
+ let count =
+ self.monomorphize(&count).eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
+
+ bx.write_operand_repeatedly(cg_elem, count, dest)
+ }
+
+ mir::Rvalue::Aggregate(ref kind, ref operands) => {
+ let (dest, active_field_index) = match **kind {
+ mir::AggregateKind::Adt(adt_def, variant_index, _, _, active_field_index) => {
+ dest.codegen_set_discr(&mut bx, variant_index);
+ if adt_def.is_enum() {
+ (dest.project_downcast(&mut bx, variant_index), active_field_index)
+ } else {
+ (dest, active_field_index)
+ }
+ }
+ _ => (dest, None),
+ };
+ for (i, operand) in operands.iter().enumerate() {
+ let op = self.codegen_operand(&mut bx, operand);
+ // Do not generate stores and GEPis for zero-sized fields.
+ if !op.layout.is_zst() {
+ let field_index = active_field_index.unwrap_or(i);
+ let field = dest.project_field(&mut bx, field_index);
+ op.val.store(&mut bx, field);
+ }
+ }
+ bx
+ }
+
+ _ => {
+ assert!(self.rvalue_creates_operand(rvalue, DUMMY_SP));
+ let (mut bx, temp) = self.codegen_rvalue_operand(bx, rvalue);
+ temp.val.store(&mut bx, dest);
+ bx
+ }
+ }
+ }
+
+ pub fn codegen_rvalue_unsized(
+ &mut self,
+ mut bx: Bx,
+ indirect_dest: PlaceRef<'tcx, Bx::Value>,
+ rvalue: &mir::Rvalue<'tcx>,
+ ) -> Bx {
+ debug!(
+ "codegen_rvalue_unsized(indirect_dest.llval={:?}, rvalue={:?})",
+ indirect_dest.llval, rvalue
+ );
+
+ match *rvalue {
+ mir::Rvalue::Use(ref operand) => {
+ let cg_operand = self.codegen_operand(&mut bx, operand);
+ cg_operand.val.store_unsized(&mut bx, indirect_dest);
+ bx
+ }
+
+ _ => bug!("unsized assignment other than `Rvalue::Use`"),
+ }
+ }
+
+ pub fn codegen_rvalue_operand(
+ &mut self,
+ mut bx: Bx,
+ rvalue: &mir::Rvalue<'tcx>,
+ ) -> (Bx, OperandRef<'tcx, Bx::Value>) {
+ assert!(
+ self.rvalue_creates_operand(rvalue, DUMMY_SP),
+ "cannot codegen {:?} to operand",
+ rvalue,
+ );
+
+ match *rvalue {
+ mir::Rvalue::Cast(ref kind, ref source, mir_cast_ty) => {
+ let operand = self.codegen_operand(&mut bx, source);
+ debug!("cast operand is {:?}", operand);
+ let cast = bx.cx().layout_of(self.monomorphize(&mir_cast_ty));
+
+ let val = match *kind {
+ mir::CastKind::Pointer(PointerCast::ReifyFnPointer) => {
+ match operand.layout.ty.kind {
+ ty::FnDef(def_id, substs) => {
+ if bx.cx().tcx().has_attr(def_id, sym::rustc_args_required_const) {
+ bug!("reifying a fn ptr that requires const arguments");
+ }
+ let instance = ty::Instance::resolve_for_fn_ptr(
+ bx.tcx(),
+ ty::ParamEnv::reveal_all(),
+ def_id,
+ substs,
+ )
+ .unwrap()
+ .polymorphize(bx.cx().tcx());
+ OperandValue::Immediate(bx.get_fn_addr(instance))
+ }
+ _ => bug!("{} cannot be reified to a fn ptr", operand.layout.ty),
+ }
+ }
+ mir::CastKind::Pointer(PointerCast::ClosureFnPointer(_)) => {
+ match operand.layout.ty.kind {
+ ty::Closure(def_id, substs) => {
+ let instance = Instance::resolve_closure(
+ bx.cx().tcx(),
+ def_id,
+ substs,
+ ty::ClosureKind::FnOnce,
+ )
+ .polymorphize(bx.cx().tcx());
+ OperandValue::Immediate(bx.cx().get_fn_addr(instance))
+ }
+ _ => bug!("{} cannot be cast to a fn ptr", operand.layout.ty),
+ }
+ }
+ mir::CastKind::Pointer(PointerCast::UnsafeFnPointer) => {
+ // This is a no-op at the LLVM level.
+ operand.val
+ }
+ mir::CastKind::Pointer(PointerCast::Unsize) => {
+ assert!(bx.cx().is_backend_scalar_pair(cast));
+ match operand.val {
+ OperandValue::Pair(lldata, llextra) => {
+ // unsize from a fat pointer -- this is a
+ // "trait-object-to-supertrait" coercion, for
+ // example, `&'a fmt::Debug + Send => &'a fmt::Debug`.
+
+ // HACK(eddyb) have to bitcast pointers
+ // until LLVM removes pointee types.
+ let lldata = bx.pointercast(
+ lldata,
+ bx.cx().scalar_pair_element_backend_type(cast, 0, true),
+ );
+ OperandValue::Pair(lldata, llextra)
+ }
+ OperandValue::Immediate(lldata) => {
+ // "standard" unsize
+ let (lldata, llextra) = base::unsize_thin_ptr(
+ &mut bx,
+ lldata,
+ operand.layout.ty,
+ cast.ty,
+ );
+ OperandValue::Pair(lldata, llextra)
+ }
+ OperandValue::Ref(..) => {
+ bug!("by-ref operand {:?} in `codegen_rvalue_operand`", operand);
+ }
+ }
+ }
+ mir::CastKind::Pointer(PointerCast::MutToConstPointer)
+ | mir::CastKind::Misc
+ if bx.cx().is_backend_scalar_pair(operand.layout) =>
+ {
+ if let OperandValue::Pair(data_ptr, meta) = operand.val {
+ if bx.cx().is_backend_scalar_pair(cast) {
+ let data_cast = bx.pointercast(
+ data_ptr,
+ bx.cx().scalar_pair_element_backend_type(cast, 0, true),
+ );
+ OperandValue::Pair(data_cast, meta)
+ } else {
+ // cast to thin-ptr
+ // Cast of fat-ptr to thin-ptr is an extraction of data-ptr and
+ // pointer-cast of that pointer to desired pointer type.
+ let llcast_ty = bx.cx().immediate_backend_type(cast);
+ let llval = bx.pointercast(data_ptr, llcast_ty);
+ OperandValue::Immediate(llval)
+ }
+ } else {
+ bug!("unexpected non-pair operand");
+ }
+ }
+ mir::CastKind::Pointer(
+ PointerCast::MutToConstPointer | PointerCast::ArrayToPointer,
+ )
+ | mir::CastKind::Misc => {
+ assert!(bx.cx().is_backend_immediate(cast));
+ let ll_t_out = bx.cx().immediate_backend_type(cast);
+ if operand.layout.abi.is_uninhabited() {
+ let val = OperandValue::Immediate(bx.cx().const_undef(ll_t_out));
+ return (bx, OperandRef { val, layout: cast });
+ }
+ let r_t_in =
+ CastTy::from_ty(operand.layout.ty).expect("bad input type for cast");
+ let r_t_out = CastTy::from_ty(cast.ty).expect("bad output type for cast");
+ let ll_t_in = bx.cx().immediate_backend_type(operand.layout);
+ match operand.layout.variants {
+ Variants::Single { index } => {
+ if let Some(discr) =
+ operand.layout.ty.discriminant_for_variant(bx.tcx(), index)
+ {
+ let discr_layout = bx.cx().layout_of(discr.ty);
+ let discr_t = bx.cx().immediate_backend_type(discr_layout);
+ let discr_val = bx.cx().const_uint_big(discr_t, discr.val);
+ let discr_val =
+ bx.intcast(discr_val, ll_t_out, discr.ty.is_signed());
+
+ return (
+ bx,
+ OperandRef {
+ val: OperandValue::Immediate(discr_val),
+ layout: cast,
+ },
+ );
+ }
+ }
+ Variants::Multiple { .. } => {}
+ }
+ let llval = operand.immediate();
+
+ let mut signed = false;
+ if let Abi::Scalar(ref scalar) = operand.layout.abi {
+ if let Int(_, s) = scalar.value {
+ // We use `i1` for bytes that are always `0` or `1`,
+ // e.g., `#[repr(i8)] enum E { A, B }`, but we can't
+ // let LLVM interpret the `i1` as signed, because
+ // then `i1 1` (i.e., E::B) is effectively `i8 -1`.
+ signed = !scalar.is_bool() && s;
+
+ let er = scalar.valid_range_exclusive(bx.cx());
+ if er.end != er.start
+ && scalar.valid_range.end() > scalar.valid_range.start()
+ {
+ // We want `table[e as usize]` to not
+ // have bound checks, and this is the most
+ // convenient place to put the `assume`.
+ let ll_t_in_const =
+ bx.cx().const_uint_big(ll_t_in, *scalar.valid_range.end());
+ let cmp = bx.icmp(IntPredicate::IntULE, llval, ll_t_in_const);
+ bx.assume(cmp);
+ }
+ }
+ }
+
+ let newval = match (r_t_in, r_t_out) {
+ (CastTy::Int(_), CastTy::Int(_)) => bx.intcast(llval, ll_t_out, signed),
+ (CastTy::Float, CastTy::Float) => {
+ let srcsz = bx.cx().float_width(ll_t_in);
+ let dstsz = bx.cx().float_width(ll_t_out);
+ if dstsz > srcsz {
+ bx.fpext(llval, ll_t_out)
+ } else if srcsz > dstsz {
+ bx.fptrunc(llval, ll_t_out)
+ } else {
+ llval
+ }
+ }
+ (CastTy::Int(_), CastTy::Float) => {
+ if signed {
+ bx.sitofp(llval, ll_t_out)
+ } else {
+ bx.uitofp(llval, ll_t_out)
+ }
+ }
+ (CastTy::Ptr(_) | CastTy::FnPtr, CastTy::Ptr(_)) => {
+ bx.pointercast(llval, ll_t_out)
+ }
+ (CastTy::Ptr(_) | CastTy::FnPtr, CastTy::Int(_)) => {
+ bx.ptrtoint(llval, ll_t_out)
+ }
+ (CastTy::Int(_), CastTy::Ptr(_)) => {
+ let usize_llval = bx.intcast(llval, bx.cx().type_isize(), signed);
+ bx.inttoptr(usize_llval, ll_t_out)
+ }
+ (CastTy::Float, CastTy::Int(IntTy::I)) => {
+ cast_float_to_int(&mut bx, true, llval, ll_t_in, ll_t_out, cast)
+ }
+ (CastTy::Float, CastTy::Int(_)) => {
+ cast_float_to_int(&mut bx, false, llval, ll_t_in, ll_t_out, cast)
+ }
+ _ => bug!("unsupported cast: {:?} to {:?}", operand.layout.ty, cast.ty),
+ };
+ OperandValue::Immediate(newval)
+ }
+ };
+ (bx, OperandRef { val, layout: cast })
+ }
+
+ mir::Rvalue::Ref(_, bk, place) => {
+ let mk_ref = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| {
+ tcx.mk_ref(
+ tcx.lifetimes.re_erased,
+ ty::TypeAndMut { ty, mutbl: bk.to_mutbl_lossy() },
+ )
+ };
+ self.codegen_place_to_pointer(bx, place, mk_ref)
+ }
+
+ mir::Rvalue::AddressOf(mutability, place) => {
+ let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| {
+ tcx.mk_ptr(ty::TypeAndMut { ty, mutbl: mutability })
+ };
+ self.codegen_place_to_pointer(bx, place, mk_ptr)
+ }
+
+ mir::Rvalue::Len(place) => {
+ let size = self.evaluate_array_len(&mut bx, place);
+ let operand = OperandRef {
+ val: OperandValue::Immediate(size),
+ layout: bx.cx().layout_of(bx.tcx().types.usize),
+ };
+ (bx, operand)
+ }
+
+ mir::Rvalue::BinaryOp(op, ref lhs, ref rhs) => {
+ let lhs = self.codegen_operand(&mut bx, lhs);
+ let rhs = self.codegen_operand(&mut bx, rhs);
+ let llresult = match (lhs.val, rhs.val) {
+ (
+ OperandValue::Pair(lhs_addr, lhs_extra),
+ OperandValue::Pair(rhs_addr, rhs_extra),
+ ) => self.codegen_fat_ptr_binop(
+ &mut bx,
+ op,
+ lhs_addr,
+ lhs_extra,
+ rhs_addr,
+ rhs_extra,
+ lhs.layout.ty,
+ ),
+
+ (OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => {
+ self.codegen_scalar_binop(&mut bx, op, lhs_val, rhs_val, lhs.layout.ty)
+ }
+
+ _ => bug!(),
+ };
+ let operand = OperandRef {
+ val: OperandValue::Immediate(llresult),
+ layout: bx.cx().layout_of(op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty)),
+ };
+ (bx, operand)
+ }
+ mir::Rvalue::CheckedBinaryOp(op, ref lhs, ref rhs) => {
+ let lhs = self.codegen_operand(&mut bx, lhs);
+ let rhs = self.codegen_operand(&mut bx, rhs);
+ let result = self.codegen_scalar_checked_binop(
+ &mut bx,
+ op,
+ lhs.immediate(),
+ rhs.immediate(),
+ lhs.layout.ty,
+ );
+ let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty);
+ let operand_ty = bx.tcx().intern_tup(&[val_ty, bx.tcx().types.bool]);
+ let operand = OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) };
+
+ (bx, operand)
+ }
+
+ mir::Rvalue::UnaryOp(op, ref operand) => {
+ let operand = self.codegen_operand(&mut bx, operand);
+ let lloperand = operand.immediate();
+ let is_float = operand.layout.ty.is_floating_point();
+ let llval = match op {
+ mir::UnOp::Not => bx.not(lloperand),
+ mir::UnOp::Neg => {
+ if is_float {
+ bx.fneg(lloperand)
+ } else {
+ bx.neg(lloperand)
+ }
+ }
+ };
+ (bx, OperandRef { val: OperandValue::Immediate(llval), layout: operand.layout })
+ }
+
+ mir::Rvalue::Discriminant(ref place) => {
+ let discr_ty = rvalue.ty(self.mir, bx.tcx());
+ let discr = self
+ .codegen_place(&mut bx, place.as_ref())
+ .codegen_get_discr(&mut bx, discr_ty);
+ (
+ bx,
+ OperandRef {
+ val: OperandValue::Immediate(discr),
+ layout: self.cx.layout_of(discr_ty),
+ },
+ )
+ }
+
+ mir::Rvalue::NullaryOp(mir::NullOp::SizeOf, ty) => {
+ assert!(bx.cx().type_is_sized(ty));
+ let val = bx.cx().const_usize(bx.cx().layout_of(ty).size.bytes());
+ let tcx = self.cx.tcx();
+ (
+ bx,
+ OperandRef {
+ val: OperandValue::Immediate(val),
+ layout: self.cx.layout_of(tcx.types.usize),
+ },
+ )
+ }
+
+ mir::Rvalue::NullaryOp(mir::NullOp::Box, content_ty) => {
+ let content_ty = self.monomorphize(&content_ty);
+ let content_layout = bx.cx().layout_of(content_ty);
+ let llsize = bx.cx().const_usize(content_layout.size.bytes());
+ let llalign = bx.cx().const_usize(content_layout.align.abi.bytes());
+ let box_layout = bx.cx().layout_of(bx.tcx().mk_box(content_ty));
+ let llty_ptr = bx.cx().backend_type(box_layout);
+
+ // Allocate space:
+ let def_id = match bx.tcx().lang_items().require(LangItem::ExchangeMalloc) {
+ Ok(id) => id,
+ Err(s) => {
+ bx.cx().sess().fatal(&format!("allocation of `{}` {}", box_layout.ty, s));
+ }
+ };
+ let instance = ty::Instance::mono(bx.tcx(), def_id);
+ let r = bx.cx().get_fn_addr(instance);
+ let call = bx.call(r, &[llsize, llalign], None);
+ let val = bx.pointercast(call, llty_ptr);
+
+ let operand = OperandRef { val: OperandValue::Immediate(val), layout: box_layout };
+ (bx, operand)
+ }
+ mir::Rvalue::ThreadLocalRef(def_id) => {
+ assert!(bx.cx().tcx().is_static(def_id));
+ let static_ = bx.get_static(def_id);
+ let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id));
+ let operand = OperandRef::from_immediate_or_packed_pair(&mut bx, static_, layout);
+ (bx, operand)
+ }
+ mir::Rvalue::Use(ref operand) => {
+ let operand = self.codegen_operand(&mut bx, operand);
+ (bx, operand)
+ }
+ mir::Rvalue::Repeat(..) | mir::Rvalue::Aggregate(..) => {
+ // According to `rvalue_creates_operand`, only ZST
+ // aggregate rvalues are allowed to be operands.
+ let ty = rvalue.ty(self.mir, self.cx.tcx());
+ let operand =
+ OperandRef::new_zst(&mut bx, self.cx.layout_of(self.monomorphize(&ty)));
+ (bx, operand)
+ }
+ }
+ }
+
+ fn evaluate_array_len(&mut self, bx: &mut Bx, place: mir::Place<'tcx>) -> Bx::Value {
+ // ZST are passed as operands and require special handling
+ // because codegen_place() panics if Local is operand.
+ if let Some(index) = place.as_local() {
+ if let LocalRef::Operand(Some(op)) = self.locals[index] {
+ if let ty::Array(_, n) = op.layout.ty.kind {
+ let n = n.eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
+ return bx.cx().const_usize(n);
+ }
+ }
+ }
+ // use common size calculation for non zero-sized types
+ let cg_value = self.codegen_place(bx, place.as_ref());
+ cg_value.len(bx.cx())
+ }
+
+ /// Codegen an `Rvalue::AddressOf` or `Rvalue::Ref`
+ fn codegen_place_to_pointer(
+ &mut self,
+ mut bx: Bx,
+ place: mir::Place<'tcx>,
+ mk_ptr_ty: impl FnOnce(TyCtxt<'tcx>, Ty<'tcx>) -> Ty<'tcx>,
+ ) -> (Bx, OperandRef<'tcx, Bx::Value>) {
+ let cg_place = self.codegen_place(&mut bx, place.as_ref());
+
+ let ty = cg_place.layout.ty;
+
+ // Note: places are indirect, so storing the `llval` into the
+ // destination effectively creates a reference.
+ let val = if !bx.cx().type_has_metadata(ty) {
+ OperandValue::Immediate(cg_place.llval)
+ } else {
+ OperandValue::Pair(cg_place.llval, cg_place.llextra.unwrap())
+ };
+ (bx, OperandRef { val, layout: self.cx.layout_of(mk_ptr_ty(self.cx.tcx(), ty)) })
+ }
+
+ pub fn codegen_scalar_binop(
+ &mut self,
+ bx: &mut Bx,
+ op: mir::BinOp,
+ lhs: Bx::Value,
+ rhs: Bx::Value,
+ input_ty: Ty<'tcx>,
+ ) -> Bx::Value {
+ let is_float = input_ty.is_floating_point();
+ let is_signed = input_ty.is_signed();
+ match op {
+ mir::BinOp::Add => {
+ if is_float {
+ bx.fadd(lhs, rhs)
+ } else {
+ bx.add(lhs, rhs)
+ }
+ }
+ mir::BinOp::Sub => {
+ if is_float {
+ bx.fsub(lhs, rhs)
+ } else {
+ bx.sub(lhs, rhs)
+ }
+ }
+ mir::BinOp::Mul => {
+ if is_float {
+ bx.fmul(lhs, rhs)
+ } else {
+ bx.mul(lhs, rhs)
+ }
+ }
+ mir::BinOp::Div => {
+ if is_float {
+ bx.fdiv(lhs, rhs)
+ } else if is_signed {
+ bx.sdiv(lhs, rhs)
+ } else {
+ bx.udiv(lhs, rhs)
+ }
+ }
+ mir::BinOp::Rem => {
+ if is_float {
+ bx.frem(lhs, rhs)
+ } else if is_signed {
+ bx.srem(lhs, rhs)
+ } else {
+ bx.urem(lhs, rhs)
+ }
+ }
+ mir::BinOp::BitOr => bx.or(lhs, rhs),
+ mir::BinOp::BitAnd => bx.and(lhs, rhs),
+ mir::BinOp::BitXor => bx.xor(lhs, rhs),
+ mir::BinOp::Offset => bx.inbounds_gep(lhs, &[rhs]),
+ mir::BinOp::Shl => common::build_unchecked_lshift(bx, lhs, rhs),
+ mir::BinOp::Shr => common::build_unchecked_rshift(bx, input_ty, lhs, rhs),
+ mir::BinOp::Ne
+ | mir::BinOp::Lt
+ | mir::BinOp::Gt
+ | mir::BinOp::Eq
+ | mir::BinOp::Le
+ | mir::BinOp::Ge => {
+ if is_float {
+ bx.fcmp(base::bin_op_to_fcmp_predicate(op.to_hir_binop()), lhs, rhs)
+ } else {
+ bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs)
+ }
+ }
+ }
+ }
+
+ pub fn codegen_fat_ptr_binop(
+ &mut self,
+ bx: &mut Bx,
+ op: mir::BinOp,
+ lhs_addr: Bx::Value,
+ lhs_extra: Bx::Value,
+ rhs_addr: Bx::Value,
+ rhs_extra: Bx::Value,
+ _input_ty: Ty<'tcx>,
+ ) -> Bx::Value {
+ match op {
+ mir::BinOp::Eq => {
+ let lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr);
+ let rhs = bx.icmp(IntPredicate::IntEQ, lhs_extra, rhs_extra);
+ bx.and(lhs, rhs)
+ }
+ mir::BinOp::Ne => {
+ let lhs = bx.icmp(IntPredicate::IntNE, lhs_addr, rhs_addr);
+ let rhs = bx.icmp(IntPredicate::IntNE, lhs_extra, rhs_extra);
+ bx.or(lhs, rhs)
+ }
+ mir::BinOp::Le | mir::BinOp::Lt | mir::BinOp::Ge | mir::BinOp::Gt => {
+ // a OP b ~ a.0 STRICT(OP) b.0 | (a.0 == b.0 && a.1 OP a.1)
+ let (op, strict_op) = match op {
+ mir::BinOp::Lt => (IntPredicate::IntULT, IntPredicate::IntULT),
+ mir::BinOp::Le => (IntPredicate::IntULE, IntPredicate::IntULT),
+ mir::BinOp::Gt => (IntPredicate::IntUGT, IntPredicate::IntUGT),
+ mir::BinOp::Ge => (IntPredicate::IntUGE, IntPredicate::IntUGT),
+ _ => bug!(),
+ };
+ let lhs = bx.icmp(strict_op, lhs_addr, rhs_addr);
+ let and_lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr);
+ let and_rhs = bx.icmp(op, lhs_extra, rhs_extra);
+ let rhs = bx.and(and_lhs, and_rhs);
+ bx.or(lhs, rhs)
+ }
+ _ => {
+ bug!("unexpected fat ptr binop");
+ }
+ }
+ }
+
+ pub fn codegen_scalar_checked_binop(
+ &mut self,
+ bx: &mut Bx,
+ op: mir::BinOp,
+ lhs: Bx::Value,
+ rhs: Bx::Value,
+ input_ty: Ty<'tcx>,
+ ) -> OperandValue<Bx::Value> {
+ // This case can currently arise only from functions marked
+ // with #[rustc_inherit_overflow_checks] and inlined from
+ // another crate (mostly core::num generic/#[inline] fns),
+ // while the current crate doesn't use overflow checks.
+ if !bx.cx().check_overflow() {
+ let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty);
+ return OperandValue::Pair(val, bx.cx().const_bool(false));
+ }
+
+ let (val, of) = match op {
+ // These are checked using intrinsics
+ mir::BinOp::Add | mir::BinOp::Sub | mir::BinOp::Mul => {
+ let oop = match op {
+ mir::BinOp::Add => OverflowOp::Add,
+ mir::BinOp::Sub => OverflowOp::Sub,
+ mir::BinOp::Mul => OverflowOp::Mul,
+ _ => unreachable!(),
+ };
+ bx.checked_binop(oop, input_ty, lhs, rhs)
+ }
+ mir::BinOp::Shl | mir::BinOp::Shr => {
+ let lhs_llty = bx.cx().val_ty(lhs);
+ let rhs_llty = bx.cx().val_ty(rhs);
+ let invert_mask = common::shift_mask_val(bx, lhs_llty, rhs_llty, true);
+ let outer_bits = bx.and(rhs, invert_mask);
+
+ let of = bx.icmp(IntPredicate::IntNE, outer_bits, bx.cx().const_null(rhs_llty));
+ let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty);
+
+ (val, of)
+ }
+ _ => bug!("Operator `{:?}` is not a checkable operator", op),
+ };
+
+ OperandValue::Pair(val, of)
+ }
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool {
+ match *rvalue {
+ mir::Rvalue::Ref(..) |
+ mir::Rvalue::AddressOf(..) |
+ mir::Rvalue::Len(..) |
+ mir::Rvalue::Cast(..) | // (*)
+ mir::Rvalue::BinaryOp(..) |
+ mir::Rvalue::CheckedBinaryOp(..) |
+ mir::Rvalue::UnaryOp(..) |
+ mir::Rvalue::Discriminant(..) |
+ mir::Rvalue::NullaryOp(..) |
+ mir::Rvalue::ThreadLocalRef(_) |
+ mir::Rvalue::Use(..) => // (*)
+ true,
+ mir::Rvalue::Repeat(..) |
+ mir::Rvalue::Aggregate(..) => {
+ let ty = rvalue.ty(self.mir, self.cx.tcx());
+ let ty = self.monomorphize(&ty);
+ self.cx.spanned_layout_of(ty, span).is_zst()
+ }
+ }
+
+ // (*) this is only true if the type is suitable
+ }
+}
+
+fn cast_float_to_int<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ signed: bool,
+ x: Bx::Value,
+ float_ty: Bx::Type,
+ int_ty: Bx::Type,
+ int_layout: TyAndLayout<'tcx>,
+) -> Bx::Value {
+ if let Some(false) = bx.cx().sess().opts.debugging_opts.saturating_float_casts {
+ return if signed { bx.fptosi(x, int_ty) } else { bx.fptoui(x, int_ty) };
+ }
+
+ let try_sat_result = if signed { bx.fptosi_sat(x, int_ty) } else { bx.fptoui_sat(x, int_ty) };
+ if let Some(try_sat_result) = try_sat_result {
+ return try_sat_result;
+ }
+
+ let int_width = bx.cx().int_width(int_ty);
+ let float_width = bx.cx().float_width(float_ty);
+ // LLVM's fpto[su]i returns undef when the input x is infinite, NaN, or does not fit into the
+ // destination integer type after rounding towards zero. This `undef` value can cause UB in
+ // safe code (see issue #10184), so we implement a saturating conversion on top of it:
+ // Semantically, the mathematical value of the input is rounded towards zero to the next
+ // mathematical integer, and then the result is clamped into the range of the destination
+ // integer type. Positive and negative infinity are mapped to the maximum and minimum value of
+ // the destination integer type. NaN is mapped to 0.
+ //
+ // Define f_min and f_max as the largest and smallest (finite) floats that are exactly equal to
+ // a value representable in int_ty.
+ // They are exactly equal to int_ty::{MIN,MAX} if float_ty has enough significand bits.
+ // Otherwise, int_ty::MAX must be rounded towards zero, as it is one less than a power of two.
+ // int_ty::MIN, however, is either zero or a negative power of two and is thus exactly
+ // representable. Note that this only works if float_ty's exponent range is sufficiently large.
+ // f16 or 256 bit integers would break this property. Right now the smallest float type is f32
+ // with exponents ranging up to 127, which is barely enough for i128::MIN = -2^127.
+ // On the other hand, f_max works even if int_ty::MAX is greater than float_ty::MAX. Because
+ // we're rounding towards zero, we just get float_ty::MAX (which is always an integer).
+ // This already happens today with u128::MAX = 2^128 - 1 > f32::MAX.
+ let int_max = |signed: bool, int_width: u64| -> u128 {
+ let shift_amount = 128 - int_width;
+ if signed { i128::MAX as u128 >> shift_amount } else { u128::MAX >> shift_amount }
+ };
+ let int_min = |signed: bool, int_width: u64| -> i128 {
+ if signed { i128::MIN >> (128 - int_width) } else { 0 }
+ };
+
+ let compute_clamp_bounds_single = |signed: bool, int_width: u64| -> (u128, u128) {
+ let rounded_min = ieee::Single::from_i128_r(int_min(signed, int_width), Round::TowardZero);
+ assert_eq!(rounded_min.status, Status::OK);
+ let rounded_max = ieee::Single::from_u128_r(int_max(signed, int_width), Round::TowardZero);
+ assert!(rounded_max.value.is_finite());
+ (rounded_min.value.to_bits(), rounded_max.value.to_bits())
+ };
+ let compute_clamp_bounds_double = |signed: bool, int_width: u64| -> (u128, u128) {
+ let rounded_min = ieee::Double::from_i128_r(int_min(signed, int_width), Round::TowardZero);
+ assert_eq!(rounded_min.status, Status::OK);
+ let rounded_max = ieee::Double::from_u128_r(int_max(signed, int_width), Round::TowardZero);
+ assert!(rounded_max.value.is_finite());
+ (rounded_min.value.to_bits(), rounded_max.value.to_bits())
+ };
+
+ let mut float_bits_to_llval = |bits| {
+ let bits_llval = match float_width {
+ 32 => bx.cx().const_u32(bits as u32),
+ 64 => bx.cx().const_u64(bits as u64),
+ n => bug!("unsupported float width {}", n),
+ };
+ bx.bitcast(bits_llval, float_ty)
+ };
+ let (f_min, f_max) = match float_width {
+ 32 => compute_clamp_bounds_single(signed, int_width),
+ 64 => compute_clamp_bounds_double(signed, int_width),
+ n => bug!("unsupported float width {}", n),
+ };
+ let f_min = float_bits_to_llval(f_min);
+ let f_max = float_bits_to_llval(f_max);
+ // To implement saturation, we perform the following steps:
+ //
+ // 1. Cast x to an integer with fpto[su]i. This may result in undef.
+ // 2. Compare x to f_min and f_max, and use the comparison results to select:
+ // a) int_ty::MIN if x < f_min or x is NaN
+ // b) int_ty::MAX if x > f_max
+ // c) the result of fpto[su]i otherwise
+ // 3. If x is NaN, return 0.0, otherwise return the result of step 2.
+ //
+ // This avoids resulting undef because values in range [f_min, f_max] by definition fit into the
+ // destination type. It creates an undef temporary, but *producing* undef is not UB. Our use of
+ // undef does not introduce any non-determinism either.
+ // More importantly, the above procedure correctly implements saturating conversion.
+ // Proof (sketch):
+ // If x is NaN, 0 is returned by definition.
+ // Otherwise, x is finite or infinite and thus can be compared with f_min and f_max.
+ // This yields three cases to consider:
+ // (1) if x in [f_min, f_max], the result of fpto[su]i is returned, which agrees with
+ // saturating conversion for inputs in that range.
+ // (2) if x > f_max, then x is larger than int_ty::MAX. This holds even if f_max is rounded
+ // (i.e., if f_max < int_ty::MAX) because in those cases, nextUp(f_max) is already larger
+ // than int_ty::MAX. Because x is larger than int_ty::MAX, the return value of int_ty::MAX
+ // is correct.
+ // (3) if x < f_min, then x is smaller than int_ty::MIN. As shown earlier, f_min exactly equals
+ // int_ty::MIN and therefore the return value of int_ty::MIN is correct.
+ // QED.
+
+ let int_max = bx.cx().const_uint_big(int_ty, int_max(signed, int_width));
+ let int_min = bx.cx().const_uint_big(int_ty, int_min(signed, int_width) as u128);
+ let zero = bx.cx().const_uint(int_ty, 0);
+
+ // The codegen here differs quite a bit depending on whether our builder's
+ // `fptosi` and `fptoui` instructions may trap for out-of-bounds values. If
+ // they don't trap then we can start doing everything inline with a
+ // `select` instruction because it's ok to execute `fptosi` and `fptoui`
+ // even if we don't use the results.
+ if !bx.fptosui_may_trap(x, int_ty) {
+ // Step 1 ...
+ let fptosui_result = if signed { bx.fptosi(x, int_ty) } else { bx.fptoui(x, int_ty) };
+ let less_or_nan = bx.fcmp(RealPredicate::RealULT, x, f_min);
+ let greater = bx.fcmp(RealPredicate::RealOGT, x, f_max);
+
+ // Step 2: We use two comparisons and two selects, with %s1 being the
+ // result:
+ // %less_or_nan = fcmp ult %x, %f_min
+ // %greater = fcmp olt %x, %f_max
+ // %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result
+ // %s1 = select %greater, int_ty::MAX, %s0
+ // Note that %less_or_nan uses an *unordered* comparison. This
+ // comparison is true if the operands are not comparable (i.e., if x is
+ // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if
+ // x is NaN.
+ //
+ // Performance note: Unordered comparison can be lowered to a "flipped"
+ // comparison and a negation, and the negation can be merged into the
+ // select. Therefore, it not necessarily any more expensive than a
+ // ordered ("normal") comparison. Whether these optimizations will be
+ // performed is ultimately up to the backend, but at least x86 does
+ // perform them.
+ let s0 = bx.select(less_or_nan, int_min, fptosui_result);
+ let s1 = bx.select(greater, int_max, s0);
+
+ // Step 3: NaN replacement.
+ // For unsigned types, the above step already yielded int_ty::MIN == 0 if x is NaN.
+ // Therefore we only need to execute this step for signed integer types.
+ if signed {
+ // LLVM has no isNaN predicate, so we use (x == x) instead
+ let cmp = bx.fcmp(RealPredicate::RealOEQ, x, x);
+ bx.select(cmp, s1, zero)
+ } else {
+ s1
+ }
+ } else {
+ // In this case we cannot execute `fptosi` or `fptoui` and then later
+ // discard the result. The builder is telling us that these instructions
+ // will trap on out-of-bounds values, so we need to use basic blocks and
+ // control flow to avoid executing the `fptosi` and `fptoui`
+ // instructions.
+ //
+ // The general idea of what we're constructing here is, for f64 -> i32:
+ //
+ // ;; block so far... %0 is the argument
+ // %result = alloca i32, align 4
+ // %inbound_lower = fcmp oge double %0, 0xC1E0000000000000
+ // %inbound_upper = fcmp ole double %0, 0x41DFFFFFFFC00000
+ // ;; match (inbound_lower, inbound_upper) {
+ // ;; (true, true) => %0 can be converted without trapping
+ // ;; (false, false) => %0 is a NaN
+ // ;; (true, false) => %0 is too large
+ // ;; (false, true) => %0 is too small
+ // ;; }
+ // ;;
+ // ;; The (true, true) check, go to %convert if so.
+ // %inbounds = and i1 %inbound_lower, %inbound_upper
+ // br i1 %inbounds, label %convert, label %specialcase
+ //
+ // convert:
+ // %cvt = call i32 @llvm.wasm.trunc.signed.i32.f64(double %0)
+ // store i32 %cvt, i32* %result, align 4
+ // br label %done
+ //
+ // specialcase:
+ // ;; Handle the cases where the number is NaN, too large or too small
+ //
+ // ;; Either (true, false) or (false, true)
+ // %is_not_nan = or i1 %inbound_lower, %inbound_upper
+ // ;; Figure out which saturated value we are interested in if not `NaN`
+ // %saturated = select i1 %inbound_lower, i32 2147483647, i32 -2147483648
+ // ;; Figure out between saturated and NaN representations
+ // %result_nan = select i1 %is_not_nan, i32 %saturated, i32 0
+ // store i32 %result_nan, i32* %result, align 4
+ // br label %done
+ //
+ // done:
+ // %r = load i32, i32* %result, align 4
+ // ;; ...
+ let done = bx.build_sibling_block("float_cast_done");
+ let mut convert = bx.build_sibling_block("float_cast_convert");
+ let mut specialcase = bx.build_sibling_block("float_cast_specialcase");
+
+ let result = PlaceRef::alloca(bx, int_layout);
+ result.storage_live(bx);
+
+ // Use control flow to figure out whether we can execute `fptosi` in a
+ // basic block, or whether we go to a different basic block to implement
+ // the saturating logic.
+ let inbound_lower = bx.fcmp(RealPredicate::RealOGE, x, f_min);
+ let inbound_upper = bx.fcmp(RealPredicate::RealOLE, x, f_max);
+ let inbounds = bx.and(inbound_lower, inbound_upper);
+ bx.cond_br(inbounds, convert.llbb(), specialcase.llbb());
+
+ // Translation of the `convert` basic block
+ let cvt = if signed { convert.fptosi(x, int_ty) } else { convert.fptoui(x, int_ty) };
+ convert.store(cvt, result.llval, result.align);
+ convert.br(done.llbb());
+
+ // Translation of the `specialcase` basic block. Note that like above
+ // we try to be a bit clever here for unsigned conversions. In those
+ // cases the `int_min` is zero so we don't need two select instructions,
+ // just one to choose whether we need `int_max` or not. If
+ // `inbound_lower` is true then we're guaranteed to not be `NaN` and
+ // since we're greater than zero we must be saturating to `int_max`. If
+ // `inbound_lower` is false then we're either NaN or less than zero, so
+ // we saturate to zero.
+ let result_nan = if signed {
+ let is_not_nan = specialcase.or(inbound_lower, inbound_upper);
+ let saturated = specialcase.select(inbound_lower, int_max, int_min);
+ specialcase.select(is_not_nan, saturated, zero)
+ } else {
+ specialcase.select(inbound_lower, int_max, int_min)
+ };
+ specialcase.store(result_nan, result.llval, result.align);
+ specialcase.br(done.llbb());
+
+ // Translation of the `done` basic block, positioning ourselves to
+ // continue from that point as well.
+ *bx = done;
+ let ret = bx.load(result.llval, result.align);
+ result.storage_dead(bx);
+ ret
+ }
+}
--- /dev/null
- use rustc_middle::ty::layout::HasParamEnv;
+use super::abi::AbiBuilderMethods;
+use super::asm::AsmBuilderMethods;
+use super::coverageinfo::CoverageInfoBuilderMethods;
+use super::debuginfo::DebugInfoBuilderMethods;
+use super::intrinsic::IntrinsicCallMethods;
+use super::type_::ArgAbiMethods;
+use super::{HasCodegen, StaticBuilderMethods};
+
+use crate::common::{
+ AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope,
+};
+use crate::mir::operand::OperandRef;
+use crate::mir::place::PlaceRef;
+use crate::MemFlags;
+
- use rustc_target::abi::{Align, Size};
++use rustc_middle::ty::layout::{HasParamEnv, TyAndLayout};
+use rustc_middle::ty::Ty;
++use rustc_target::abi::{Abi, Align, Scalar, Size};
+use rustc_target::spec::HasTargetSpec;
+
+use std::iter::TrustedLen;
+use std::ops::Range;
+
+#[derive(Copy, Clone)]
+pub enum OverflowOp {
+ Add,
+ Sub,
+ Mul,
+}
+
+pub trait BuilderMethods<'a, 'tcx>:
+ HasCodegen<'tcx>
+ + CoverageInfoBuilderMethods<'tcx>
+ + DebugInfoBuilderMethods
+ + ArgAbiMethods<'tcx>
+ + AbiBuilderMethods<'tcx>
+ + IntrinsicCallMethods<'tcx>
+ + AsmBuilderMethods<'tcx>
+ + StaticBuilderMethods
+ + HasParamEnv<'tcx>
+ + HasTargetSpec
+{
+ fn new_block<'b>(cx: &'a Self::CodegenCx, llfn: Self::Function, name: &'b str) -> Self;
+ fn with_cx(cx: &'a Self::CodegenCx) -> Self;
+ fn build_sibling_block(&self, name: &str) -> Self;
+ fn cx(&self) -> &Self::CodegenCx;
+ fn llbb(&self) -> Self::BasicBlock;
+
+ fn position_at_end(&mut self, llbb: Self::BasicBlock);
+ fn ret_void(&mut self);
+ fn ret(&mut self, v: Self::Value);
+ fn br(&mut self, dest: Self::BasicBlock);
+ fn cond_br(
+ &mut self,
+ cond: Self::Value,
+ then_llbb: Self::BasicBlock,
+ else_llbb: Self::BasicBlock,
+ );
+ fn switch(
+ &mut self,
+ v: Self::Value,
+ else_llbb: Self::BasicBlock,
+ cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock)> + TrustedLen,
+ );
+ fn invoke(
+ &mut self,
+ llfn: Self::Value,
+ args: &[Self::Value],
+ then: Self::BasicBlock,
+ catch: Self::BasicBlock,
+ funclet: Option<&Self::Funclet>,
+ ) -> Self::Value;
+ fn unreachable(&mut self);
+
+ fn add(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn fadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn fadd_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn sub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn fsub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn fsub_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn mul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn fmul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn fmul_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn udiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn exactudiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn sdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn exactsdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn fdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn fdiv_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn urem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn srem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn frem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn frem_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn shl(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn lshr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn unchecked_ssub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn unchecked_usub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn unchecked_smul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn and(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn or(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn xor(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn neg(&mut self, v: Self::Value) -> Self::Value;
+ fn fneg(&mut self, v: Self::Value) -> Self::Value;
+ fn not(&mut self, v: Self::Value) -> Self::Value;
+
+ fn checked_binop(
+ &mut self,
+ oop: OverflowOp,
+ ty: Ty<'_>,
+ lhs: Self::Value,
+ rhs: Self::Value,
+ ) -> (Self::Value, Self::Value);
+
++ fn from_immediate(&mut self, val: Self::Value) -> Self::Value;
++ fn to_immediate(&mut self, val: Self::Value, layout: TyAndLayout<'_>) -> Self::Value {
++ if let Abi::Scalar(ref scalar) = layout.abi {
++ self.to_immediate_scalar(val, scalar)
++ } else {
++ val
++ }
++ }
++ fn to_immediate_scalar(&mut self, val: Self::Value, scalar: &Scalar) -> Self::Value;
++
+ fn alloca(&mut self, ty: Self::Type, align: Align) -> Self::Value;
+ fn dynamic_alloca(&mut self, ty: Self::Type, align: Align) -> Self::Value;
+ fn array_alloca(&mut self, ty: Self::Type, len: Self::Value, align: Align) -> Self::Value;
+
+ fn load(&mut self, ptr: Self::Value, align: Align) -> Self::Value;
+ fn volatile_load(&mut self, ptr: Self::Value) -> Self::Value;
+ fn atomic_load(&mut self, ptr: Self::Value, order: AtomicOrdering, size: Size) -> Self::Value;
+ fn load_operand(&mut self, place: PlaceRef<'tcx, Self::Value>)
+ -> OperandRef<'tcx, Self::Value>;
+
+ /// Called for Rvalue::Repeat when the elem is neither a ZST nor optimizable using memset.
+ fn write_operand_repeatedly(
+ self,
+ elem: OperandRef<'tcx, Self::Value>,
+ count: u64,
+ dest: PlaceRef<'tcx, Self::Value>,
+ ) -> Self;
+
+ fn range_metadata(&mut self, load: Self::Value, range: Range<u128>);
+ fn nonnull_metadata(&mut self, load: Self::Value);
+
+ fn store(&mut self, val: Self::Value, ptr: Self::Value, align: Align) -> Self::Value;
+ fn store_with_flags(
+ &mut self,
+ val: Self::Value,
+ ptr: Self::Value,
+ align: Align,
+ flags: MemFlags,
+ ) -> Self::Value;
+ fn atomic_store(
+ &mut self,
+ val: Self::Value,
+ ptr: Self::Value,
+ order: AtomicOrdering,
+ size: Size,
+ );
+
+ fn gep(&mut self, ptr: Self::Value, indices: &[Self::Value]) -> Self::Value;
+ fn inbounds_gep(&mut self, ptr: Self::Value, indices: &[Self::Value]) -> Self::Value;
+ fn struct_gep(&mut self, ptr: Self::Value, idx: u64) -> Self::Value;
+
+ fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option<Self::Value>;
+ fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option<Self::Value>;
+ fn fptosui_may_trap(&self, val: Self::Value, dest_ty: Self::Type) -> bool;
+ fn fptoui(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn fptosi(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn uitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn sitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn fptrunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn fpext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn ptrtoint(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn inttoptr(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn bitcast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn intcast(&mut self, val: Self::Value, dest_ty: Self::Type, is_signed: bool) -> Self::Value;
+ fn pointercast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+
+ fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+ fn fcmp(&mut self, op: RealPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
+
+ fn memcpy(
+ &mut self,
+ dst: Self::Value,
+ dst_align: Align,
+ src: Self::Value,
+ src_align: Align,
+ size: Self::Value,
+ flags: MemFlags,
+ );
+ fn memmove(
+ &mut self,
+ dst: Self::Value,
+ dst_align: Align,
+ src: Self::Value,
+ src_align: Align,
+ size: Self::Value,
+ flags: MemFlags,
+ );
+ fn memset(
+ &mut self,
+ ptr: Self::Value,
+ fill_byte: Self::Value,
+ size: Self::Value,
+ align: Align,
+ flags: MemFlags,
+ );
+
+ fn select(
+ &mut self,
+ cond: Self::Value,
+ then_val: Self::Value,
+ else_val: Self::Value,
+ ) -> Self::Value;
+
+ fn va_arg(&mut self, list: Self::Value, ty: Self::Type) -> Self::Value;
+ fn extract_element(&mut self, vec: Self::Value, idx: Self::Value) -> Self::Value;
+ fn vector_splat(&mut self, num_elts: usize, elt: Self::Value) -> Self::Value;
+ fn extract_value(&mut self, agg_val: Self::Value, idx: u64) -> Self::Value;
+ fn insert_value(&mut self, agg_val: Self::Value, elt: Self::Value, idx: u64) -> Self::Value;
+
+ fn landing_pad(
+ &mut self,
+ ty: Self::Type,
+ pers_fn: Self::Value,
+ num_clauses: usize,
+ ) -> Self::Value;
+ fn set_cleanup(&mut self, landing_pad: Self::Value);
+ fn resume(&mut self, exn: Self::Value) -> Self::Value;
+ fn cleanup_pad(&mut self, parent: Option<Self::Value>, args: &[Self::Value]) -> Self::Funclet;
+ fn cleanup_ret(
+ &mut self,
+ funclet: &Self::Funclet,
+ unwind: Option<Self::BasicBlock>,
+ ) -> Self::Value;
+ fn catch_pad(&mut self, parent: Self::Value, args: &[Self::Value]) -> Self::Funclet;
+ fn catch_switch(
+ &mut self,
+ parent: Option<Self::Value>,
+ unwind: Option<Self::BasicBlock>,
+ num_handlers: usize,
+ ) -> Self::Value;
+ fn add_handler(&mut self, catch_switch: Self::Value, handler: Self::BasicBlock);
+ fn set_personality_fn(&mut self, personality: Self::Value);
+
+ fn atomic_cmpxchg(
+ &mut self,
+ dst: Self::Value,
+ cmp: Self::Value,
+ src: Self::Value,
+ order: AtomicOrdering,
+ failure_order: AtomicOrdering,
+ weak: bool,
+ ) -> Self::Value;
+ fn atomic_rmw(
+ &mut self,
+ op: AtomicRmwBinOp,
+ dst: Self::Value,
+ src: Self::Value,
+ order: AtomicOrdering,
+ ) -> Self::Value;
+ fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope);
+ fn set_invariant_load(&mut self, load: Self::Value);
+
+ /// Called for `StorageLive`
+ fn lifetime_start(&mut self, ptr: Self::Value, size: Size);
+
+ /// Called for `StorageDead`
+ fn lifetime_end(&mut self, ptr: Self::Value, size: Size);
+
+ fn instrprof_increment(
+ &mut self,
+ fn_name: Self::Value,
+ hash: Self::Value,
+ num_counters: Self::Value,
+ index: Self::Value,
+ );
+
+ fn call(
+ &mut self,
+ llfn: Self::Value,
+ args: &[Self::Value],
+ funclet: Option<&Self::Funclet>,
+ ) -> Self::Value;
+ fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+
+ unsafe fn delete_basic_block(&mut self, bb: Self::BasicBlock);
+ fn do_not_inline(&mut self, llret: Self::Value);
+}
projects / rust.git / commitdiff