1 // Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 use llvm::{self, ValueRef, AttributePlace};
13 use builder::{Builder, MemFlags};
14 use common::{ty_fn_sig, C_usize};
15 use context::CodegenCx;
16 use mir::place::PlaceRef;
17 use mir::operand::OperandValue;
19 use type_of::{LayoutLlvmExt, PointerKind};
21 use rustc_target::abi::{LayoutOf, Size, TyLayout};
22 use rustc::ty::{self, Ty};
23 use rustc::ty::layout;
27 pub use rustc_target::spec::abi::Abi;
28 pub use rustc::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
29 pub use rustc_target::abi::call::*;
31 macro_rules! for_each_kind {
32 ($flags: ident, $f: ident, $($kind: ident),+) => ({
33 $(if $flags.contains(ArgAttribute::$kind) { $f(llvm::Attribute::$kind) })+
37 trait ArgAttributeExt {
38 fn for_each_kind<F>(&self, f: F) where F: FnMut(llvm::Attribute);
41 impl ArgAttributeExt for ArgAttribute {
42 fn for_each_kind<F>(&self, mut f: F) where F: FnMut(llvm::Attribute) {
43 for_each_kind!(self, f,
44 ByVal, NoAlias, NoCapture, NonNull, ReadOnly, SExt, StructRet, ZExt, InReg)
48 pub trait ArgAttributesExt {
49 fn apply_llfn(&self, idx: AttributePlace, llfn: ValueRef);
50 fn apply_callsite(&self, idx: AttributePlace, callsite: ValueRef);
53 impl ArgAttributesExt for ArgAttributes {
54 fn apply_llfn(&self, idx: AttributePlace, llfn: ValueRef) {
55 let mut regular = self.regular;
57 let deref = self.pointee_size.bytes();
59 if regular.contains(ArgAttribute::NonNull) {
60 llvm::LLVMRustAddDereferenceableAttr(llfn,
64 llvm::LLVMRustAddDereferenceableOrNullAttr(llfn,
68 regular -= ArgAttribute::NonNull;
70 if let Some(align) = self.pointee_align {
71 llvm::LLVMRustAddAlignmentAttr(llfn,
75 regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn));
79 fn apply_callsite(&self, idx: AttributePlace, callsite: ValueRef) {
80 let mut regular = self.regular;
82 let deref = self.pointee_size.bytes();
84 if regular.contains(ArgAttribute::NonNull) {
85 llvm::LLVMRustAddDereferenceableCallSiteAttr(callsite,
89 llvm::LLVMRustAddDereferenceableOrNullCallSiteAttr(callsite,
93 regular -= ArgAttribute::NonNull;
95 if let Some(align) = self.pointee_align {
96 llvm::LLVMRustAddAlignmentCallSiteAttr(callsite,
100 regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite));
106 fn llvm_type(&self, cx: &CodegenCx) -> Type;
109 impl LlvmType for Reg {
110 fn llvm_type(&self, cx: &CodegenCx) -> Type {
112 RegKind::Integer => Type::ix(cx, self.size.bits()),
114 match self.size.bits() {
117 _ => bug!("unsupported float: {:?}", self)
121 Type::vector(&Type::i8(cx), self.size.bytes())
127 impl LlvmType for CastTarget {
128 fn llvm_type(&self, cx: &CodegenCx) -> Type {
129 let rest_ll_unit = self.rest.unit.llvm_type(cx);
130 let (rest_count, rem_bytes) = if self.rest.unit.size.bytes() == 0 {
133 (self.rest.total.bytes() / self.rest.unit.size.bytes(),
134 self.rest.total.bytes() % self.rest.unit.size.bytes())
137 if self.prefix.iter().all(|x| x.is_none()) {
138 // Simplify to a single unit when there is no prefix and size <= unit size
139 if self.rest.total <= self.rest.unit.size {
143 // Simplify to array when all chunks are the same size and type
145 return Type::array(&rest_ll_unit, rest_count);
149 // Create list of fields in the main structure
150 let mut args: Vec<_> =
151 self.prefix.iter().flat_map(|option_kind| option_kind.map(
152 |kind| Reg { kind: kind, size: self.prefix_chunk }.llvm_type(cx)))
153 .chain((0..rest_count).map(|_| rest_ll_unit))
156 // Append final integer
158 // Only integers can be really split further.
159 assert_eq!(self.rest.unit.kind, RegKind::Integer);
160 args.push(Type::ix(cx, rem_bytes * 8));
163 Type::struct_(cx, &args, false)
167 pub trait ArgTypeExt<'a, 'tcx> {
168 fn memory_ty(&self, cx: &CodegenCx<'a, 'tcx>) -> Type;
169 fn store(&self, bx: &Builder<'a, 'tcx>, val: ValueRef, dst: PlaceRef<'tcx>);
170 fn store_fn_arg(&self, bx: &Builder<'a, 'tcx>, idx: &mut usize, dst: PlaceRef<'tcx>);
173 impl<'a, 'tcx> ArgTypeExt<'a, 'tcx> for ArgType<'tcx, Ty<'tcx>> {
174 /// Get the LLVM type for a place of the original Rust type of
175 /// this argument/return, i.e. the result of `type_of::type_of`.
176 fn memory_ty(&self, cx: &CodegenCx<'a, 'tcx>) -> Type {
177 self.layout.llvm_type(cx)
180 /// Store a direct/indirect value described by this ArgType into a
181 /// place for the original Rust type of this argument/return.
182 /// Can be used for both storing formal arguments into Rust variables
183 /// or results of call/invoke instructions into their destinations.
184 fn store(&self, bx: &Builder<'a, 'tcx>, val: ValueRef, dst: PlaceRef<'tcx>) {
185 if self.is_ignore() {
189 if self.is_indirect() {
190 OperandValue::Ref(val, self.layout.align).store(bx, dst)
191 } else if let PassMode::Cast(cast) = self.mode {
192 // FIXME(eddyb): Figure out when the simpler Store is safe, clang
193 // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}.
194 let can_store_through_cast_ptr = false;
195 if can_store_through_cast_ptr {
196 let cast_dst = bx.pointercast(dst.llval, cast.llvm_type(cx).ptr_to());
197 bx.store(val, cast_dst, self.layout.align);
199 // The actual return type is a struct, but the ABI
200 // adaptation code has cast it into some scalar type. The
201 // code that follows is the only reliable way I have
202 // found to do a transform like i64 -> {i32,i32}.
203 // Basically we dump the data onto the stack then memcpy it.
205 // Other approaches I tried:
206 // - Casting rust ret pointer to the foreign type and using Store
207 // is (a) unsafe if size of foreign type > size of rust type and
208 // (b) runs afoul of strict aliasing rules, yielding invalid
209 // assembly under -O (specifically, the store gets removed).
210 // - Truncating foreign type to correct integral type and then
211 // bitcasting to the struct type yields invalid cast errors.
213 // We instead thus allocate some scratch space...
214 let scratch_size = cast.size(cx);
215 let scratch_align = cast.align(cx);
216 let llscratch = bx.alloca(cast.llvm_type(cx), "abi_cast", scratch_align);
217 bx.lifetime_start(llscratch, scratch_size);
219 // ...where we first store the value...
220 bx.store(val, llscratch, scratch_align);
222 // ...and then memcpy it to the intended destination.
223 base::call_memcpy(bx,
224 bx.pointercast(dst.llval, Type::i8p(cx)),
225 bx.pointercast(llscratch, Type::i8p(cx)),
226 C_usize(cx, self.layout.size.bytes()),
227 self.layout.align.min(scratch_align),
230 bx.lifetime_end(llscratch, scratch_size);
233 OperandValue::Immediate(val).store(bx, dst);
237 fn store_fn_arg(&self, bx: &Builder<'a, 'tcx>, idx: &mut usize, dst: PlaceRef<'tcx>) {
239 let val = llvm::get_param(bx.llfn(), *idx as c_uint);
244 PassMode::Ignore => {},
245 PassMode::Pair(..) => {
246 OperandValue::Pair(next(), next()).store(bx, dst);
248 PassMode::Direct(_) | PassMode::Indirect(_) | PassMode::Cast(_) => {
249 self.store(bx, next(), dst);
255 pub trait FnTypeExt<'a, 'tcx> {
256 fn of_instance(cx: &CodegenCx<'a, 'tcx>, instance: &ty::Instance<'tcx>)
258 fn new(cx: &CodegenCx<'a, 'tcx>,
259 sig: ty::FnSig<'tcx>,
260 extra_args: &[Ty<'tcx>]) -> Self;
261 fn new_vtable(cx: &CodegenCx<'a, 'tcx>,
262 sig: ty::FnSig<'tcx>,
263 extra_args: &[Ty<'tcx>]) -> Self;
265 cx: &CodegenCx<'a, 'tcx>,
266 sig: ty::FnSig<'tcx>,
267 extra_args: &[Ty<'tcx>],
268 mk_arg_type: impl Fn(Ty<'tcx>, Option<usize>) -> ArgType<'tcx, Ty<'tcx>>,
270 fn adjust_for_abi(&mut self,
271 cx: &CodegenCx<'a, 'tcx>,
273 fn llvm_type(&self, cx: &CodegenCx<'a, 'tcx>) -> Type;
274 fn llvm_cconv(&self) -> llvm::CallConv;
275 fn apply_attrs_llfn(&self, llfn: ValueRef);
276 fn apply_attrs_callsite(&self, bx: &Builder<'a, 'tcx>, callsite: ValueRef);
279 impl<'a, 'tcx> FnTypeExt<'a, 'tcx> for FnType<'tcx, Ty<'tcx>> {
280 fn of_instance(cx: &CodegenCx<'a, 'tcx>, instance: &ty::Instance<'tcx>)
282 let fn_ty = instance.ty(cx.tcx);
283 let sig = ty_fn_sig(cx, fn_ty);
284 let sig = cx.tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig);
285 FnType::new(cx, sig, &[])
288 fn new(cx: &CodegenCx<'a, 'tcx>,
289 sig: ty::FnSig<'tcx>,
290 extra_args: &[Ty<'tcx>]) -> Self {
291 FnType::new_internal(cx, sig, extra_args, |ty, _| {
292 ArgType::new(cx.layout_of(ty))
296 fn new_vtable(cx: &CodegenCx<'a, 'tcx>,
297 sig: ty::FnSig<'tcx>,
298 extra_args: &[Ty<'tcx>]) -> Self {
299 FnType::new_internal(cx, sig, extra_args, |ty, arg_idx| {
300 let mut layout = cx.layout_of(ty);
301 // Don't pass the vtable, it's not an argument of the virtual fn.
302 // Instead, pass just the (thin pointer) first field of `*dyn Trait`.
303 if arg_idx == Some(0) {
304 // FIXME(eddyb) `layout.field(cx, 0)` is not enough because e.g.
305 // `Box<dyn Trait>` has a few newtype wrappers around the raw
306 // pointer, so we'd have to "dig down" to find `*dyn Trait`.
307 let pointee = layout.ty.builtin_deref(true)
309 bug!("FnType::new_vtable: non-pointer self {:?}", layout)
311 let fat_ptr_ty = cx.tcx.mk_mut_ptr(pointee);
312 layout = cx.layout_of(fat_ptr_ty).field(cx, 0);
319 cx: &CodegenCx<'a, 'tcx>,
320 sig: ty::FnSig<'tcx>,
321 extra_args: &[Ty<'tcx>],
322 mk_arg_type: impl Fn(Ty<'tcx>, Option<usize>) -> ArgType<'tcx, Ty<'tcx>>,
324 debug!("FnType::new_internal({:?}, {:?})", sig, extra_args);
327 let conv = match cx.sess().target.target.adjust_abi(sig.abi) {
328 RustIntrinsic | PlatformIntrinsic |
329 Rust | RustCall => Conv::C,
331 // It's the ABI's job to select this, not us.
332 System => bug!("system abi should be selected elsewhere"),
334 Stdcall => Conv::X86Stdcall,
335 Fastcall => Conv::X86Fastcall,
336 Vectorcall => Conv::X86VectorCall,
337 Thiscall => Conv::X86ThisCall,
339 Unadjusted => Conv::C,
340 Win64 => Conv::X86_64Win64,
341 SysV64 => Conv::X86_64SysV,
342 Aapcs => Conv::ArmAapcs,
343 PtxKernel => Conv::PtxKernel,
344 Msp430Interrupt => Conv::Msp430Intr,
345 X86Interrupt => Conv::X86Intr,
346 AmdGpuKernel => Conv::AmdGpuKernel,
348 // These API constants ought to be more specific...
352 let mut inputs = sig.inputs();
353 let extra_args = if sig.abi == RustCall {
354 assert!(!sig.variadic && extra_args.is_empty());
356 match sig.inputs().last().unwrap().sty {
357 ty::TyTuple(ref tupled_arguments) => {
358 inputs = &sig.inputs()[0..sig.inputs().len() - 1];
362 bug!("argument to function with \"rust-call\" ABI \
367 assert!(sig.variadic || extra_args.is_empty());
371 let target = &cx.sess().target.target;
372 let win_x64_gnu = target.target_os == "windows"
373 && target.arch == "x86_64"
374 && target.target_env == "gnu";
375 let linux_s390x = target.target_os == "linux"
376 && target.arch == "s390x"
377 && target.target_env == "gnu";
378 let rust_abi = match sig.abi {
379 RustIntrinsic | PlatformIntrinsic | Rust | RustCall => true,
383 // Handle safe Rust thin and fat pointers.
384 let adjust_for_rust_scalar = |attrs: &mut ArgAttributes,
385 scalar: &layout::Scalar,
386 layout: TyLayout<'tcx, Ty<'tcx>>,
389 // Booleans are always an i1 that needs to be zero-extended.
390 if scalar.is_bool() {
391 attrs.set(ArgAttribute::ZExt);
395 // Only pointer types handled below.
396 if scalar.value != layout::Pointer {
400 if scalar.valid_range.start() < scalar.valid_range.end() {
401 if *scalar.valid_range.start() > 0 {
402 attrs.set(ArgAttribute::NonNull);
406 if let Some(pointee) = layout.pointee_info_at(cx, offset) {
407 if let Some(kind) = pointee.safe {
408 attrs.pointee_size = pointee.size;
409 attrs.pointee_align = Some(pointee.align);
411 // HACK(eddyb) LLVM inserts `llvm.assume` calls when inlining functions
412 // with align attributes, and those calls later block optimizations.
413 if !is_return && !cx.tcx.sess.opts.debugging_opts.arg_align_attributes {
414 attrs.pointee_align = None;
417 // `Box` pointer parameters never alias because ownership is transferred
418 // `&mut` pointer parameters never alias other parameters,
419 // or mutable global data
421 // `&T` where `T` contains no `UnsafeCell<U>` is immutable,
422 // and can be marked as both `readonly` and `noalias`, as
423 // LLVM's definition of `noalias` is based solely on memory
424 // dependencies rather than pointer equality
425 let no_alias = match kind {
426 PointerKind::Shared => false,
427 PointerKind::UniqueOwned => true,
428 PointerKind::Frozen |
429 PointerKind::UniqueBorrowed => !is_return
432 attrs.set(ArgAttribute::NoAlias);
435 if kind == PointerKind::Frozen && !is_return {
436 attrs.set(ArgAttribute::ReadOnly);
442 let arg_of = |ty: Ty<'tcx>, arg_idx: Option<usize>| {
443 let is_return = arg_idx.is_none();
444 let mut arg = mk_arg_type(ty, arg_idx);
445 if arg.layout.is_zst() {
446 // For some forsaken reason, x86_64-pc-windows-gnu
447 // doesn't ignore zero-sized struct arguments.
448 // The same is true for s390x-unknown-linux-gnu.
449 if is_return || rust_abi || (!win_x64_gnu && !linux_s390x) {
450 arg.mode = PassMode::Ignore;
454 // FIXME(eddyb) other ABIs don't have logic for scalar pairs.
455 if !is_return && rust_abi {
456 if let layout::Abi::ScalarPair(ref a, ref b) = arg.layout.abi {
457 let mut a_attrs = ArgAttributes::new();
458 let mut b_attrs = ArgAttributes::new();
459 adjust_for_rust_scalar(&mut a_attrs,
464 adjust_for_rust_scalar(&mut b_attrs,
467 a.value.size(cx).abi_align(b.value.align(cx)),
469 arg.mode = PassMode::Pair(a_attrs, b_attrs);
474 if let layout::Abi::Scalar(ref scalar) = arg.layout.abi {
475 if let PassMode::Direct(ref mut attrs) = arg.mode {
476 adjust_for_rust_scalar(attrs,
487 let mut fn_ty = FnType {
488 ret: arg_of(sig.output(), None),
489 args: inputs.iter().chain(extra_args).enumerate().map(|(i, ty)| {
492 variadic: sig.variadic,
495 fn_ty.adjust_for_abi(cx, sig.abi);
499 fn adjust_for_abi(&mut self,
500 cx: &CodegenCx<'a, 'tcx>,
502 if abi == Abi::Unadjusted { return }
504 if abi == Abi::Rust || abi == Abi::RustCall ||
505 abi == Abi::RustIntrinsic || abi == Abi::PlatformIntrinsic {
506 let fixup = |arg: &mut ArgType<'tcx, Ty<'tcx>>| {
507 if arg.is_ignore() { return; }
509 match arg.layout.abi {
510 layout::Abi::Aggregate { .. } => {}
512 // This is a fun case! The gist of what this is doing is
513 // that we want callers and callees to always agree on the
514 // ABI of how they pass SIMD arguments. If we were to *not*
515 // make these arguments indirect then they'd be immediates
516 // in LLVM, which means that they'd used whatever the
517 // appropriate ABI is for the callee and the caller. That
518 // means, for example, if the caller doesn't have AVX
519 // enabled but the callee does, then passing an AVX argument
520 // across this boundary would cause corrupt data to show up.
522 // This problem is fixed by unconditionally passing SIMD
523 // arguments through memory between callers and callees
524 // which should get them all to agree on ABI regardless of
525 // target feature sets. Some more information about this
526 // issue can be found in #44367.
528 // Note that the platform intrinsic ABI is exempt here as
529 // that's how we connect up to LLVM and it's unstable
530 // anyway, we control all calls to it in libstd.
531 layout::Abi::Vector { .. } if abi != Abi::PlatformIntrinsic => {
539 let size = arg.layout.size;
540 if size > layout::Pointer.size(cx) {
543 // We want to pass small aggregates as immediates, but using
544 // a LLVM aggregate type for this leads to bad optimizations,
545 // so we pick an appropriately sized integer type instead.
547 kind: RegKind::Integer,
552 fixup(&mut self.ret);
553 for arg in &mut self.args {
556 if let PassMode::Indirect(ref mut attrs) = self.ret.mode {
557 attrs.set(ArgAttribute::StructRet);
562 if let Err(msg) = self.adjust_for_cabi(cx, abi) {
563 cx.sess().fatal(&msg);
567 fn llvm_type(&self, cx: &CodegenCx<'a, 'tcx>) -> Type {
568 let args_capacity: usize = self.args.iter().map(|arg|
569 if arg.pad.is_some() { 1 } else { 0 } +
570 if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 }
572 let mut llargument_tys = Vec::with_capacity(
573 if let PassMode::Indirect(_) = self.ret.mode { 1 } else { 0 } + args_capacity
576 let llreturn_ty = match self.ret.mode {
577 PassMode::Ignore => Type::void(cx),
578 PassMode::Direct(_) | PassMode::Pair(..) => {
579 self.ret.layout.immediate_llvm_type(cx)
581 PassMode::Cast(cast) => cast.llvm_type(cx),
582 PassMode::Indirect(_) => {
583 llargument_tys.push(self.ret.memory_ty(cx).ptr_to());
588 for arg in &self.args {
590 if let Some(ty) = arg.pad {
591 llargument_tys.push(ty.llvm_type(cx));
594 let llarg_ty = match arg.mode {
595 PassMode::Ignore => continue,
596 PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx),
597 PassMode::Pair(..) => {
598 llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true));
599 llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true));
602 PassMode::Cast(cast) => cast.llvm_type(cx),
603 PassMode::Indirect(_) => arg.memory_ty(cx).ptr_to(),
605 llargument_tys.push(llarg_ty);
609 Type::variadic_func(&llargument_tys, &llreturn_ty)
611 Type::func(&llargument_tys, &llreturn_ty)
615 fn llvm_cconv(&self) -> llvm::CallConv {
617 Conv::C => llvm::CCallConv,
618 Conv::AmdGpuKernel => llvm::AmdGpuKernel,
619 Conv::ArmAapcs => llvm::ArmAapcsCallConv,
620 Conv::Msp430Intr => llvm::Msp430Intr,
621 Conv::PtxKernel => llvm::PtxKernel,
622 Conv::X86Fastcall => llvm::X86FastcallCallConv,
623 Conv::X86Intr => llvm::X86_Intr,
624 Conv::X86Stdcall => llvm::X86StdcallCallConv,
625 Conv::X86ThisCall => llvm::X86_ThisCall,
626 Conv::X86VectorCall => llvm::X86_VectorCall,
627 Conv::X86_64SysV => llvm::X86_64_SysV,
628 Conv::X86_64Win64 => llvm::X86_64_Win64,
632 fn apply_attrs_llfn(&self, llfn: ValueRef) {
634 let mut apply = |attrs: &ArgAttributes| {
635 attrs.apply_llfn(llvm::AttributePlace::Argument(i), llfn);
638 match self.ret.mode {
639 PassMode::Direct(ref attrs) => {
640 attrs.apply_llfn(llvm::AttributePlace::ReturnValue, llfn);
642 PassMode::Indirect(ref attrs) => apply(attrs),
645 for arg in &self.args {
646 if arg.pad.is_some() {
647 apply(&ArgAttributes::new());
650 PassMode::Ignore => {}
651 PassMode::Direct(ref attrs) |
652 PassMode::Indirect(ref attrs) => apply(attrs),
653 PassMode::Pair(ref a, ref b) => {
657 PassMode::Cast(_) => apply(&ArgAttributes::new()),
662 fn apply_attrs_callsite(&self, bx: &Builder<'a, 'tcx>, callsite: ValueRef) {
664 let mut apply = |attrs: &ArgAttributes| {
665 attrs.apply_callsite(llvm::AttributePlace::Argument(i), callsite);
668 match self.ret.mode {
669 PassMode::Direct(ref attrs) => {
670 attrs.apply_callsite(llvm::AttributePlace::ReturnValue, callsite);
672 PassMode::Indirect(ref attrs) => apply(attrs),
675 if let layout::Abi::Scalar(ref scalar) = self.ret.layout.abi {
676 // If the value is a boolean, the range is 0..2 and that ultimately
677 // become 0..0 when the type becomes i1, which would be rejected
678 // by the LLVM verifier.
680 layout::Int(..) if !scalar.is_bool() => {
681 let range = scalar.valid_range_exclusive(bx.cx);
682 if range.start != range.end {
683 bx.range_metadata(callsite, range);
689 for arg in &self.args {
690 if arg.pad.is_some() {
691 apply(&ArgAttributes::new());
694 PassMode::Ignore => {}
695 PassMode::Direct(ref attrs) |
696 PassMode::Indirect(ref attrs) => apply(attrs),
697 PassMode::Pair(ref a, ref b) => {
701 PassMode::Cast(_) => apply(&ArgAttributes::new()),
705 let cconv = self.llvm_cconv();
706 if cconv != llvm::CCallConv {
707 llvm::SetInstructionCallConv(callsite, cconv);