src/librustc_codegen_ssa/mir/block.rs

   1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 use rustc::middle::lang_items;
  12 use rustc::ty::{self, Ty, TypeFoldable};
  13 use rustc::ty::layout::{self, LayoutOf, HasTyCtxt};
  14 use rustc::mir;
  15 use rustc::mir::interpret::EvalErrorKind;
  16 use rustc_target::abi::call::{ArgType, FnType, PassMode};
  17 use rustc_target::spec::abi::Abi;
  18 use base;
  19 use MemFlags;
  20 use common::{self, IntPredicate};
  21 use meth;
  22 use rustc_mir::monomorphize;
  23
  24 use interfaces::*;
  25
  26 use syntax::symbol::Symbol;
  27 use syntax_pos::Pos;
  28
  29 use super::{FunctionCx, LocalRef};
  30 use super::place::PlaceRef;
  31 use super::operand::OperandRef;
  32 use super::operand::OperandValue::{Pair, Ref, Immediate};
  33
  34 impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  35     pub fn codegen_block(
  36         &mut self,
  37         bb: mir::BasicBlock,
  38     ) {
  39         let mut bx = self.build_block(bb);
  40         let data = &self.mir[bb];
  41
  42         debug!("codegen_block({:?}={:?})", bb, data);
  43
  44         for statement in &data.statements {
  45             bx = self.codegen_statement(bx, statement);
  46         }
  47
  48         self.codegen_terminator(bx, bb, data.terminator());
  49     }
  50
  51     fn codegen_terminator(
  52         &mut self,
  53         mut bx: Bx,
  54         bb: mir::BasicBlock,
  55         terminator: &mir::Terminator<'tcx>
  56     ) {
  57         debug!("codegen_terminator: {:?}", terminator);
  58
  59         // Create the cleanup bundle, if needed.
  60         let tcx = self.cx.tcx();
  61         let span = terminator.source_info.span;
  62         let funclet_bb = self.cleanup_kinds[bb].funclet_bb(bb);
  63
  64         // HACK(eddyb) force the right lifetimes, NLL can't figure them out.
  65         fn funclet_closure_factory<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
  66             funclet_bb: Option<mir::BasicBlock>
  67         ) -> impl for<'b> Fn(
  68             &'b FunctionCx<'a, 'tcx, Bx>,
  69         ) -> Option<&'b Bx::Funclet> {
  70             move |this| {
  71                 match funclet_bb {
  72                     Some(funclet_bb) => this.funclets[funclet_bb].as_ref(),
  73                     None => None,
  74                 }
  75             }
  76         }
  77         let funclet = funclet_closure_factory(funclet_bb);
  78
  79         let lltarget = |this: &mut Self, target: mir::BasicBlock| {
  80             let lltarget = this.blocks[target];
  81             let target_funclet = this.cleanup_kinds[target].funclet_bb(target);
  82             match (funclet_bb, target_funclet) {
  83                 (None, None) => (lltarget, false),
  84                 (Some(f), Some(t_f))
  85                     if f == t_f || !base::wants_msvc_seh(tcx.sess)
  86                     => (lltarget, false),
  87                 (None, Some(_)) => {
  88                     // jump *into* cleanup - need a landing pad if GNU
  89                     (this.landing_pad_to(target), false)
  90                 }
  91                 (Some(_), None) => span_bug!(span, "{:?} - jump out of cleanup?", terminator),
  92                 (Some(_), Some(_)) => {
  93                     (this.landing_pad_to(target), true)
  94                 }
  95             }
  96         };
  97
  98         let llblock = |this: &mut Self, target: mir::BasicBlock| {
  99             let (lltarget, is_cleanupret) = lltarget(this, target);
 100             if is_cleanupret {
 101                 // MSVC cross-funclet jump - need a trampoline
 102
 103                 debug!("llblock: creating cleanup trampoline for {:?}", target);
 104                 let name = &format!("{:?}_cleanup_trampoline_{:?}", bb, target);
 105                 let trampoline = this.new_block(name);
 106                 trampoline.cleanup_ret(funclet(this).unwrap(), Some(lltarget));
 107                 trampoline.llbb()
 108             } else {
 109                 lltarget
 110             }
 111         };
 112
 113         let funclet_br =
 114             |this: &mut Self, bx: &mut Bx, target: mir::BasicBlock| {
 115                 let (lltarget, is_cleanupret) = lltarget(this, target);
 116                 if is_cleanupret {
 117                     // micro-optimization: generate a `ret` rather than a jump
 118                     // to a trampoline.
 119                     bx.cleanup_ret(funclet(this).unwrap(), Some(lltarget));
 120                 } else {
 121                     bx.br(lltarget);
 122                 }
 123             };
 124
 125         let do_call = |
 126             this: &mut Self,
 127             bx: &mut Bx,
 128             fn_ty: FnType<'tcx, Ty<'tcx>>,
 129             fn_ptr: Bx::Value,
 130             llargs: &[Bx::Value],
 131             destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>,
 132             cleanup: Option<mir::BasicBlock>
 133         | {
 134             if let Some(cleanup) = cleanup {
 135                 let ret_bx = if let Some((_, target)) = destination {
 136                     this.blocks[target]
 137                 } else {
 138                     this.unreachable_block()
 139                 };
 140                 let invokeret = bx.invoke(fn_ptr,
 141                                           &llargs,
 142                                           ret_bx,
 143                                           llblock(this, cleanup),
 144                                           funclet(this));
 145                 bx.apply_attrs_callsite(&fn_ty, invokeret);
 146
 147                 if let Some((ret_dest, target)) = destination {
 148                     let ret_bx = this.build_block(target);
 149                     this.set_debug_loc(&ret_bx, terminator.source_info);
 150                     this.store_return(&ret_bx, ret_dest, &fn_ty.ret, invokeret);
 151                 }
 152             } else {
 153                 let llret = bx.call(fn_ptr, &llargs, funclet(this));
 154                 bx.apply_attrs_callsite(&fn_ty, llret);
 155                 if this.mir[bb].is_cleanup {
 156                     // Cleanup is always the cold path. Don't inline
 157                     // drop glue. Also, when there is a deeply-nested
 158                     // struct, there are "symmetry" issues that cause
 159                     // exponential inlining - see issue #41696.
 160                     bx.do_not_inline(llret);
 161                 }
 162
 163                 if let Some((ret_dest, target)) = destination {
 164                     this.store_return(bx, ret_dest, &fn_ty.ret, llret);
 165                     funclet_br(this, bx, target);
 166                 } else {
 167                     bx.unreachable();
 168                 }
 169             }
 170         };
 171
 172         self.set_debug_loc(&bx, terminator.source_info);
 173         match terminator.kind {
 174             mir::TerminatorKind::Resume => {
 175                 if let Some(funclet) = funclet(self) {
 176                     bx.cleanup_ret(funclet, None);
 177                 } else {
 178                     let slot = self.get_personality_slot(&bx);
 179                     let lp0 = bx.load_operand(slot.project_field(&bx, 0)).immediate();
 180                     let lp1 = bx.load_operand(slot.project_field(&bx, 1)).immediate();
 181                     slot.storage_dead(&bx);
 182
 183                     if !bx.cx().sess().target.target.options.custom_unwind_resume {
 184                         let mut lp = bx.cx().const_undef(self.landing_pad_type());
 185                         lp = bx.insert_value(lp, lp0, 0);
 186                         lp = bx.insert_value(lp, lp1, 1);
 187                         bx.resume(lp);
 188                     } else {
 189                         bx.call(bx.cx().eh_unwind_resume(), &[lp0], funclet(self));
 190                         bx.unreachable();
 191                     }
 192                 }
 193             }
 194
 195             mir::TerminatorKind::Abort => {
 196                 // Call core::intrinsics::abort()
 197                 let fnname = bx.cx().get_intrinsic(&("llvm.trap"));
 198                 bx.call(fnname, &[], None);
 199                 bx.unreachable();
 200             }
 201
 202             mir::TerminatorKind::Goto { target } => {
 203                 funclet_br(self, &mut bx, target);
 204             }
 205
 206             mir::TerminatorKind::SwitchInt { ref discr, switch_ty, ref values, ref targets } => {
 207                 let discr = self.codegen_operand(&bx, discr);
 208                 if targets.len() == 2 {
 209                     // If there are two targets, emit br instead of switch
 210                     let lltrue = llblock(self, targets[0]);
 211                     let llfalse = llblock(self, targets[1]);
 212                     if switch_ty == bx.tcx().types.bool {
 213                         // Don't generate trivial icmps when switching on bool
 214                         if let [0] = values[..] {
 215                             bx.cond_br(discr.immediate(), llfalse, lltrue);
 216                         } else {
 217                             assert_eq!(&values[..], &[1]);
 218                             bx.cond_br(discr.immediate(), lltrue, llfalse);
 219                         }
 220                     } else {
 221                         let switch_llty = bx.cx().immediate_backend_type(
 222                             bx.cx().layout_of(switch_ty)
 223                         );
 224                         let llval = bx.cx().const_uint_big(switch_llty, values[0]);
 225                         let cmp = bx.icmp(IntPredicate::IntEQ, discr.immediate(), llval);
 226                         bx.cond_br(cmp, lltrue, llfalse);
 227                     }
 228                 } else {
 229                     let (otherwise, targets) = targets.split_last().unwrap();
 230                     let switch = bx.switch(discr.immediate(),
 231                                            llblock(self, *otherwise),
 232                                            values.len());
 233                     let switch_llty = bx.cx().immediate_backend_type(
 234                         bx.cx().layout_of(switch_ty)
 235                     );
 236                     for (&value, target) in values.iter().zip(targets) {
 237                         let llval = bx.cx().const_uint_big(switch_llty, value);
 238                         let llbb = llblock(self, *target);
 239                         bx.add_case(switch, llval, llbb)
 240                     }
 241                 }
 242             }
 243
 244             mir::TerminatorKind::Return => {
 245                 let llval = match self.fn_ty.ret.mode {
 246                     PassMode::Ignore | PassMode::Indirect(..) => {
 247                         bx.ret_void();
 248                         return;
 249                     }
 250
 251                     PassMode::Direct(_) | PassMode::Pair(..) => {
 252                         let op = self.codegen_consume(&bx, &mir::Place::Local(mir::RETURN_PLACE));
 253                         if let Ref(llval, _, align) = op.val {
 254                             bx.load(llval, align)
 255                         } else {
 256                             op.immediate_or_packed_pair(&bx)
 257                         }
 258                     }
 259
 260                     PassMode::Cast(cast_ty) => {
 261                         let op = match self.locals[mir::RETURN_PLACE] {
 262                             LocalRef::Operand(Some(op)) => op,
 263                             LocalRef::Operand(None) => bug!("use of return before def"),
 264                             LocalRef::Place(cg_place) => {
 265                                 OperandRef {
 266                                     val: Ref(cg_place.llval, None, cg_place.align),
 267                                     layout: cg_place.layout
 268                                 }
 269                             }
 270                             LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
 271                         };
 272                         let llslot = match op.val {
 273                             Immediate(_) | Pair(..) => {
 274                                 let scratch = PlaceRef::alloca(&bx, self.fn_ty.ret.layout, "ret");
 275                                 op.val.store(&bx, scratch);
 276                                 scratch.llval
 277                             }
 278                             Ref(llval, _, align) => {
 279                                 assert_eq!(align.abi(), op.layout.align.abi(),
 280                                            "return place is unaligned!");
 281                                 llval
 282                             }
 283                         };
 284                         bx.load(
 285                             bx.pointercast(llslot, bx.cx().type_ptr_to(
 286                                 bx.cx().cast_backend_type(&cast_ty)
 287                             )),
 288                             self.fn_ty.ret.layout.align)
 289                     }
 290                 };
 291                 bx.ret(llval);
 292             }
 293
 294             mir::TerminatorKind::Unreachable => {
 295                 bx.unreachable();
 296             }
 297
 298             mir::TerminatorKind::Drop { ref location, target, unwind } => {
 299                 let ty = location.ty(self.mir, bx.tcx()).to_ty(bx.tcx());
 300                 let ty = self.monomorphize(&ty);
 301                 let drop_fn = monomorphize::resolve_drop_in_place(bx.cx().tcx(), ty);
 302
 303                 if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def {
 304                     // we don't actually need to drop anything.
 305                     funclet_br(self, &mut bx, target);
 306                     return
 307                 }
 308
 309                 let place = self.codegen_place(&bx, location);
 310                 let (args1, args2);
 311                 let mut args = if let Some(llextra) = place.llextra {
 312                     args2 = [place.llval, llextra];
 313                     &args2[..]
 314                 } else {
 315                     args1 = [place.llval];
 316                     &args1[..]
 317                 };
 318                 let (drop_fn, fn_ty) = match ty.sty {
 319                     ty::Dynamic(..) => {
 320                         let sig = drop_fn.fn_sig(tcx);
 321                         let sig = tcx.normalize_erasing_late_bound_regions(
 322                             ty::ParamEnv::reveal_all(),
 323                             &sig,
 324                         );
 325                         let fn_ty = bx.cx().new_vtable(sig, &[]);
 326                         let vtable = args[1];
 327                         args = &args[..1];
 328                         (meth::DESTRUCTOR.get_fn(&bx, vtable, &fn_ty), fn_ty)
 329                     }
 330                     _ => {
 331                         (bx.cx().get_fn(drop_fn),
 332                          bx.cx().fn_type_of_instance(&drop_fn))
 333                     }
 334                 };
 335                 do_call(self, &mut bx, fn_ty, drop_fn, args,
 336                         Some((ReturnDest::Nothing, target)),
 337                         unwind);
 338             }
 339
 340             mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, cleanup } => {
 341                 let cond = self.codegen_operand(&bx, cond).immediate();
 342                 let mut const_cond = bx.cx().const_to_opt_u128(cond, false).map(|c| c == 1);
 343
 344                 // This case can currently arise only from functions marked
 345                 // with #[rustc_inherit_overflow_checks] and inlined from
 346                 // another crate (mostly core::num generic/#[inline] fns),
 347                 // while the current crate doesn't use overflow checks.
 348                 // NOTE: Unlike binops, negation doesn't have its own
 349                 // checked operation, just a comparison with the minimum
 350                 // value, so we have to check for the assert message.
 351                 if !bx.cx().check_overflow() {
 352                     if let mir::interpret::EvalErrorKind::OverflowNeg = *msg {
 353                         const_cond = Some(expected);
 354                     }
 355                 }
 356
 357                 // Don't codegen the panic block if success if known.
 358                 if const_cond == Some(expected) {
 359                     funclet_br(self, &mut bx, target);
 360                     return;
 361                 }
 362
 363                 // Pass the condition through llvm.expect for branch hinting.
 364                 let expect = bx.cx().get_intrinsic(&"llvm.expect.i1");
 365                 let cond = bx.call(expect, &[cond, bx.cx().const_bool(expected)], None);
 366
 367                 // Create the failure block and the conditional branch to it.
 368                 let lltarget = llblock(self, target);
 369                 let panic_block = self.new_block("panic");
 370                 if expected {
 371                     bx.cond_br(cond, lltarget, panic_block.llbb());
 372                 } else {
 373                     bx.cond_br(cond, panic_block.llbb(), lltarget);
 374                 }
 375
 376                 // After this point, bx is the block for the call to panic.
 377                 bx = panic_block;
 378                 self.set_debug_loc(&bx, terminator.source_info);
 379
 380                 // Get the location information.
 381                 let loc = bx.cx().sess().source_map().lookup_char_pos(span.lo());
 382                 let filename = Symbol::intern(&loc.file.name.to_string()).as_str();
 383                 let filename = bx.cx().const_str_slice(filename);
 384                 let line = bx.cx().const_u32(loc.line as u32);
 385                 let col = bx.cx().const_u32(loc.col.to_usize() as u32 + 1);
 386                 let align = tcx.data_layout.aggregate_align
 387                     .max(tcx.data_layout.i32_align)
 388                     .max(tcx.data_layout.pointer_align);
 389
 390                 // Put together the arguments to the panic entry point.
 391                 let (lang_item, args) = match *msg {
 392                     EvalErrorKind::BoundsCheck { ref len, ref index } => {
 393                         let len = self.codegen_operand(&bx, len).immediate();
 394                         let index = self.codegen_operand(&bx, index).immediate();
 395
 396                         let file_line_col = bx.cx().const_struct(&[filename, line, col], false);
 397                         let file_line_col = bx.cx().static_addr_of(
 398                             file_line_col,
 399                             align,
 400                             Some("panic_bounds_check_loc")
 401                         );
 402                         (lang_items::PanicBoundsCheckFnLangItem,
 403                          vec![file_line_col, index, len])
 404                     }
 405                     _ => {
 406                         let str = msg.description();
 407                         let msg_str = Symbol::intern(str).as_str();
 408                         let msg_str = bx.cx().const_str_slice(msg_str);
 409                         let msg_file_line_col = bx.cx().const_struct(
 410                             &[msg_str, filename, line, col],
 411                             false
 412                         );
 413                         let msg_file_line_col = bx.cx().static_addr_of(
 414                             msg_file_line_col,
 415                             align,
 416                             Some("panic_loc")
 417                         );
 418                         (lang_items::PanicFnLangItem,
 419                          vec![msg_file_line_col])
 420                     }
 421                 };
 422
 423                 // Obtain the panic entry point.
 424                 let def_id = common::langcall(bx.tcx(), Some(span), "", lang_item);
 425                 let instance = ty::Instance::mono(bx.tcx(), def_id);
 426                 let fn_ty = bx.cx().fn_type_of_instance(&instance);
 427                 let llfn = bx.cx().get_fn(instance);
 428
 429                 // Codegen the actual panic invoke/call.
 430                 do_call(self, &mut bx, fn_ty, llfn, &args, None, cleanup);
 431             }
 432
 433             mir::TerminatorKind::DropAndReplace { .. } => {
 434                 bug!("undesugared DropAndReplace in codegen: {:?}", terminator);
 435             }
 436
 437             mir::TerminatorKind::Call {
 438                 ref func,
 439                 ref args,
 440                 ref destination,
 441                 cleanup,
 442                 from_hir_call: _
 443             } => {
 444                 // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar.
 445                 let callee = self.codegen_operand(&bx, func);
 446
 447                 let (instance, mut llfn) = match callee.layout.ty.sty {
 448                     ty::FnDef(def_id, substs) => {
 449                         (Some(ty::Instance::resolve(bx.cx().tcx(),
 450                                                     ty::ParamEnv::reveal_all(),
 451                                                     def_id,
 452                                                     substs).unwrap()),
 453                          None)
 454                     }
 455                     ty::FnPtr(_) => {
 456                         (None, Some(callee.immediate()))
 457                     }
 458                     _ => bug!("{} is not callable", callee.layout.ty)
 459                 };
 460                 let def = instance.map(|i| i.def);
 461                 let sig = callee.layout.ty.fn_sig(bx.tcx());
 462                 let sig = bx.tcx().normalize_erasing_late_bound_regions(
 463                     ty::ParamEnv::reveal_all(),
 464                     &sig,
 465                 );
 466                 let abi = sig.abi;
 467
 468                 // Handle intrinsics old codegen wants Expr's for, ourselves.
 469                 let intrinsic = match def {
 470                     Some(ty::InstanceDef::Intrinsic(def_id))
 471                         => Some(bx.tcx().item_name(def_id).as_str()),
 472                     _ => None
 473                 };
 474                 let intrinsic = intrinsic.as_ref().map(|s| &s[..]);
 475
 476                 if intrinsic == Some("transmute") {
 477                     if let Some(destination_ref) = destination.as_ref() {
 478                         let &(ref dest, target) = destination_ref;
 479                         self.codegen_transmute(&bx, &args[0], dest);
 480                         funclet_br(self, &mut bx, target);
 481                     } else {
 482                         // If we are trying to transmute to an uninhabited type,
 483                         // it is likely there is no allotted destination. In fact,
 484                         // transmuting to an uninhabited type is UB, which means
 485                         // we can do what we like. Here, we declare that transmuting
 486                         // into an uninhabited type is impossible, so anything following
 487                         // it must be unreachable.
 488                         assert_eq!(bx.cx().layout_of(sig.output()).abi, layout::Abi::Uninhabited);
 489                         bx.unreachable();
 490                     }
 491                     return;
 492                 }
 493
 494                 let extra_args = &args[sig.inputs().len()..];
 495                 let extra_args = extra_args.iter().map(|op_arg| {
 496                     let op_ty = op_arg.ty(self.mir, bx.tcx());
 497                     self.monomorphize(&op_ty)
 498                 }).collect::<Vec<_>>();
 499
 500                 let fn_ty = match def {
 501                     Some(ty::InstanceDef::Virtual(..)) => {
 502                         bx.cx().new_vtable(sig, &extra_args)
 503                     }
 504                     Some(ty::InstanceDef::DropGlue(_, None)) => {
 505                         // empty drop glue - a nop.
 506                         let &(_, target) = destination.as_ref().unwrap();
 507                         funclet_br(self, &mut bx, target);
 508                         return;
 509                     }
 510                     _ => bx.cx().new_fn_type(sig, &extra_args)
 511                 };
 512
 513                 // emit a panic instead of instantiating an uninhabited type
 514                 if (intrinsic == Some("init") || intrinsic == Some("uninit")) &&
 515                     fn_ty.ret.layout.abi.is_uninhabited()
 516                 {
 517                     let loc = bx.cx().sess().source_map().lookup_char_pos(span.lo());
 518                     let filename = Symbol::intern(&loc.file.name.to_string()).as_str();
 519                     let filename = bx.cx().const_str_slice(filename);
 520                     let line = bx.cx().const_u32(loc.line as u32);
 521                     let col = bx.cx().const_u32(loc.col.to_usize() as u32 + 1);
 522                     let align = tcx.data_layout.aggregate_align
 523                         .max(tcx.data_layout.i32_align)
 524                         .max(tcx.data_layout.pointer_align);
 525
 526                     let str = format!(
 527                         "Attempted to instantiate uninhabited type {} using mem::{}",
 528                         sig.output(),
 529                         if intrinsic == Some("init") { "zeroed" } else { "uninitialized" }
 530                     );
 531                     let msg_str = Symbol::intern(&str).as_str();
 532                     let msg_str = bx.cx().const_str_slice(msg_str);
 533                     let msg_file_line_col = bx.cx().const_struct(
 534                         &[msg_str, filename, line, col],
 535                         false,
 536                     );
 537                     let msg_file_line_col = bx.cx().static_addr_of(
 538                         msg_file_line_col,
 539                         align,
 540                         Some("panic_loc"),
 541                     );
 542
 543                     // Obtain the panic entry point.
 544                     let def_id =
 545                         common::langcall(bx.tcx(), Some(span), "", lang_items::PanicFnLangItem);
 546                     let instance = ty::Instance::mono(bx.tcx(), def_id);
 547                     let fn_ty = bx.cx().fn_type_of_instance(&instance);
 548                     let llfn = bx.cx().get_fn(instance);
 549
 550                     // Codegen the actual panic invoke/call.
 551                     do_call(
 552                         self,
 553                         &mut bx,
 554                         fn_ty,
 555                         llfn,
 556                         &[msg_file_line_col],
 557                         destination.as_ref().map(|(_, bb)| (ReturnDest::Nothing, *bb)),
 558                         cleanup,
 559                     );
 560                     return;
 561                 }
 562
 563                 // The arguments we'll be passing. Plus one to account for outptr, if used.
 564                 let arg_count = fn_ty.args.len() + fn_ty.ret.is_indirect() as usize;
 565                 let mut llargs = Vec::with_capacity(arg_count);
 566
 567                 // Prepare the return value destination
 568                 let ret_dest = if let Some((ref dest, _)) = *destination {
 569                     let is_intrinsic = intrinsic.is_some();
 570                     self.make_return_dest(&bx, dest, &fn_ty.ret, &mut llargs,
 571                                           is_intrinsic)
 572                 } else {
 573                     ReturnDest::Nothing
 574                 };
 575
 576                 if intrinsic.is_some() && intrinsic != Some("drop_in_place") {
 577                     let dest = match ret_dest {
 578                         _ if fn_ty.ret.is_indirect() => llargs[0],
 579                         ReturnDest::Nothing => {
 580                             bx.cx().const_undef(bx.cx().type_ptr_to(bx.memory_ty(&fn_ty.ret)))
 581                         }
 582                         ReturnDest::IndirectOperand(dst, _) |
 583                         ReturnDest::Store(dst) => dst.llval,
 584                         ReturnDest::DirectOperand(_) =>
 585                             bug!("Cannot use direct operand with an intrinsic call")
 586                     };
 587
 588                     let args: Vec<_> = args.iter().enumerate().map(|(i, arg)| {
 589                         // The indices passed to simd_shuffle* in the
 590                         // third argument must be constant. This is
 591                         // checked by const-qualification, which also
 592                         // promotes any complex rvalues to constants.
 593                         if i == 2 && intrinsic.unwrap().starts_with("simd_shuffle") {
 594                             match *arg {
 595                                 // The shuffle array argument is usually not an explicit constant,
 596                                 // but specified directly in the code. This means it gets promoted
 597                                 // and we can then extract the value by evaluating the promoted.
 598                                 mir::Operand::Copy(mir::Place::Promoted(box(index, ty))) |
 599                                 mir::Operand::Move(mir::Place::Promoted(box(index, ty))) => {
 600                                     let param_env = ty::ParamEnv::reveal_all();
 601                                     let cid = mir::interpret::GlobalId {
 602                                         instance: self.instance,
 603                                         promoted: Some(index),
 604                                     };
 605                                     let c = bx.tcx().const_eval(param_env.and(cid));
 606                                     let (llval, ty) = self.simd_shuffle_indices(
 607                                         &bx,
 608                                         terminator.source_info.span,
 609                                         ty,
 610                                         c,
 611                                     );
 612                                     return OperandRef {
 613                                         val: Immediate(llval),
 614                                         layout: bx.cx().layout_of(ty),
 615                                     };
 616
 617                                 },
 618                                 mir::Operand::Copy(_) |
 619                                 mir::Operand::Move(_) => {
 620                                     span_bug!(span, "shuffle indices must be constant");
 621                                 }
 622                                 mir::Operand::Constant(ref constant) => {
 623                                     let c = self.eval_mir_constant(&bx, constant);
 624                                     let (llval, ty) = self.simd_shuffle_indices(
 625                                         &bx,
 626                                         constant.span,
 627                                         constant.ty,
 628                                         c,
 629                                     );
 630                                     return OperandRef {
 631                                         val: Immediate(llval),
 632                                         layout: bx.cx().layout_of(ty)
 633                                     };
 634                                 }
 635                             }
 636                         }
 637
 638                         self.codegen_operand(&bx, arg)
 639                     }).collect();
 640
 641
 642                     let callee_ty = instance.as_ref().unwrap().ty(bx.cx().tcx());
 643                     bx.codegen_intrinsic_call(callee_ty, &fn_ty, &args, dest,
 644                                                terminator.source_info.span);
 645
 646                     if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
 647                         self.store_return(&bx, ret_dest, &fn_ty.ret, dst.llval);
 648                     }
 649
 650                     if let Some((_, target)) = *destination {
 651                         funclet_br(self, &mut bx, target);
 652                     } else {
 653                         bx.unreachable();
 654                     }
 655
 656                     return;
 657                 }
 658
 659                 // Split the rust-call tupled arguments off.
 660                 let (first_args, untuple) = if abi == Abi::RustCall && !args.is_empty() {
 661                     let (tup, args) = args.split_last().unwrap();
 662                     (args, Some(tup))
 663                 } else {
 664                     (&args[..], None)
 665                 };
 666
 667                 'make_args: for (i, arg) in first_args.iter().enumerate() {
 668                     let mut op = self.codegen_operand(&bx, arg);
 669
 670                     if let (0, Some(ty::InstanceDef::Virtual(_, idx))) = (i, def) {
 671                         if let Pair(..) = op.val {
 672                             // In the case of Rc<Self>, we need to explicitly pass a
 673                             // *mut RcBox<Self> with a Scalar (not ScalarPair) ABI. This is a hack
 674                             // that is understood elsewhere in the compiler as a method on
 675                             // `dyn Trait`.
 676                             // To get a `*mut RcBox<Self>`, we just keep unwrapping newtypes until
 677                             // we get a value of a built-in pointer type
 678                             'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
 679                                             && !op.layout.ty.is_region_ptr()
 680                             {
 681                                 'iter_fields: for i in 0..op.layout.fields.count() {
 682                                     let field = op.extract_field(&bx, i);
 683                                     if !field.layout.is_zst() {
 684                                         // we found the one non-zero-sized field that is allowed
 685                                         // now find *its* non-zero-sized field, or stop if it's a
 686                                         // pointer
 687                                         op = field;
 688                                         continue 'descend_newtypes
 689                                     }
 690                                 }
 691
 692                                 span_bug!(span, "receiver has no non-zero-sized fields {:?}", op);
 693                             }
 694
 695                             // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
 696                             // data pointer and vtable. Look up the method in the vtable, and pass
 697                             // the data pointer as the first argument
 698                             match op.val {
 699                                 Pair(data_ptr, meta) => {
 700                                     llfn = Some(meth::VirtualIndex::from_index(idx)
 701                                         .get_fn(&bx, meta, &fn_ty));
 702                                     llargs.push(data_ptr);
 703                                     continue 'make_args
 704                                 }
 705                                 other => bug!("expected a Pair, got {:?}", other)
 706                             }
 707                         } else if let Ref(data_ptr, Some(meta), _) = op.val {
 708                             // by-value dynamic dispatch
 709                             llfn = Some(meth::VirtualIndex::from_index(idx)
 710                                 .get_fn(&bx, meta, &fn_ty));
 711                             llargs.push(data_ptr);
 712                             continue;
 713                         } else {
 714                             span_bug!(span, "can't codegen a virtual call on {:?}", op);
 715                         }
 716                     }
 717
 718                     // The callee needs to own the argument memory if we pass it
 719                     // by-ref, so make a local copy of non-immediate constants.
 720                     match (arg, op.val) {
 721                         (&mir::Operand::Copy(_), Ref(_, None, _)) |
 722                         (&mir::Operand::Constant(_), Ref(_, None, _)) => {
 723                             let tmp = PlaceRef::alloca(&bx, op.layout, "const");
 724                             op.val.store(&bx, tmp);
 725                             op.val = Ref(tmp.llval, None, tmp.align);
 726                         }
 727                         _ => {}
 728                     }
 729
 730                     self.codegen_argument(&bx, op, &mut llargs, &fn_ty.args[i]);
 731                 }
 732                 if let Some(tup) = untuple {
 733                     self.codegen_arguments_untupled(&bx, tup, &mut llargs,
 734                         &fn_ty.args[first_args.len()..])
 735                 }
 736
 737                 let fn_ptr = match (llfn, instance) {
 738                     (Some(llfn), _) => llfn,
 739                     (None, Some(instance)) => bx.cx().get_fn(instance),
 740                     _ => span_bug!(span, "no llfn for call"),
 741                 };
 742
 743                 do_call(self, &mut bx, fn_ty, fn_ptr, &llargs,
 744                         destination.as_ref().map(|&(_, target)| (ret_dest, target)),
 745                         cleanup);
 746             }
 747             mir::TerminatorKind::GeneratorDrop |
 748             mir::TerminatorKind::Yield { .. } => bug!("generator ops in codegen"),
 749             mir::TerminatorKind::FalseEdges { .. } |
 750             mir::TerminatorKind::FalseUnwind { .. } => bug!("borrowck false edges in codegen"),
 751         }
 752     }
 753
 754     fn codegen_argument(
 755         &mut self,
 756         bx: &Bx,
 757         op: OperandRef<'tcx, Bx::Value>,
 758         llargs: &mut Vec<Bx::Value>,
 759         arg: &ArgType<'tcx, Ty<'tcx>>
 760     ) {
 761         // Fill padding with undef value, where applicable.
 762         if let Some(ty) = arg.pad {
 763             llargs.push(bx.cx().const_undef(bx.cx().reg_backend_type(&ty)))
 764         }
 765
 766         if arg.is_ignore() {
 767             return;
 768         }
 769
 770         if let PassMode::Pair(..) = arg.mode {
 771             match op.val {
 772                 Pair(a, b) => {
 773                     llargs.push(a);
 774                     llargs.push(b);
 775                     return;
 776                 }
 777                 _ => bug!("codegen_argument: {:?} invalid for pair argument", op)
 778             }
 779         } else if arg.is_unsized_indirect() {
 780             match op.val {
 781                 Ref(a, Some(b), _) => {
 782                     llargs.push(a);
 783                     llargs.push(b);
 784                     return;
 785                 }
 786                 _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op)
 787             }
 788         }
 789
 790         // Force by-ref if we have to load through a cast pointer.
 791         let (mut llval, align, by_ref) = match op.val {
 792             Immediate(_) | Pair(..) => {
 793                 match arg.mode {
 794                     PassMode::Indirect(..) | PassMode::Cast(_) => {
 795                         let scratch = PlaceRef::alloca(bx, arg.layout, "arg");
 796                         op.val.store(bx, scratch);
 797                         (scratch.llval, scratch.align, true)
 798                     }
 799                     _ => {
 800                         (op.immediate_or_packed_pair(bx), arg.layout.align, false)
 801                     }
 802                 }
 803             }
 804             Ref(llval, _, align) => {
 805                 if arg.is_indirect() && align.abi() < arg.layout.align.abi() {
 806                     // `foo(packed.large_field)`. We can't pass the (unaligned) field directly. I
 807                     // think that ATM (Rust 1.16) we only pass temporaries, but we shouldn't
 808                     // have scary latent bugs around.
 809
 810                     let scratch = PlaceRef::alloca(bx, arg.layout, "arg");
 811                     base::memcpy_ty(bx, scratch.llval, scratch.align, llval, align,
 812                                     op.layout, MemFlags::empty());
 813                     (scratch.llval, scratch.align, true)
 814                 } else {
 815                     (llval, align, true)
 816                 }
 817             }
 818         };
 819
 820         if by_ref && !arg.is_indirect() {
 821             // Have to load the argument, maybe while casting it.
 822             if let PassMode::Cast(ty) = arg.mode {
 823                 llval = bx.load(bx.pointercast(llval, bx.cx().type_ptr_to(
 824                     bx.cx().cast_backend_type(&ty))
 825                 ), align.min(arg.layout.align));
 826             } else {
 827                 // We can't use `PlaceRef::load` here because the argument
 828                 // may have a type we don't treat as immediate, but the ABI
 829                 // used for this call is passing it by-value. In that case,
 830                 // the load would just produce `OperandValue::Ref` instead
 831                 // of the `OperandValue::Immediate` we need for the call.
 832                 llval = bx.load(llval, align);
 833                 if let layout::Abi::Scalar(ref scalar) = arg.layout.abi {
 834                     if scalar.is_bool() {
 835                         bx.range_metadata(llval, 0..2);
 836                     }
 837                 }
 838                 // We store bools as i8 so we need to truncate to i1.
 839                 llval = base::to_immediate(bx, llval, arg.layout);
 840             }
 841         }
 842
 843         llargs.push(llval);
 844     }
 845
 846     fn codegen_arguments_untupled(
 847         &mut self,
 848         bx: &Bx,
 849         operand: &mir::Operand<'tcx>,
 850         llargs: &mut Vec<Bx::Value>,
 851         args: &[ArgType<'tcx, Ty<'tcx>>]
 852     ) {
 853         let tuple = self.codegen_operand(bx, operand);
 854
 855         // Handle both by-ref and immediate tuples.
 856         if let Ref(llval, None, align) = tuple.val {
 857             let tuple_ptr = PlaceRef::new_sized(llval, tuple.layout, align);
 858             for i in 0..tuple.layout.fields.count() {
 859                 let field_ptr = tuple_ptr.project_field(bx, i);
 860                 self.codegen_argument(bx, bx.load_operand(field_ptr), llargs, &args[i]);
 861             }
 862         } else if let Ref(_, Some(_), _) = tuple.val {
 863             bug!("closure arguments must be sized")
 864         } else {
 865             // If the tuple is immediate, the elements are as well.
 866             for i in 0..tuple.layout.fields.count() {
 867                 let op = tuple.extract_field(bx, i);
 868                 self.codegen_argument(bx, op, llargs, &args[i]);
 869             }
 870         }
 871     }
 872
 873     fn get_personality_slot(
 874         &mut self,
 875         bx: &Bx
 876     ) -> PlaceRef<'tcx, Bx::Value> {
 877         let cx = bx.cx();
 878         if let Some(slot) = self.personality_slot {
 879             slot
 880         } else {
 881             let layout = cx.layout_of(cx.tcx().intern_tup(&[
 882                 cx.tcx().mk_mut_ptr(cx.tcx().types.u8),
 883                 cx.tcx().types.i32
 884             ]));
 885             let slot = PlaceRef::alloca(bx, layout, "personalityslot");
 886             self.personality_slot = Some(slot);
 887             slot
 888         }
 889     }
 890
 891     /// Return the landingpad wrapper around the given basic block
 892     ///
 893     /// No-op in MSVC SEH scheme.
 894     fn landing_pad_to(
 895         &mut self,
 896         target_bb: mir::BasicBlock
 897     ) -> Bx::BasicBlock {
 898         if let Some(block) = self.landing_pads[target_bb] {
 899             return block;
 900         }
 901
 902         let block = self.blocks[target_bb];
 903         let landing_pad = self.landing_pad_uncached(block);
 904         self.landing_pads[target_bb] = Some(landing_pad);
 905         landing_pad
 906     }
 907
 908     fn landing_pad_uncached(
 909         &mut self,
 910         target_bb: Bx::BasicBlock
 911     ) -> Bx::BasicBlock {
 912         if base::wants_msvc_seh(self.cx.sess()) {
 913             span_bug!(self.mir.span, "landing pad was not inserted?")
 914         }
 915
 916         let mut bx = self.new_block("cleanup");
 917
 918         let llpersonality = self.cx.eh_personality();
 919         let llretty = self.landing_pad_type();
 920         let lp = bx.landing_pad(llretty, llpersonality, 1);
 921         bx.set_cleanup(lp);
 922
 923         let slot = self.get_personality_slot(&bx);
 924         slot.storage_live(&bx);
 925         Pair(bx.extract_value(lp, 0), bx.extract_value(lp, 1)).store(&bx, slot);
 926
 927         bx.br(target_bb);
 928         bx.llbb()
 929     }
 930
 931     fn landing_pad_type(&self) -> Bx::Type {
 932         let cx = self.cx;
 933         cx.type_struct(&[cx.type_i8p(), cx.type_i32()], false)
 934     }
 935
 936     fn unreachable_block(
 937         &mut self
 938     ) -> Bx::BasicBlock {
 939         self.unreachable_block.unwrap_or_else(|| {
 940             let bx = self.new_block("unreachable");
 941             bx.unreachable();
 942             self.unreachable_block = Some(bx.llbb());
 943             bx.llbb()
 944         })
 945     }
 946
 947     pub fn new_block(&self, name: &str) -> Bx {
 948         Bx::new_block(self.cx, self.llfn, name)
 949     }
 950
 951     pub fn build_block(
 952         &self,
 953         bb: mir::BasicBlock
 954     ) -> Bx {
 955         let mut bx = Bx::with_cx(self.cx);
 956         bx.position_at_end(self.blocks[bb]);
 957         bx
 958     }
 959
 960     fn make_return_dest(
 961         &mut self,
 962         bx: &Bx,
 963         dest: &mir::Place<'tcx>,
 964         fn_ret: &ArgType<'tcx, Ty<'tcx>>,
 965         llargs: &mut Vec<Bx::Value>, is_intrinsic: bool
 966     ) -> ReturnDest<'tcx, Bx::Value> {
 967         // If the return is ignored, we can just return a do-nothing ReturnDest
 968         if fn_ret.is_ignore() {
 969             return ReturnDest::Nothing;
 970         }
 971         let dest = if let mir::Place::Local(index) = *dest {
 972             match self.locals[index] {
 973                 LocalRef::Place(dest) => dest,
 974                 LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
 975                 LocalRef::Operand(None) => {
 976                     // Handle temporary places, specifically Operand ones, as
 977                     // they don't have allocas
 978                     return if fn_ret.is_indirect() {
 979                         // Odd, but possible, case, we have an operand temporary,
 980                         // but the calling convention has an indirect return.
 981                         let tmp = PlaceRef::alloca(bx, fn_ret.layout, "tmp_ret");
 982                         tmp.storage_live(bx);
 983                         llargs.push(tmp.llval);
 984                         ReturnDest::IndirectOperand(tmp, index)
 985                     } else if is_intrinsic {
 986                         // Currently, intrinsics always need a location to store
 987                         // the result. so we create a temporary alloca for the
 988                         // result
 989                         let tmp = PlaceRef::alloca(bx, fn_ret.layout, "tmp_ret");
 990                         tmp.storage_live(bx);
 991                         ReturnDest::IndirectOperand(tmp, index)
 992                     } else {
 993                         ReturnDest::DirectOperand(index)
 994                     };
 995                 }
 996                 LocalRef::Operand(Some(_)) => {
 997                     bug!("place local already assigned to");
 998                 }
 999             }
1000         } else {
1001             self.codegen_place(bx, dest)
1002         };
1003         if fn_ret.is_indirect() {
1004             if dest.align.abi() < dest.layout.align.abi() {
1005                 // Currently, MIR code generation does not create calls
1006                 // that store directly to fields of packed structs (in
1007                 // fact, the calls it creates write only to temps),
1008                 //
1009                 // If someone changes that, please update this code path
1010                 // to create a temporary.
1011                 span_bug!(self.mir.span, "can't directly store to unaligned value");
1012             }
1013             llargs.push(dest.llval);
1014             ReturnDest::Nothing
1015         } else {
1016             ReturnDest::Store(dest)
1017         }
1018     }
1019
1020     fn codegen_transmute(
1021         &mut self,
1022         bx: &Bx,
1023         src: &mir::Operand<'tcx>,
1024         dst: &mir::Place<'tcx>
1025     ) {
1026         if let mir::Place::Local(index) = *dst {
1027             match self.locals[index] {
1028                 LocalRef::Place(place) => self.codegen_transmute_into(bx, src, place),
1029                 LocalRef::UnsizedPlace(_) => bug!("transmute must not involve unsized locals"),
1030                 LocalRef::Operand(None) => {
1031                     let dst_layout = bx.cx().layout_of(self.monomorphized_place_ty(dst));
1032                     assert!(!dst_layout.ty.has_erasable_regions());
1033                     let place = PlaceRef::alloca(bx, dst_layout, "transmute_temp");
1034                     place.storage_live(bx);
1035                     self.codegen_transmute_into(bx, src, place);
1036                     let op = bx.load_operand(place);
1037                     place.storage_dead(bx);
1038                     self.locals[index] = LocalRef::Operand(Some(op));
1039                 }
1040                 LocalRef::Operand(Some(op)) => {
1041                     assert!(op.layout.is_zst(),
1042                             "assigning to initialized SSAtemp");
1043                 }
1044             }
1045         } else {
1046             let dst = self.codegen_place(bx, dst);
1047             self.codegen_transmute_into(bx, src, dst);
1048         }
1049     }
1050
1051     fn codegen_transmute_into(
1052         &mut self,
1053         bx: &Bx,
1054         src: &mir::Operand<'tcx>,
1055         dst: PlaceRef<'tcx, Bx::Value>
1056     ) {
1057         let src = self.codegen_operand(bx, src);
1058         let llty = bx.cx().backend_type(src.layout);
1059         let cast_ptr = bx.pointercast(dst.llval, bx.cx().type_ptr_to(llty));
1060         let align = src.layout.align.min(dst.layout.align);
1061         src.val.store(bx, PlaceRef::new_sized(cast_ptr, src.layout, align));
1062     }
1063
1064
1065     // Stores the return value of a function call into it's final location.
1066     fn store_return(
1067         &mut self,
1068         bx: &Bx,
1069         dest: ReturnDest<'tcx, Bx::Value>,
1070         ret_ty: &ArgType<'tcx, Ty<'tcx>>,
1071         llval: Bx::Value
1072     ) {
1073         use self::ReturnDest::*;
1074
1075         match dest {
1076             Nothing => (),
1077             Store(dst) => bx.store_arg_ty(&ret_ty, llval, dst),
1078             IndirectOperand(tmp, index) => {
1079                 let op = bx.load_operand(tmp);
1080                 tmp.storage_dead(bx);
1081                 self.locals[index] = LocalRef::Operand(Some(op));
1082             }
1083             DirectOperand(index) => {
1084                 // If there is a cast, we have to store and reload.
1085                 let op = if let PassMode::Cast(_) = ret_ty.mode {
1086                     let tmp = PlaceRef::alloca(bx, ret_ty.layout, "tmp_ret");
1087                     tmp.storage_live(bx);
1088                     bx.store_arg_ty(&ret_ty, llval, tmp);
1089                     let op = bx.load_operand(tmp);
1090                     tmp.storage_dead(bx);
1091                     op
1092                 } else {
1093                     OperandRef::from_immediate_or_packed_pair(bx, llval, ret_ty.layout)
1094                 };
1095                 self.locals[index] = LocalRef::Operand(Some(op));
1096             }
1097         }
1098     }
1099 }
1100
1101 enum ReturnDest<'tcx, V> {
1102     // Do nothing, the return value is indirect or ignored
1103     Nothing,
1104     // Store the return value to the pointer
1105     Store(PlaceRef<'tcx, V>),
1106     // Stores an indirect return value to an operand local place
1107     IndirectOperand(PlaceRef<'tcx, V>, mir::Local),
1108     // Stores a direct return value to an operand local place
1109     DirectOperand(mir::Local)
1110 }