src/librustc_trans/mir/block.rs

   1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 use llvm::{self, ValueRef, BasicBlockRef};
  12 use rustc::middle::lang_items;
  13 use rustc::middle::const_val::{ConstEvalErr, ConstInt, ErrKind};
  14 use rustc::ty::{self, TypeFoldable};
  15 use rustc::ty::layout::{self, LayoutTyper};
  16 use rustc::mir;
  17 use abi::{Abi, FnType, ArgType};
  18 use adt;
  19 use base::{self, Lifetime};
  20 use callee;
  21 use builder::Builder;
  22 use common::{self, C_bool, C_str_slice, C_struct, C_u32, C_undef};
  23 use consts;
  24 use machine::llalign_of_min;
  25 use meth;
  26 use monomorphize;
  27 use type_of;
  28 use type_::Type;
  29
  30 use syntax::symbol::Symbol;
  31
  32 use std::cmp;
  33
  34 use super::{MirContext, LocalRef};
  35 use super::constant::Const;
  36 use super::lvalue::{Alignment, LvalueRef};
  37 use super::operand::OperandRef;
  38 use super::operand::OperandValue::{Pair, Ref, Immediate};
  39
  40 impl<'a, 'tcx> MirContext<'a, 'tcx> {
  41     pub fn trans_block(&mut self, bb: mir::BasicBlock) {
  42         let mut bcx = self.get_builder(bb);
  43         let data = &self.mir[bb];
  44
  45         debug!("trans_block({:?}={:?})", bb, data);
  46
  47         for statement in &data.statements {
  48             bcx = self.trans_statement(bcx, statement);
  49         }
  50
  51         self.trans_terminator(bcx, bb, data.terminator());
  52     }
  53
  54     fn trans_terminator(&mut self,
  55                         mut bcx: Builder<'a, 'tcx>,
  56                         bb: mir::BasicBlock,
  57                         terminator: &mir::Terminator<'tcx>)
  58     {
  59         debug!("trans_terminator: {:?}", terminator);
  60
  61         // Create the cleanup bundle, if needed.
  62         let tcx = bcx.tcx();
  63         let span = terminator.source_info.span;
  64         let funclet_bb = self.cleanup_kinds[bb].funclet_bb(bb);
  65         let funclet = funclet_bb.and_then(|funclet_bb| self.funclets[funclet_bb].as_ref());
  66
  67         let cleanup_pad = funclet.map(|lp| lp.cleanuppad());
  68         let cleanup_bundle = funclet.map(|l| l.bundle());
  69
  70         let lltarget = |this: &mut Self, target: mir::BasicBlock| {
  71             let lltarget = this.blocks[target];
  72             let target_funclet = this.cleanup_kinds[target].funclet_bb(target);
  73             match (funclet_bb, target_funclet) {
  74                 (None, None) => (lltarget, false),
  75                 (Some(f), Some(t_f))
  76                     if f == t_f || !base::wants_msvc_seh(tcx.sess)
  77                     => (lltarget, false),
  78                 (None, Some(_)) => {
  79                     // jump *into* cleanup - need a landing pad if GNU
  80                     (this.landing_pad_to(target), false)
  81                 }
  82                 (Some(_), None) => span_bug!(span, "{:?} - jump out of cleanup?", terminator),
  83                 (Some(_), Some(_)) => {
  84                     (this.landing_pad_to(target), true)
  85                 }
  86             }
  87         };
  88
  89         let llblock = |this: &mut Self, target: mir::BasicBlock| {
  90             let (lltarget, is_cleanupret) = lltarget(this, target);
  91             if is_cleanupret {
  92                 // MSVC cross-funclet jump - need a trampoline
  93
  94                 debug!("llblock: creating cleanup trampoline for {:?}", target);
  95                 let name = &format!("{:?}_cleanup_trampoline_{:?}", bb, target);
  96                 let trampoline = this.new_block(name);
  97                 trampoline.cleanup_ret(cleanup_pad.unwrap(), Some(lltarget));
  98                 trampoline.llbb()
  99             } else {
 100                 lltarget
 101             }
 102         };
 103
 104         let funclet_br = |this: &mut Self, bcx: Builder, target: mir::BasicBlock| {
 105             let (lltarget, is_cleanupret) = lltarget(this, target);
 106             if is_cleanupret {
 107                 // micro-optimization: generate a `ret` rather than a jump
 108                 // to a trampoline.
 109                 bcx.cleanup_ret(cleanup_pad.unwrap(), Some(lltarget));
 110             } else {
 111                 bcx.br(lltarget);
 112             }
 113         };
 114
 115         let do_call = |
 116             this: &mut Self,
 117             bcx: Builder<'a, 'tcx>,
 118             fn_ty: FnType<'tcx>,
 119             fn_ptr: ValueRef,
 120             llargs: &[ValueRef],
 121             destination: Option<(ReturnDest, ty::Ty<'tcx>, mir::BasicBlock)>,
 122             cleanup: Option<mir::BasicBlock>
 123         | {
 124             if let Some(cleanup) = cleanup {
 125                 let ret_bcx = if let Some((_, _, target)) = destination {
 126                     this.blocks[target]
 127                 } else {
 128                     this.unreachable_block()
 129                 };
 130                 let invokeret = bcx.invoke(fn_ptr,
 131                                            &llargs,
 132                                            ret_bcx,
 133                                            llblock(this, cleanup),
 134                                            cleanup_bundle);
 135                 fn_ty.apply_attrs_callsite(invokeret);
 136
 137                 if let Some((ret_dest, ret_ty, target)) = destination {
 138                     let ret_bcx = this.get_builder(target);
 139                     this.set_debug_loc(&ret_bcx, terminator.source_info);
 140                     let op = OperandRef {
 141                         val: Immediate(invokeret),
 142                         ty: ret_ty,
 143                     };
 144                     this.store_return(&ret_bcx, ret_dest, &fn_ty.ret, op);
 145                 }
 146             } else {
 147                 let llret = bcx.call(fn_ptr, &llargs, cleanup_bundle);
 148                 fn_ty.apply_attrs_callsite(llret);
 149                 if this.mir[bb].is_cleanup {
 150                     // Cleanup is always the cold path. Don't inline
 151                     // drop glue. Also, when there is a deeply-nested
 152                     // struct, there are "symmetry" issues that cause
 153                     // exponential inlining - see issue #41696.
 154                     llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret);
 155                 }
 156
 157                 if let Some((ret_dest, ret_ty, target)) = destination {
 158                     let op = OperandRef {
 159                         val: Immediate(llret),
 160                         ty: ret_ty,
 161                     };
 162                     this.store_return(&bcx, ret_dest, &fn_ty.ret, op);
 163                     funclet_br(this, bcx, target);
 164                 } else {
 165                     bcx.unreachable();
 166                 }
 167             }
 168         };
 169
 170         self.set_debug_loc(&bcx, terminator.source_info);
 171         match terminator.kind {
 172             mir::TerminatorKind::Resume => {
 173                 if let Some(cleanup_pad) = cleanup_pad {
 174                     bcx.cleanup_ret(cleanup_pad, None);
 175                 } else {
 176                     let ps = self.get_personality_slot(&bcx);
 177                     let lp = bcx.load(ps, None);
 178                     Lifetime::End.call(&bcx, ps);
 179                     if !bcx.sess().target.target.options.custom_unwind_resume {
 180                         bcx.resume(lp);
 181                     } else {
 182                         let exc_ptr = bcx.extract_value(lp, 0);
 183                         bcx.call(bcx.ccx.eh_unwind_resume(), &[exc_ptr], cleanup_bundle);
 184                         bcx.unreachable();
 185                     }
 186                 }
 187             }
 188
 189             mir::TerminatorKind::Goto { target } => {
 190                 funclet_br(self, bcx, target);
 191             }
 192
 193             mir::TerminatorKind::SwitchInt { ref discr, switch_ty, ref values, ref targets } => {
 194                 let discr = self.trans_operand(&bcx, discr);
 195                 if switch_ty == bcx.tcx().types.bool {
 196                     let lltrue = llblock(self, targets[0]);
 197                     let llfalse = llblock(self, targets[1]);
 198                     if let [ConstInt::U8(0)] = values[..] {
 199                         bcx.cond_br(discr.immediate(), llfalse, lltrue);
 200                     } else {
 201                         bcx.cond_br(discr.immediate(), lltrue, llfalse);
 202                     }
 203                 } else {
 204                     let (otherwise, targets) = targets.split_last().unwrap();
 205                     let switch = bcx.switch(discr.immediate(),
 206                                             llblock(self, *otherwise), values.len());
 207                     for (value, target) in values.iter().zip(targets) {
 208                         let val = Const::from_constint(bcx.ccx, value);
 209                         let llbb = llblock(self, *target);
 210                         bcx.add_case(switch, val.llval, llbb)
 211                     }
 212                 }
 213             }
 214
 215             mir::TerminatorKind::Return => {
 216                 let ret = self.fn_ty.ret;
 217                 if ret.is_ignore() || ret.is_indirect() {
 218                     bcx.ret_void();
 219                     return;
 220                 }
 221
 222                 let llval = if let Some(cast_ty) = ret.cast {
 223                     let op = match self.locals[mir::RETURN_POINTER] {
 224                         LocalRef::Operand(Some(op)) => op,
 225                         LocalRef::Operand(None) => bug!("use of return before def"),
 226                         LocalRef::Lvalue(tr_lvalue) => {
 227                             OperandRef {
 228                                 val: Ref(tr_lvalue.llval, tr_lvalue.alignment),
 229                                 ty: tr_lvalue.ty.to_ty(bcx.tcx())
 230                             }
 231                         }
 232                     };
 233                     let llslot = match op.val {
 234                         Immediate(_) | Pair(..) => {
 235                             let llscratch = bcx.alloca(ret.memory_ty(bcx.ccx), "ret", None);
 236                             self.store_operand(&bcx, llscratch, None, op);
 237                             llscratch
 238                         }
 239                         Ref(llval, align) => {
 240                             assert_eq!(align, Alignment::AbiAligned,
 241                                        "return pointer is unaligned!");
 242                             llval
 243                         }
 244                     };
 245                     let load = bcx.load(
 246                         bcx.pointercast(llslot, cast_ty.ptr_to()),
 247                         Some(ret.layout.align(bcx.ccx).abi() as u32));
 248                     load
 249                 } else {
 250                     let op = self.trans_consume(&bcx, &mir::Lvalue::Local(mir::RETURN_POINTER));
 251                     if let Ref(llval, align) = op.val {
 252                         base::load_ty(&bcx, llval, align, op.ty)
 253                     } else {
 254                         op.pack_if_pair(&bcx).immediate()
 255                     }
 256                 };
 257                 bcx.ret(llval);
 258             }
 259
 260             mir::TerminatorKind::Unreachable => {
 261                 bcx.unreachable();
 262             }
 263
 264             mir::TerminatorKind::Drop { ref location, target, unwind } => {
 265                 let ty = location.ty(&self.mir, bcx.tcx()).to_ty(bcx.tcx());
 266                 let ty = self.monomorphize(&ty);
 267                 let drop_fn = monomorphize::resolve_drop_in_place(bcx.ccx.shared(), ty);
 268
 269                 if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def {
 270                     // we don't actually need to drop anything.
 271                     funclet_br(self, bcx, target);
 272                     return
 273                 }
 274
 275                 let lvalue = self.trans_lvalue(&bcx, location);
 276                 let fn_ty = FnType::of_instance(bcx.ccx, &drop_fn);
 277                 let (drop_fn, need_extra) = match ty.sty {
 278                     ty::TyDynamic(..) => (meth::DESTRUCTOR.get_fn(&bcx, lvalue.llextra),
 279                                           false),
 280                     _ => (callee::get_fn(bcx.ccx, drop_fn), lvalue.has_extra())
 281                 };
 282                 let args = &[lvalue.llval, lvalue.llextra][..1 + need_extra as usize];
 283                 do_call(self, bcx, fn_ty, drop_fn, args,
 284                         Some((ReturnDest::Nothing, tcx.mk_nil(), target)),
 285                         unwind);
 286             }
 287
 288             mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, cleanup } => {
 289                 let cond = self.trans_operand(&bcx, cond).immediate();
 290                 let mut const_cond = common::const_to_opt_u128(cond, false).map(|c| c == 1);
 291
 292                 // This case can currently arise only from functions marked
 293                 // with #[rustc_inherit_overflow_checks] and inlined from
 294                 // another crate (mostly core::num generic/#[inline] fns),
 295                 // while the current crate doesn't use overflow checks.
 296                 // NOTE: Unlike binops, negation doesn't have its own
 297                 // checked operation, just a comparison with the minimum
 298                 // value, so we have to check for the assert message.
 299                 if !bcx.ccx.check_overflow() {
 300                     use rustc_const_math::ConstMathErr::Overflow;
 301                     use rustc_const_math::Op::Neg;
 302
 303                     if let mir::AssertMessage::Math(Overflow(Neg)) = *msg {
 304                         const_cond = Some(expected);
 305                     }
 306                 }
 307
 308                 // Don't translate the panic block if success if known.
 309                 if const_cond == Some(expected) {
 310                     funclet_br(self, bcx, target);
 311                     return;
 312                 }
 313
 314                 // Pass the condition through llvm.expect for branch hinting.
 315                 let expect = bcx.ccx.get_intrinsic(&"llvm.expect.i1");
 316                 let cond = bcx.call(expect, &[cond, C_bool(bcx.ccx, expected)], None);
 317
 318                 // Create the failure block and the conditional branch to it.
 319                 let lltarget = llblock(self, target);
 320                 let panic_block = self.new_block("panic");
 321                 if expected {
 322                     bcx.cond_br(cond, lltarget, panic_block.llbb());
 323                 } else {
 324                     bcx.cond_br(cond, panic_block.llbb(), lltarget);
 325                 }
 326
 327                 // After this point, bcx is the block for the call to panic.
 328                 bcx = panic_block;
 329                 self.set_debug_loc(&bcx, terminator.source_info);
 330
 331                 // Get the location information.
 332                 let loc = bcx.sess().codemap().lookup_char_pos(span.lo);
 333                 let filename = Symbol::intern(&loc.file.name).as_str();
 334                 let filename = C_str_slice(bcx.ccx, filename);
 335                 let line = C_u32(bcx.ccx, loc.line as u32);
 336
 337                 // Put together the arguments to the panic entry point.
 338                 let (lang_item, args, const_err) = match *msg {
 339                     mir::AssertMessage::BoundsCheck { ref len, ref index } => {
 340                         let len = self.trans_operand(&mut bcx, len).immediate();
 341                         let index = self.trans_operand(&mut bcx, index).immediate();
 342
 343                         let const_err = common::const_to_opt_u128(len, false)
 344                             .and_then(|len| common::const_to_opt_u128(index, false)
 345                                 .map(|index| ErrKind::IndexOutOfBounds {
 346                                     len: len as u64,
 347                                     index: index as u64
 348                                 }));
 349
 350                         let file_line = C_struct(bcx.ccx, &[filename, line], false);
 351                         let align = llalign_of_min(bcx.ccx, common::val_ty(file_line));
 352                         let file_line = consts::addr_of(bcx.ccx,
 353                                                         file_line,
 354                                                         align,
 355                                                         "panic_bounds_check_loc");
 356                         (lang_items::PanicBoundsCheckFnLangItem,
 357                          vec![file_line, index, len],
 358                          const_err)
 359                     }
 360                     mir::AssertMessage::Math(ref err) => {
 361                         let msg_str = Symbol::intern(err.description()).as_str();
 362                         let msg_str = C_str_slice(bcx.ccx, msg_str);
 363                         let msg_file_line = C_struct(bcx.ccx,
 364                                                      &[msg_str, filename, line],
 365                                                      false);
 366                         let align = llalign_of_min(bcx.ccx, common::val_ty(msg_file_line));
 367                         let msg_file_line = consts::addr_of(bcx.ccx,
 368                                                             msg_file_line,
 369                                                             align,
 370                                                             "panic_loc");
 371                         (lang_items::PanicFnLangItem,
 372                          vec![msg_file_line],
 373                          Some(ErrKind::Math(err.clone())))
 374                     }
 375                 };
 376
 377                 // If we know we always panic, and the error message
 378                 // is also constant, then we can produce a warning.
 379                 if const_cond == Some(!expected) {
 380                     if let Some(err) = const_err {
 381                         let err = ConstEvalErr{ span: span, kind: err };
 382                         let mut diag = bcx.tcx().sess.struct_span_warn(
 383                             span, "this expression will panic at run-time");
 384                         err.note(bcx.tcx(), span, "expression", &mut diag);
 385                         diag.emit();
 386                     }
 387                 }
 388
 389                 // Obtain the panic entry point.
 390                 let def_id = common::langcall(bcx.tcx(), Some(span), "", lang_item);
 391                 let instance = ty::Instance::mono(bcx.tcx(), def_id);
 392                 let fn_ty = FnType::of_instance(bcx.ccx, &instance);
 393                 let llfn = callee::get_fn(bcx.ccx, instance);
 394
 395                 // Translate the actual panic invoke/call.
 396                 do_call(self, bcx, fn_ty, llfn, &args, None, cleanup);
 397             }
 398
 399             mir::TerminatorKind::DropAndReplace { .. } => {
 400                 bug!("undesugared DropAndReplace in trans: {:?}", terminator);
 401             }
 402
 403             mir::TerminatorKind::Call { ref func, ref args, ref destination, cleanup } => {
 404                 // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar.
 405                 let callee = self.trans_operand(&bcx, func);
 406
 407                 let (instance, mut llfn) = match callee.ty.sty {
 408                     ty::TyFnDef(def_id, substs) => {
 409                         (Some(monomorphize::resolve(bcx.ccx.shared(), def_id, substs)),
 410                          None)
 411                     }
 412                     ty::TyFnPtr(_) => {
 413                         (None, Some(callee.immediate()))
 414                     }
 415                     _ => bug!("{} is not callable", callee.ty)
 416                 };
 417                 let def = instance.map(|i| i.def);
 418                 let sig = callee.ty.fn_sig(bcx.tcx());
 419                 let sig = bcx.tcx().erase_late_bound_regions_and_normalize(&sig);
 420                 let abi = sig.abi;
 421
 422                 // Handle intrinsics old trans wants Expr's for, ourselves.
 423                 let intrinsic = match def {
 424                     Some(ty::InstanceDef::Intrinsic(def_id))
 425                         => Some(bcx.tcx().item_name(def_id).as_str()),
 426                     _ => None
 427                 };
 428                 let intrinsic = intrinsic.as_ref().map(|s| &s[..]);
 429
 430                 if intrinsic == Some("transmute") {
 431                     let &(ref dest, target) = destination.as_ref().unwrap();
 432                     self.trans_transmute(&bcx, &args[0], dest);
 433                     funclet_br(self, bcx, target);
 434                     return;
 435                 }
 436
 437                 let extra_args = &args[sig.inputs().len()..];
 438                 let extra_args = extra_args.iter().map(|op_arg| {
 439                     let op_ty = op_arg.ty(&self.mir, bcx.tcx());
 440                     self.monomorphize(&op_ty)
 441                 }).collect::<Vec<_>>();
 442
 443                 let fn_ty = match def {
 444                     Some(ty::InstanceDef::Virtual(..)) => {
 445                         FnType::new_vtable(bcx.ccx, sig, &extra_args)
 446                     }
 447                     Some(ty::InstanceDef::DropGlue(_, None)) => {
 448                         // empty drop glue - a nop.
 449                         let &(_, target) = destination.as_ref().unwrap();
 450                         funclet_br(self, bcx, target);
 451                         return;
 452                     }
 453                     _ => FnType::new(bcx.ccx, sig, &extra_args)
 454                 };
 455
 456                 // The arguments we'll be passing. Plus one to account for outptr, if used.
 457                 let arg_count = fn_ty.args.len() + fn_ty.ret.is_indirect() as usize;
 458                 let mut llargs = Vec::with_capacity(arg_count);
 459
 460                 // Prepare the return value destination
 461                 let ret_dest = if let Some((ref dest, _)) = *destination {
 462                     let is_intrinsic = intrinsic.is_some();
 463                     self.make_return_dest(&bcx, dest, &fn_ty.ret, &mut llargs,
 464                                           is_intrinsic)
 465                 } else {
 466                     ReturnDest::Nothing
 467                 };
 468
 469                 // Split the rust-call tupled arguments off.
 470                 let (first_args, untuple) = if abi == Abi::RustCall && !args.is_empty() {
 471                     let (tup, args) = args.split_last().unwrap();
 472                     (args, Some(tup))
 473                 } else {
 474                     (&args[..], None)
 475                 };
 476
 477                 let is_shuffle = intrinsic.map_or(false, |name| {
 478                     name.starts_with("simd_shuffle")
 479                 });
 480                 let mut idx = 0;
 481                 for arg in first_args {
 482                     // The indices passed to simd_shuffle* in the
 483                     // third argument must be constant. This is
 484                     // checked by const-qualification, which also
 485                     // promotes any complex rvalues to constants.
 486                     if is_shuffle && idx == 2 {
 487                         match *arg {
 488                             mir::Operand::Consume(_) => {
 489                                 span_bug!(span, "shuffle indices must be constant");
 490                             }
 491                             mir::Operand::Constant(ref constant) => {
 492                                 let val = self.trans_constant(&bcx, constant);
 493                                 llargs.push(val.llval);
 494                                 idx += 1;
 495                                 continue;
 496                             }
 497                         }
 498                     }
 499
 500                     let op = self.trans_operand(&bcx, arg);
 501                     self.trans_argument(&bcx, op, &mut llargs, &fn_ty,
 502                                         &mut idx, &mut llfn, &def);
 503                 }
 504                 if let Some(tup) = untuple {
 505                     self.trans_arguments_untupled(&bcx, tup, &mut llargs, &fn_ty,
 506                                                   &mut idx, &mut llfn, &def)
 507                 }
 508
 509                 if intrinsic.is_some() && intrinsic != Some("drop_in_place") {
 510                     use intrinsic::trans_intrinsic_call;
 511
 512                     let (dest, llargs) = match ret_dest {
 513                         _ if fn_ty.ret.is_indirect() => {
 514                             (llargs[0], &llargs[1..])
 515                         }
 516                         ReturnDest::Nothing => {
 517                             (C_undef(fn_ty.ret.memory_ty(bcx.ccx).ptr_to()), &llargs[..])
 518                         }
 519                         ReturnDest::IndirectOperand(dst, _) |
 520                         ReturnDest::Store(dst) => (dst, &llargs[..]),
 521                         ReturnDest::DirectOperand(_) =>
 522                             bug!("Cannot use direct operand with an intrinsic call")
 523                     };
 524
 525                     let callee_ty = common::instance_ty(
 526                         bcx.ccx.shared(), instance.as_ref().unwrap());
 527                     trans_intrinsic_call(&bcx, callee_ty, &fn_ty, &llargs, dest,
 528                                          terminator.source_info.span);
 529
 530                     if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
 531                         // Make a fake operand for store_return
 532                         let op = OperandRef {
 533                             val: Ref(dst, Alignment::AbiAligned),
 534                             ty: sig.output(),
 535                         };
 536                         self.store_return(&bcx, ret_dest, &fn_ty.ret, op);
 537                     }
 538
 539                     if let Some((_, target)) = *destination {
 540                         funclet_br(self, bcx, target);
 541                     } else {
 542                         bcx.unreachable();
 543                     }
 544
 545                     return;
 546                 }
 547
 548                 let fn_ptr = match (llfn, instance) {
 549                     (Some(llfn), _) => llfn,
 550                     (None, Some(instance)) => callee::get_fn(bcx.ccx, instance),
 551                     _ => span_bug!(span, "no llfn for call"),
 552                 };
 553
 554                 do_call(self, bcx, fn_ty, fn_ptr, &llargs,
 555                         destination.as_ref().map(|&(_, target)| (ret_dest, sig.output(), target)),
 556                         cleanup);
 557             }
 558         }
 559     }
 560
 561     fn trans_argument(&mut self,
 562                       bcx: &Builder<'a, 'tcx>,
 563                       op: OperandRef<'tcx>,
 564                       llargs: &mut Vec<ValueRef>,
 565                       fn_ty: &FnType<'tcx>,
 566                       next_idx: &mut usize,
 567                       llfn: &mut Option<ValueRef>,
 568                       def: &Option<ty::InstanceDef<'tcx>>) {
 569         if let Pair(a, b) = op.val {
 570             // Treat the values in a fat pointer separately.
 571             if common::type_is_fat_ptr(bcx.ccx, op.ty) {
 572                 let (ptr, meta) = (a, b);
 573                 if *next_idx == 0 {
 574                     if let Some(ty::InstanceDef::Virtual(_, idx)) = *def {
 575                         let llmeth = meth::VirtualIndex::from_index(idx).get_fn(bcx, meta);
 576                         let llty = fn_ty.llvm_type(bcx.ccx).ptr_to();
 577                         *llfn = Some(bcx.pointercast(llmeth, llty));
 578                     }
 579                 }
 580
 581                 let imm_op = |x| OperandRef {
 582                     val: Immediate(x),
 583                     // We won't be checking the type again.
 584                     ty: bcx.tcx().types.err
 585                 };
 586                 self.trans_argument(bcx, imm_op(ptr), llargs, fn_ty, next_idx, llfn, def);
 587                 self.trans_argument(bcx, imm_op(meta), llargs, fn_ty, next_idx, llfn, def);
 588                 return;
 589             }
 590         }
 591
 592         let arg = &fn_ty.args[*next_idx];
 593         *next_idx += 1;
 594
 595         // Fill padding with undef value, where applicable.
 596         if let Some(ty) = arg.pad {
 597             llargs.push(C_undef(ty));
 598         }
 599
 600         if arg.is_ignore() {
 601             return;
 602         }
 603
 604         // Force by-ref if we have to load through a cast pointer.
 605         let (mut llval, align, by_ref) = match op.val {
 606             Immediate(_) | Pair(..) => {
 607                 if arg.is_indirect() || arg.cast.is_some() {
 608                     let llscratch = bcx.alloca(arg.memory_ty(bcx.ccx), "arg", None);
 609                     self.store_operand(bcx, llscratch, None, op);
 610                     (llscratch, Alignment::AbiAligned, true)
 611                 } else {
 612                     (op.pack_if_pair(bcx).immediate(), Alignment::AbiAligned, false)
 613                 }
 614             }
 615             Ref(llval, Alignment::Packed) if arg.is_indirect() => {
 616                 // `foo(packed.large_field)`. We can't pass the (unaligned) field directly. I
 617                 // think that ATM (Rust 1.16) we only pass temporaries, but we shouldn't
 618                 // have scary latent bugs around.
 619
 620                 let llscratch = bcx.alloca(arg.memory_ty(bcx.ccx), "arg", None);
 621                 base::memcpy_ty(bcx, llscratch, llval, op.ty, Some(1));
 622                 (llscratch, Alignment::AbiAligned, true)
 623             }
 624             Ref(llval, align) => (llval, align, true)
 625         };
 626
 627         if by_ref && !arg.is_indirect() {
 628             // Have to load the argument, maybe while casting it.
 629             if arg.layout.ty == bcx.tcx().types.bool {
 630                 // We store bools as i8 so we need to truncate to i1.
 631                 llval = bcx.load_range_assert(llval, 0, 2, llvm::False, None);
 632                 llval = bcx.trunc(llval, Type::i1(bcx.ccx));
 633             } else if let Some(ty) = arg.cast {
 634                 llval = bcx.load(bcx.pointercast(llval, ty.ptr_to()),
 635                                  align.min_with(arg.layout.align(bcx.ccx).abi() as u32));
 636             } else {
 637                 llval = bcx.load(llval, align.to_align());
 638             }
 639         }
 640
 641         llargs.push(llval);
 642     }
 643
 644     fn trans_arguments_untupled(&mut self,
 645                                 bcx: &Builder<'a, 'tcx>,
 646                                 operand: &mir::Operand<'tcx>,
 647                                 llargs: &mut Vec<ValueRef>,
 648                                 fn_ty: &FnType<'tcx>,
 649                                 next_idx: &mut usize,
 650                                 llfn: &mut Option<ValueRef>,
 651                                 def: &Option<ty::InstanceDef<'tcx>>) {
 652         let tuple = self.trans_operand(bcx, operand);
 653
 654         let arg_types = match tuple.ty.sty {
 655             ty::TyTuple(ref tys, _) => tys,
 656             _ => span_bug!(self.mir.span,
 657                            "bad final argument to \"rust-call\" fn {:?}", tuple.ty)
 658         };
 659
 660         // Handle both by-ref and immediate tuples.
 661         match tuple.val {
 662             Ref(llval, align) => {
 663                 for (n, &ty) in arg_types.iter().enumerate() {
 664                     let ptr = LvalueRef::new_sized_ty(llval, tuple.ty, align);
 665                     let (ptr, align) = ptr.trans_field_ptr(bcx, n);
 666                     let val = if common::type_is_fat_ptr(bcx.ccx, ty) {
 667                         let (lldata, llextra) = base::load_fat_ptr(bcx, ptr, align, ty);
 668                         Pair(lldata, llextra)
 669                     } else {
 670                         // trans_argument will load this if it needs to
 671                         Ref(ptr, align)
 672                     };
 673                     let op = OperandRef {
 674                         val: val,
 675                         ty: ty
 676                     };
 677                     self.trans_argument(bcx, op, llargs, fn_ty, next_idx, llfn, def);
 678                 }
 679
 680             }
 681             Immediate(llval) => {
 682                 let l = bcx.ccx.layout_of(tuple.ty);
 683                 let v = if let layout::Univariant { ref variant, .. } = *l {
 684                     variant
 685                 } else {
 686                     bug!("Not a tuple.");
 687                 };
 688                 for (n, &ty) in arg_types.iter().enumerate() {
 689                     let mut elem = bcx.extract_value(
 690                         llval, adt::struct_llfields_index(v, n));
 691                     // Truncate bools to i1, if needed
 692                     if ty.is_bool() && common::val_ty(elem) != Type::i1(bcx.ccx) {
 693                         elem = bcx.trunc(elem, Type::i1(bcx.ccx));
 694                     }
 695                     // If the tuple is immediate, the elements are as well
 696                     let op = OperandRef {
 697                         val: Immediate(elem),
 698                         ty: ty
 699                     };
 700                     self.trans_argument(bcx, op, llargs, fn_ty, next_idx, llfn, def);
 701                 }
 702             }
 703             Pair(a, b) => {
 704                 let elems = [a, b];
 705                 for (n, &ty) in arg_types.iter().enumerate() {
 706                     let mut elem = elems[n];
 707                     // Truncate bools to i1, if needed
 708                     if ty.is_bool() && common::val_ty(elem) != Type::i1(bcx.ccx) {
 709                         elem = bcx.trunc(elem, Type::i1(bcx.ccx));
 710                     }
 711                     // Pair is always made up of immediates
 712                     let op = OperandRef {
 713                         val: Immediate(elem),
 714                         ty: ty
 715                     };
 716                     self.trans_argument(bcx, op, llargs, fn_ty, next_idx, llfn, def);
 717                 }
 718             }
 719         }
 720
 721     }
 722
 723     fn get_personality_slot(&mut self, bcx: &Builder<'a, 'tcx>) -> ValueRef {
 724         let ccx = bcx.ccx;
 725         if let Some(slot) = self.llpersonalityslot {
 726             slot
 727         } else {
 728             let llretty = Type::struct_(ccx, &[Type::i8p(ccx), Type::i32(ccx)], false);
 729             let slot = bcx.alloca(llretty, "personalityslot", None);
 730             self.llpersonalityslot = Some(slot);
 731             slot
 732         }
 733     }
 734
 735     /// Return the landingpad wrapper around the given basic block
 736     ///
 737     /// No-op in MSVC SEH scheme.
 738     fn landing_pad_to(&mut self, target_bb: mir::BasicBlock) -> BasicBlockRef {
 739         if let Some(block) = self.landing_pads[target_bb] {
 740             return block;
 741         }
 742
 743         let block = self.blocks[target_bb];
 744         let landing_pad = self.landing_pad_uncached(block);
 745         self.landing_pads[target_bb] = Some(landing_pad);
 746         landing_pad
 747     }
 748
 749     fn landing_pad_uncached(&mut self, target_bb: BasicBlockRef) -> BasicBlockRef {
 750         if base::wants_msvc_seh(self.ccx.sess()) {
 751             span_bug!(self.mir.span, "landing pad was not inserted?")
 752         }
 753
 754         let bcx = self.new_block("cleanup");
 755
 756         let ccx = bcx.ccx;
 757         let llpersonality = self.ccx.eh_personality();
 758         let llretty = Type::struct_(ccx, &[Type::i8p(ccx), Type::i32(ccx)], false);
 759         let llretval = bcx.landing_pad(llretty, llpersonality, 1, self.llfn);
 760         bcx.set_cleanup(llretval);
 761         let slot = self.get_personality_slot(&bcx);
 762         Lifetime::Start.call(&bcx, slot);
 763         bcx.store(llretval, slot, None);
 764         bcx.br(target_bb);
 765         bcx.llbb()
 766     }
 767
 768     fn unreachable_block(&mut self) -> BasicBlockRef {
 769         self.unreachable_block.unwrap_or_else(|| {
 770             let bl = self.new_block("unreachable");
 771             bl.unreachable();
 772             self.unreachable_block = Some(bl.llbb());
 773             bl.llbb()
 774         })
 775     }
 776
 777     pub fn new_block(&self, name: &str) -> Builder<'a, 'tcx> {
 778         Builder::new_block(self.ccx, self.llfn, name)
 779     }
 780
 781     pub fn get_builder(&self, bb: mir::BasicBlock) -> Builder<'a, 'tcx> {
 782         let builder = Builder::with_ccx(self.ccx);
 783         builder.position_at_end(self.blocks[bb]);
 784         builder
 785     }
 786
 787     fn make_return_dest(&mut self, bcx: &Builder<'a, 'tcx>,
 788                         dest: &mir::Lvalue<'tcx>, fn_ret_ty: &ArgType,
 789                         llargs: &mut Vec<ValueRef>, is_intrinsic: bool) -> ReturnDest {
 790         // If the return is ignored, we can just return a do-nothing ReturnDest
 791         if fn_ret_ty.is_ignore() {
 792             return ReturnDest::Nothing;
 793         }
 794         let dest = if let mir::Lvalue::Local(index) = *dest {
 795             let ret_ty = self.monomorphized_lvalue_ty(dest);
 796             match self.locals[index] {
 797                 LocalRef::Lvalue(dest) => dest,
 798                 LocalRef::Operand(None) => {
 799                     // Handle temporary lvalues, specifically Operand ones, as
 800                     // they don't have allocas
 801                     return if fn_ret_ty.is_indirect() {
 802                         // Odd, but possible, case, we have an operand temporary,
 803                         // but the calling convention has an indirect return.
 804                         let tmp = LvalueRef::alloca(bcx, ret_ty, "tmp_ret");
 805                         llargs.push(tmp.llval);
 806                         ReturnDest::IndirectOperand(tmp.llval, index)
 807                     } else if is_intrinsic {
 808                         // Currently, intrinsics always need a location to store
 809                         // the result. so we create a temporary alloca for the
 810                         // result
 811                         let tmp = LvalueRef::alloca(bcx, ret_ty, "tmp_ret");
 812                         ReturnDest::IndirectOperand(tmp.llval, index)
 813                     } else {
 814                         ReturnDest::DirectOperand(index)
 815                     };
 816                 }
 817                 LocalRef::Operand(Some(_)) => {
 818                     bug!("lvalue local already assigned to");
 819                 }
 820             }
 821         } else {
 822             self.trans_lvalue(bcx, dest)
 823         };
 824         if fn_ret_ty.is_indirect() {
 825             match dest.alignment {
 826                 Alignment::AbiAligned => {
 827                     llargs.push(dest.llval);
 828                     ReturnDest::Nothing
 829                 },
 830                 Alignment::Packed => {
 831                     // Currently, MIR code generation does not create calls
 832                     // that store directly to fields of packed structs (in
 833                     // fact, the calls it creates write only to temps),
 834                     //
 835                     // If someone changes that, please update this code path
 836                     // to create a temporary.
 837                     span_bug!(self.mir.span, "can't directly store to unaligned value");
 838                 }
 839             }
 840         } else {
 841             ReturnDest::Store(dest.llval)
 842         }
 843     }
 844
 845     fn trans_transmute(&mut self, bcx: &Builder<'a, 'tcx>,
 846                        src: &mir::Operand<'tcx>,
 847                        dst: &mir::Lvalue<'tcx>) {
 848         if let mir::Lvalue::Local(index) = *dst {
 849             match self.locals[index] {
 850                 LocalRef::Lvalue(lvalue) => self.trans_transmute_into(bcx, src, &lvalue),
 851                 LocalRef::Operand(None) => {
 852                     let lvalue_ty = self.monomorphized_lvalue_ty(dst);
 853                     assert!(!lvalue_ty.has_erasable_regions());
 854                     let lvalue = LvalueRef::alloca(bcx, lvalue_ty, "transmute_temp");
 855                     self.trans_transmute_into(bcx, src, &lvalue);
 856                     let op = self.trans_load(bcx, lvalue.llval, lvalue.alignment, lvalue_ty);
 857                     self.locals[index] = LocalRef::Operand(Some(op));
 858                 }
 859                 LocalRef::Operand(Some(_)) => {
 860                     let ty = self.monomorphized_lvalue_ty(dst);
 861                     assert!(common::type_is_zero_size(bcx.ccx, ty),
 862                             "assigning to initialized SSAtemp");
 863                 }
 864             }
 865         } else {
 866             let dst = self.trans_lvalue(bcx, dst);
 867             self.trans_transmute_into(bcx, src, &dst);
 868         }
 869     }
 870
 871     fn trans_transmute_into(&mut self, bcx: &Builder<'a, 'tcx>,
 872                             src: &mir::Operand<'tcx>,
 873                             dst: &LvalueRef<'tcx>) {
 874         let val = self.trans_operand(bcx, src);
 875         let llty = type_of::type_of(bcx.ccx, val.ty);
 876         let cast_ptr = bcx.pointercast(dst.llval, llty.ptr_to());
 877         let in_type = val.ty;
 878         let out_type = dst.ty.to_ty(bcx.tcx());
 879         let llalign = cmp::min(bcx.ccx.align_of(in_type), bcx.ccx.align_of(out_type));
 880         self.store_operand(bcx, cast_ptr, Some(llalign), val);
 881     }
 882
 883
 884     // Stores the return value of a function call into it's final location.
 885     fn store_return(&mut self,
 886                     bcx: &Builder<'a, 'tcx>,
 887                     dest: ReturnDest,
 888                     ret_ty: &ArgType<'tcx>,
 889                     op: OperandRef<'tcx>) {
 890         use self::ReturnDest::*;
 891
 892         match dest {
 893             Nothing => (),
 894             Store(dst) => ret_ty.store(bcx, op.immediate(), dst),
 895             IndirectOperand(tmp, index) => {
 896                 let op = self.trans_load(bcx, tmp, Alignment::AbiAligned, op.ty);
 897                 self.locals[index] = LocalRef::Operand(Some(op));
 898             }
 899             DirectOperand(index) => {
 900                 // If there is a cast, we have to store and reload.
 901                 let op = if ret_ty.cast.is_some() {
 902                     let tmp = LvalueRef::alloca(bcx, op.ty, "tmp_ret");
 903                     ret_ty.store(bcx, op.immediate(), tmp.llval);
 904                     self.trans_load(bcx, tmp.llval, tmp.alignment, op.ty)
 905                 } else {
 906                     op.unpack_if_pair(bcx)
 907                 };
 908                 self.locals[index] = LocalRef::Operand(Some(op));
 909             }
 910         }
 911     }
 912 }
 913
 914 enum ReturnDest {
 915     // Do nothing, the return value is indirect or ignored
 916     Nothing,
 917     // Store the return value to the pointer
 918     Store(ValueRef),
 919     // Stores an indirect return value to an operand local lvalue
 920     IndirectOperand(ValueRef, mir::Local),
 921     // Stores a direct return value to an operand local lvalue
 922     DirectOperand(mir::Local)
 923 }