src/librustc_mir/transform/inline.rs

   1 //! Inlining pass for MIR functions
   2
   3 use rustc::hir::CodegenFnAttrFlags;
   4 use rustc::hir::def_id::DefId;
   5
   6 use rustc_data_structures::bit_set::BitSet;
   7 use rustc_data_structures::indexed_vec::{Idx, IndexVec};
   8
   9 use rustc::mir::*;
  10 use rustc::mir::visit::*;
  11 use rustc::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
  12 use rustc::ty::subst::{Subst, SubstsRef};
  13
  14 use std::collections::VecDeque;
  15 use std::iter;
  16 use crate::transform::{MirPass, MirSource};
  17 use super::simplify::{remove_dead_blocks, CfgSimplifier};
  18
  19 use syntax::attr;
  20 use rustc_target::spec::abi::Abi;
  21
  22 const DEFAULT_THRESHOLD: usize = 50;
  23 const HINT_THRESHOLD: usize = 100;
  24
  25 const INSTR_COST: usize = 5;
  26 const CALL_PENALTY: usize = 25;
  27
  28 const UNKNOWN_SIZE_COST: usize = 10;
  29
  30 pub struct Inline;
  31
  32 #[derive(Copy, Clone, Debug)]
  33 struct CallSite<'tcx> {
  34     callee: DefId,
  35     substs: SubstsRef<'tcx>,
  36     bb: BasicBlock,
  37     location: SourceInfo,
  38 }
  39
  40 impl MirPass for Inline {
  41     fn run_pass<'tcx>(&self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut Body<'tcx>) {
  42         if tcx.sess.opts.debugging_opts.mir_opt_level >= 2 {
  43             Inliner { tcx, source }.run_pass(body);
  44         }
  45     }
  46 }
  47
  48 struct Inliner<'tcx> {
  49     tcx: TyCtxt<'tcx>,
  50     source: MirSource<'tcx>,
  51 }
  52
  53 impl Inliner<'tcx> {
  54     fn run_pass(&self, caller_body: &mut Body<'tcx>) {
  55         // Keep a queue of callsites to try inlining on. We take
  56         // advantage of the fact that queries detect cycles here to
  57         // allow us to try and fetch the fully optimized MIR of a
  58         // call; if it succeeds, we can inline it and we know that
  59         // they do not call us.  Otherwise, we just don't try to
  60         // inline.
  61         //
  62         // We use a queue so that we inline "broadly" before we inline
  63         // in depth. It is unclear if this is the best heuristic,
  64         // really, but that's true of all the heuristics in this
  65         // file. =)
  66
  67         let mut callsites = VecDeque::new();
  68
  69         let param_env = self.tcx.param_env(self.source.def_id());
  70
  71         // Only do inlining into fn bodies.
  72         let id = self.tcx.hir().as_local_hir_id(self.source.def_id()).unwrap();
  73         if self.tcx.hir().body_owner_kind(id).is_fn_or_closure()
  74             && self.source.promoted.is_none()
  75         {
  76             for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated() {
  77                 if let Some(callsite) = self.get_valid_function_call(bb,
  78                                                                     bb_data,
  79                                                                     caller_body,
  80                                                                     param_env) {
  81                     callsites.push_back(callsite);
  82                 }
  83             }
  84         } else {
  85             return;
  86         }
  87
  88         let mut local_change;
  89         let mut changed = false;
  90
  91         loop {
  92             local_change = false;
  93             while let Some(callsite) = callsites.pop_front() {
  94                 debug!("checking whether to inline callsite {:?}", callsite);
  95                 if !self.tcx.is_mir_available(callsite.callee) {
  96                     debug!("checking whether to inline callsite {:?} - MIR unavailable", callsite);
  97                     continue;
  98                 }
  99
 100                 let self_node_id = self.tcx.hir().as_local_node_id(self.source.def_id()).unwrap();
 101                 let callee_node_id = self.tcx.hir().as_local_node_id(callsite.callee);
 102
 103                 let callee_body = if let Some(callee_node_id) = callee_node_id {
 104                     // Avoid a cycle here by only using `optimized_mir` only if we have
 105                     // a lower node id than the callee. This ensures that the callee will
 106                     // not inline us. This trick only works without incremental compilation.
 107                     // So don't do it if that is enabled.
 108                     if !self.tcx.dep_graph.is_fully_enabled()
 109                         && self_node_id.as_u32() < callee_node_id.as_u32() {
 110                         self.tcx.optimized_mir(callsite.callee)
 111                     } else {
 112                         continue;
 113                     }
 114                 } else {
 115                     // This cannot result in a cycle since the callee MIR is from another crate
 116                     // and is already optimized.
 117                     self.tcx.optimized_mir(callsite.callee)
 118                 };
 119
 120                 let callee_body = if self.consider_optimizing(callsite, callee_body) {
 121                     self.tcx.subst_and_normalize_erasing_regions(
 122                         &callsite.substs,
 123                         param_env,
 124                         callee_body,
 125                     )
 126                 } else {
 127                     continue;
 128                 };
 129
 130                 let start = caller_body.basic_blocks().len();
 131                 debug!("attempting to inline callsite {:?} - body={:?}", callsite, callee_body);
 132                 if !self.inline_call(callsite, caller_body, callee_body) {
 133                     debug!("attempting to inline callsite {:?} - failure", callsite);
 134                     continue;
 135                 }
 136                 debug!("attempting to inline callsite {:?} - success", callsite);
 137
 138                 // Add callsites from inlined function
 139                 for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated().skip(start) {
 140                     if let Some(new_callsite) = self.get_valid_function_call(bb,
 141                                                                              bb_data,
 142                                                                              caller_body,
 143                                                                              param_env) {
 144                         // Don't inline the same function multiple times.
 145                         if callsite.callee != new_callsite.callee {
 146                             callsites.push_back(new_callsite);
 147                         }
 148                     }
 149                 }
 150
 151                 local_change = true;
 152                 changed = true;
 153             }
 154
 155             if !local_change {
 156                 break;
 157             }
 158         }
 159
 160         // Simplify if we inlined anything.
 161         if changed {
 162             debug!("running simplify cfg on {:?}", self.source);
 163             CfgSimplifier::new(caller_body).simplify();
 164             remove_dead_blocks(caller_body);
 165         }
 166     }
 167
 168     fn get_valid_function_call(&self,
 169                                bb: BasicBlock,
 170                                bb_data: &BasicBlockData<'tcx>,
 171                                caller_body: &Body<'tcx>,
 172                                param_env: ParamEnv<'tcx>,
 173     ) -> Option<CallSite<'tcx>> {
 174         // Don't inline calls that are in cleanup blocks.
 175         if bb_data.is_cleanup { return None; }
 176
 177         // Only consider direct calls to functions
 178         let terminator = bb_data.terminator();
 179         if let TerminatorKind::Call { func: ref op, .. } = terminator.kind {
 180             if let ty::FnDef(callee_def_id, substs) = op.ty(caller_body, self.tcx).sty {
 181                 let instance = Instance::resolve(self.tcx,
 182                                                  param_env,
 183                                                  callee_def_id,
 184                                                  substs)?;
 185
 186                 if let InstanceDef::Virtual(..) = instance.def {
 187                     return None;
 188                 }
 189
 190                 return Some(CallSite {
 191                     callee: instance.def_id(),
 192                     substs: instance.substs,
 193                     bb,
 194                     location: terminator.source_info
 195                 });
 196             }
 197         }
 198
 199         None
 200     }
 201
 202     fn consider_optimizing(&self,
 203                            callsite: CallSite<'tcx>,
 204                            callee_body: &Body<'tcx>)
 205                            -> bool
 206     {
 207         debug!("consider_optimizing({:?})", callsite);
 208         self.should_inline(callsite, callee_body)
 209             && self.tcx.consider_optimizing(|| format!("Inline {:?} into {:?}",
 210                                                        callee_body.span,
 211                                                        callsite))
 212     }
 213
 214     fn should_inline(&self,
 215                      callsite: CallSite<'tcx>,
 216                      callee_body: &Body<'tcx>)
 217                      -> bool
 218     {
 219         debug!("should_inline({:?})", callsite);
 220         let tcx = self.tcx;
 221
 222         // Don't inline closures that have capture debuginfo
 223         // FIXME: Handle closures better
 224         if callee_body.__upvar_debuginfo_codegen_only_do_not_use.len() > 0 {
 225             debug!("    upvar debuginfo present - not inlining");
 226             return false;
 227         }
 228
 229         // Cannot inline generators which haven't been transformed yet
 230         if callee_body.yield_ty.is_some() {
 231             debug!("    yield ty present - not inlining");
 232             return false;
 233         }
 234
 235         let codegen_fn_attrs = tcx.codegen_fn_attrs(callsite.callee);
 236
 237         let hinted = match codegen_fn_attrs.inline {
 238             // Just treat inline(always) as a hint for now,
 239             // there are cases that prevent inlining that we
 240             // need to check for first.
 241             attr::InlineAttr::Always => true,
 242             attr::InlineAttr::Never => {
 243                 debug!("`#[inline(never)]` present - not inlining");
 244                 return false
 245             }
 246             attr::InlineAttr::Hint => true,
 247             attr::InlineAttr::None => false,
 248         };
 249
 250         // Only inline local functions if they would be eligible for cross-crate
 251         // inlining. This is to ensure that the final crate doesn't have MIR that
 252         // reference unexported symbols
 253         if callsite.callee.is_local() {
 254             if callsite.substs.non_erasable_generics().count() == 0 && !hinted {
 255                 debug!("    callee is an exported function - not inlining");
 256                 return false;
 257             }
 258         }
 259
 260         let mut threshold = if hinted {
 261             HINT_THRESHOLD
 262         } else {
 263             DEFAULT_THRESHOLD
 264         };
 265
 266         // Significantly lower the threshold for inlining cold functions
 267         if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) {
 268             threshold /= 5;
 269         }
 270
 271         // Give a bonus functions with a small number of blocks,
 272         // We normally have two or three blocks for even
 273         // very small functions.
 274         if callee_body.basic_blocks().len() <= 3 {
 275             threshold += threshold / 4;
 276         }
 277         debug!("    final inline threshold = {}", threshold);
 278
 279         // FIXME: Give a bonus to functions with only a single caller
 280
 281         let param_env = tcx.param_env(self.source.def_id());
 282
 283         let mut first_block = true;
 284         let mut cost = 0;
 285
 286         // Traverse the MIR manually so we can account for the effects of
 287         // inlining on the CFG.
 288         let mut work_list = vec![START_BLOCK];
 289         let mut visited = BitSet::new_empty(callee_body.basic_blocks().len());
 290         while let Some(bb) = work_list.pop() {
 291             if !visited.insert(bb.index()) { continue; }
 292             let blk = &callee_body.basic_blocks()[bb];
 293
 294             for stmt in &blk.statements {
 295                 // Don't count StorageLive/StorageDead in the inlining cost.
 296                 match stmt.kind {
 297                     StatementKind::StorageLive(_) |
 298                     StatementKind::StorageDead(_) |
 299                     StatementKind::Nop => {}
 300                     _ => cost += INSTR_COST
 301                 }
 302             }
 303             let term = blk.terminator();
 304             let mut is_drop = false;
 305             match term.kind {
 306                 TerminatorKind::Drop { ref location, target, unwind } |
 307                 TerminatorKind::DropAndReplace { ref location, target, unwind, .. } => {
 308                     is_drop = true;
 309                     work_list.push(target);
 310                     // If the location doesn't actually need dropping, treat it like
 311                     // a regular goto.
 312                     let ty = location.ty(callee_body, tcx).subst(tcx, callsite.substs).ty;
 313                     if ty.needs_drop(tcx, param_env) {
 314                         cost += CALL_PENALTY;
 315                         if let Some(unwind) = unwind {
 316                             work_list.push(unwind);
 317                         }
 318                     } else {
 319                         cost += INSTR_COST;
 320                     }
 321                 }
 322
 323                 TerminatorKind::Unreachable |
 324                 TerminatorKind::Call { destination: None, .. } if first_block => {
 325                     // If the function always diverges, don't inline
 326                     // unless the cost is zero
 327                     threshold = 0;
 328                 }
 329
 330                 TerminatorKind::Call {func: Operand::Constant(ref f), .. } => {
 331                     if let ty::FnDef(def_id, _) = f.ty.sty {
 332                         // Don't give intrinsics the extra penalty for calls
 333                         let f = tcx.fn_sig(def_id);
 334                         if f.abi() == Abi::RustIntrinsic || f.abi() == Abi::PlatformIntrinsic {
 335                             cost += INSTR_COST;
 336                         } else {
 337                             cost += CALL_PENALTY;
 338                         }
 339                     }
 340                 }
 341                 TerminatorKind::Assert { .. } => cost += CALL_PENALTY,
 342                 _ => cost += INSTR_COST
 343             }
 344
 345             if !is_drop {
 346                 for &succ in term.successors() {
 347                     work_list.push(succ);
 348                 }
 349             }
 350
 351             first_block = false;
 352         }
 353
 354         // Count up the cost of local variables and temps, if we know the size
 355         // use that, otherwise we use a moderately-large dummy cost.
 356
 357         let ptr_size = tcx.data_layout.pointer_size.bytes();
 358
 359         for v in callee_body.vars_and_temps_iter() {
 360             let v = &callee_body.local_decls[v];
 361             let ty = v.ty.subst(tcx, callsite.substs);
 362             // Cost of the var is the size in machine-words, if we know
 363             // it.
 364             if let Some(size) = type_size_of(tcx, param_env.clone(), ty) {
 365                 cost += (size / ptr_size) as usize;
 366             } else {
 367                 cost += UNKNOWN_SIZE_COST;
 368             }
 369         }
 370
 371         if let attr::InlineAttr::Always = codegen_fn_attrs.inline {
 372             debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost);
 373             true
 374         } else {
 375             if cost <= threshold {
 376                 debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold);
 377                 true
 378             } else {
 379                 debug!("NOT inlining {:?} [cost={} > threshold={}]", callsite, cost, threshold);
 380                 false
 381             }
 382         }
 383     }
 384
 385     fn inline_call(&self,
 386                    callsite: CallSite<'tcx>,
 387                    caller_body: &mut Body<'tcx>,
 388                    mut callee_body: Body<'tcx>) -> bool {
 389         let terminator = caller_body[callsite.bb].terminator.take().unwrap();
 390         match terminator.kind {
 391             // FIXME: Handle inlining of diverging calls
 392             TerminatorKind::Call { args, destination: Some(destination), cleanup, .. } => {
 393                 debug!("inlined {:?} into {:?}", callsite.callee, self.source);
 394
 395                 let mut local_map = IndexVec::with_capacity(callee_body.local_decls.len());
 396                 let mut scope_map = IndexVec::with_capacity(callee_body.source_scopes.len());
 397                 let mut promoted_map = IndexVec::with_capacity(callee_body.promoted.len());
 398
 399                 for mut scope in callee_body.source_scopes.iter().cloned() {
 400                     if scope.parent_scope.is_none() {
 401                         scope.parent_scope = Some(callsite.location.scope);
 402                         scope.span = callee_body.span;
 403                     }
 404
 405                     scope.span = callsite.location.span;
 406
 407                     let idx = caller_body.source_scopes.push(scope);
 408                     scope_map.push(idx);
 409                 }
 410
 411                 for loc in callee_body.vars_and_temps_iter() {
 412                     let mut local = callee_body.local_decls[loc].clone();
 413
 414                     local.source_info.scope =
 415                         scope_map[local.source_info.scope];
 416                     local.source_info.span = callsite.location.span;
 417                     local.visibility_scope = scope_map[local.visibility_scope];
 418
 419                     let idx = caller_body.local_decls.push(local);
 420                     local_map.push(idx);
 421                 }
 422
 423                 promoted_map.extend(
 424                     callee_body.promoted.iter().cloned().map(|p| caller_body.promoted.push(p))
 425                 );
 426
 427                 // If the call is something like `a[*i] = f(i)`, where
 428                 // `i : &mut usize`, then just duplicating the `a[*i]`
 429                 // Place could result in two different locations if `f`
 430                 // writes to `i`. To prevent this we need to create a temporary
 431                 // borrow of the place and pass the destination as `*temp` instead.
 432                 fn dest_needs_borrow(place: &Place<'_>) -> bool {
 433                     place.iterate(|place_base, place_projection| {
 434                         for proj in place_projection {
 435                             match proj.elem {
 436                                 ProjectionElem::Deref |
 437                                 ProjectionElem::Index(_) => return true,
 438                                 _ => {}
 439                             }
 440                         }
 441
 442                         match place_base {
 443                             // Static variables need a borrow because the callee
 444                             // might modify the same static.
 445                             PlaceBase::Static(_) => true,
 446                             _ => false
 447                         }
 448                     })
 449                 }
 450
 451                 let dest = if dest_needs_borrow(&destination.0) {
 452                     debug!("creating temp for return destination");
 453                     let dest = Rvalue::Ref(
 454                         self.tcx.lifetimes.re_erased,
 455                         BorrowKind::Mut { allow_two_phase_borrow: false },
 456                         destination.0);
 457
 458                     let ty = dest.ty(caller_body, self.tcx);
 459
 460                     let temp = LocalDecl::new_temp(ty, callsite.location.span);
 461
 462                     let tmp = caller_body.local_decls.push(temp);
 463                     let tmp = Place::from(tmp);
 464
 465                     let stmt = Statement {
 466                         source_info: callsite.location,
 467                         kind: StatementKind::Assign(tmp.clone(), box dest)
 468                     };
 469                     caller_body[callsite.bb]
 470                         .statements.push(stmt);
 471                     tmp.deref()
 472                 } else {
 473                     destination.0
 474                 };
 475
 476                 let return_block = destination.1;
 477
 478                 // Copy the arguments if needed.
 479                 let args: Vec<_> = self.make_call_args(args, &callsite, caller_body);
 480
 481                 let bb_len = caller_body.basic_blocks().len();
 482                 let mut integrator = Integrator {
 483                     block_idx: bb_len,
 484                     args: &args,
 485                     local_map,
 486                     scope_map,
 487                     promoted_map,
 488                     _callsite: callsite,
 489                     destination: dest,
 490                     return_block,
 491                     cleanup_block: cleanup,
 492                     in_cleanup_block: false
 493                 };
 494
 495
 496                 for (bb, mut block) in callee_body.basic_blocks_mut().drain_enumerated(..) {
 497                     integrator.visit_basic_block_data(bb, &mut block);
 498                     caller_body.basic_blocks_mut().push(block);
 499                 }
 500
 501                 let terminator = Terminator {
 502                     source_info: callsite.location,
 503                     kind: TerminatorKind::Goto { target: BasicBlock::new(bb_len) }
 504                 };
 505
 506                 caller_body[callsite.bb].terminator = Some(terminator);
 507
 508                 true
 509             }
 510             kind => {
 511                 caller_body[callsite.bb].terminator = Some(Terminator {
 512                     source_info: terminator.source_info,
 513                     kind,
 514                 });
 515                 false
 516             }
 517         }
 518     }
 519
 520     fn make_call_args(
 521         &self,
 522         args: Vec<Operand<'tcx>>,
 523         callsite: &CallSite<'tcx>,
 524         caller_body: &mut Body<'tcx>,
 525     ) -> Vec<Local> {
 526         let tcx = self.tcx;
 527
 528         // There is a bit of a mismatch between the *caller* of a closure and the *callee*.
 529         // The caller provides the arguments wrapped up in a tuple:
 530         //
 531         //     tuple_tmp = (a, b, c)
 532         //     Fn::call(closure_ref, tuple_tmp)
 533         //
 534         // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`)
 535         // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has
 536         // the job of unpacking this tuple. But here, we are codegen. =) So we want to create
 537         // a vector like
 538         //
 539         //     [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2]
 540         //
 541         // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient
 542         // if we "spill" that into *another* temporary, so that we can map the argument
 543         // variable in the callee MIR directly to an argument variable on our side.
 544         // So we introduce temporaries like:
 545         //
 546         //     tmp0 = tuple_tmp.0
 547         //     tmp1 = tuple_tmp.1
 548         //     tmp2 = tuple_tmp.2
 549         //
 550         // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`.
 551         if tcx.is_closure(callsite.callee) {
 552             let mut args = args.into_iter();
 553             let self_ = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
 554             let tuple = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
 555             assert!(args.next().is_none());
 556
 557             let tuple = Place::from(tuple);
 558             let tuple_tys = if let ty::Tuple(s) = tuple.ty(caller_body, tcx).ty.sty {
 559                 s
 560             } else {
 561                 bug!("Closure arguments are not passed as a tuple");
 562             };
 563
 564             // The `closure_ref` in our example above.
 565             let closure_ref_arg = iter::once(self_);
 566
 567             // The `tmp0`, `tmp1`, and `tmp2` in our example abonve.
 568             let tuple_tmp_args =
 569                 tuple_tys.iter().enumerate().map(|(i, ty)| {
 570                     // This is e.g., `tuple_tmp.0` in our example above.
 571                     let tuple_field = Operand::Move(tuple.clone().field(
 572                         Field::new(i),
 573                         ty.expect_ty(),
 574                     ));
 575
 576                     // Spill to a local to make e.g., `tmp0`.
 577                     self.create_temp_if_necessary(tuple_field, callsite, caller_body)
 578                 });
 579
 580             closure_ref_arg.chain(tuple_tmp_args).collect()
 581         } else {
 582             args.into_iter()
 583                 .map(|a| self.create_temp_if_necessary(a, callsite, caller_body))
 584                 .collect()
 585         }
 586     }
 587
 588     /// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh
 589     /// temporary `T` and an instruction `T = arg`, and returns `T`.
 590     fn create_temp_if_necessary(
 591         &self,
 592         arg: Operand<'tcx>,
 593         callsite: &CallSite<'tcx>,
 594         caller_body: &mut Body<'tcx>,
 595     ) -> Local {
 596         // FIXME: Analysis of the usage of the arguments to avoid
 597         // unnecessary temporaries.
 598
 599         if let Operand::Move(Place {
 600             base: PlaceBase::Local(local),
 601             projection: None,
 602         }) = arg {
 603             if caller_body.local_kind(local) == LocalKind::Temp {
 604                 // Reuse the operand if it's a temporary already
 605                 return local;
 606             }
 607         }
 608
 609         debug!("creating temp for argument {:?}", arg);
 610         // Otherwise, create a temporary for the arg
 611         let arg = Rvalue::Use(arg);
 612
 613         let ty = arg.ty(caller_body, self.tcx);
 614
 615         let arg_tmp = LocalDecl::new_temp(ty, callsite.location.span);
 616         let arg_tmp = caller_body.local_decls.push(arg_tmp);
 617
 618         let stmt = Statement {
 619             source_info: callsite.location,
 620             kind: StatementKind::Assign(Place::from(arg_tmp), box arg),
 621         };
 622         caller_body[callsite.bb].statements.push(stmt);
 623         arg_tmp
 624     }
 625 }
 626
 627 fn type_size_of<'tcx>(
 628     tcx: TyCtxt<'tcx>,
 629     param_env: ty::ParamEnv<'tcx>,
 630     ty: Ty<'tcx>,
 631 ) -> Option<u64> {
 632     tcx.layout_of(param_env.and(ty)).ok().map(|layout| layout.size.bytes())
 633 }
 634
 635 /**
 636  * Integrator.
 637  *
 638  * Integrates blocks from the callee function into the calling function.
 639  * Updates block indices, references to locals and other control flow
 640  * stuff.
 641 */
 642 struct Integrator<'a, 'tcx> {
 643     block_idx: usize,
 644     args: &'a [Local],
 645     local_map: IndexVec<Local, Local>,
 646     scope_map: IndexVec<SourceScope, SourceScope>,
 647     promoted_map: IndexVec<Promoted, Promoted>,
 648     _callsite: CallSite<'tcx>,
 649     destination: Place<'tcx>,
 650     return_block: BasicBlock,
 651     cleanup_block: Option<BasicBlock>,
 652     in_cleanup_block: bool,
 653 }
 654
 655 impl<'a, 'tcx> Integrator<'a, 'tcx> {
 656     fn update_target(&self, tgt: BasicBlock) -> BasicBlock {
 657         let new = BasicBlock::new(tgt.index() + self.block_idx);
 658         debug!("updating target `{:?}`, new: `{:?}`", tgt, new);
 659         new
 660     }
 661 }
 662
 663 impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> {
 664     fn visit_local(&mut self,
 665                    local: &mut Local,
 666                    _ctxt: PlaceContext,
 667                    _location: Location) {
 668         if *local == RETURN_PLACE {
 669             match self.destination {
 670                 Place {
 671                     base: PlaceBase::Local(l),
 672                     projection: None,
 673                 } => {
 674                     *local = l;
 675                     return;
 676                 },
 677                 ref place => bug!("Return place is {:?}, not local", place)
 678             }
 679         }
 680         let idx = local.index() - 1;
 681         if idx < self.args.len() {
 682             *local = self.args[idx];
 683             return;
 684         }
 685         *local = self.local_map[Local::new(idx - self.args.len())];
 686     }
 687
 688     fn visit_place(&mut self,
 689                     place: &mut Place<'tcx>,
 690                     _ctxt: PlaceContext,
 691                     _location: Location) {
 692
 693         match place {
 694             Place {
 695                 base: PlaceBase::Local(RETURN_PLACE),
 696                 projection: None,
 697             } => {
 698                 // Return pointer; update the place itself
 699                 *place = self.destination.clone();
 700             },
 701             Place {
 702                 base: PlaceBase::Static(box Static {
 703                     kind: StaticKind::Promoted(promoted),
 704                     ..
 705                 }),
 706                 projection: None,
 707             } => {
 708                 if let Some(p) = self.promoted_map.get(*promoted).cloned() {
 709                     *promoted = p;
 710                 }
 711             },
 712             _ => self.super_place(place, _ctxt, _location)
 713         }
 714     }
 715
 716     fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) {
 717         self.in_cleanup_block = data.is_cleanup;
 718         self.super_basic_block_data(block, data);
 719         self.in_cleanup_block = false;
 720     }
 721
 722     fn visit_retag(
 723         &mut self,
 724         kind: &mut RetagKind,
 725         place: &mut Place<'tcx>,
 726         loc: Location,
 727     ) {
 728         self.super_retag(kind, place, loc);
 729
 730         // We have to patch all inlined retags to be aware that they are no longer
 731         // happening on function entry.
 732         if *kind == RetagKind::FnEntry {
 733             *kind = RetagKind::Default;
 734         }
 735     }
 736
 737     fn visit_terminator_kind(&mut self,
 738                              kind: &mut TerminatorKind<'tcx>, loc: Location) {
 739         self.super_terminator_kind(kind, loc);
 740
 741         match *kind {
 742             TerminatorKind::GeneratorDrop |
 743             TerminatorKind::Yield { .. } => bug!(),
 744             TerminatorKind::Goto { ref mut target} => {
 745                 *target = self.update_target(*target);
 746             }
 747             TerminatorKind::SwitchInt { ref mut targets, .. } => {
 748                 for tgt in targets {
 749                     *tgt = self.update_target(*tgt);
 750                 }
 751             }
 752             TerminatorKind::Drop { ref mut target, ref mut unwind, .. } |
 753             TerminatorKind::DropAndReplace { ref mut target, ref mut unwind, .. } => {
 754                 *target = self.update_target(*target);
 755                 if let Some(tgt) = *unwind {
 756                     *unwind = Some(self.update_target(tgt));
 757                 } else if !self.in_cleanup_block {
 758                     // Unless this drop is in a cleanup block, add an unwind edge to
 759                     // the original call's cleanup block
 760                     *unwind = self.cleanup_block;
 761                 }
 762             }
 763             TerminatorKind::Call { ref mut destination, ref mut cleanup, .. } => {
 764                 if let Some((_, ref mut tgt)) = *destination {
 765                     *tgt = self.update_target(*tgt);
 766                 }
 767                 if let Some(tgt) = *cleanup {
 768                     *cleanup = Some(self.update_target(tgt));
 769                 } else if !self.in_cleanup_block {
 770                     // Unless this call is in a cleanup block, add an unwind edge to
 771                     // the original call's cleanup block
 772                     *cleanup = self.cleanup_block;
 773                 }
 774             }
 775             TerminatorKind::Assert { ref mut target, ref mut cleanup, .. } => {
 776                 *target = self.update_target(*target);
 777                 if let Some(tgt) = *cleanup {
 778                     *cleanup = Some(self.update_target(tgt));
 779                 } else if !self.in_cleanup_block {
 780                     // Unless this assert is in a cleanup block, add an unwind edge to
 781                     // the original call's cleanup block
 782                     *cleanup = self.cleanup_block;
 783                 }
 784             }
 785             TerminatorKind::Return => {
 786                 *kind = TerminatorKind::Goto { target: self.return_block };
 787             }
 788             TerminatorKind::Resume => {
 789                 if let Some(tgt) = self.cleanup_block {
 790                     *kind = TerminatorKind::Goto { target: tgt }
 791                 }
 792             }
 793             TerminatorKind::Abort => { }
 794             TerminatorKind::Unreachable => { }
 795             TerminatorKind::FalseEdges { ref mut real_target, ref mut imaginary_target } => {
 796                 *real_target = self.update_target(*real_target);
 797                 *imaginary_target = self.update_target(*imaginary_target);
 798             }
 799             TerminatorKind::FalseUnwind { real_target: _ , unwind: _ } =>
 800                 // see the ordering of passes in the optimized_mir query.
 801                 bug!("False unwinds should have been removed before inlining")
 802         }
 803     }
 804
 805     fn visit_source_scope(&mut self, scope: &mut SourceScope) {
 806         *scope = self.scope_map[*scope];
 807     }
 808 }