compiler/rustc_mir/src/transform/inline.rs

   1 //! Inlining pass for MIR functions
   2
   3 use rustc_attr as attr;
   4 use rustc_index::bit_set::BitSet;
   5 use rustc_index::vec::Idx;
   6 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
   7 use rustc_middle::mir::visit::*;
   8 use rustc_middle::mir::*;
   9 use rustc_middle::ty::subst::Subst;
  10 use rustc_middle::ty::{self, ConstKind, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
  11 use rustc_target::spec::abi::Abi;
  12
  13 use super::simplify::{remove_dead_blocks, CfgSimplifier};
  14 use crate::transform::MirPass;
  15 use std::collections::VecDeque;
  16 use std::iter;
  17 use std::ops::RangeFrom;
  18
  19 const DEFAULT_THRESHOLD: usize = 50;
  20 const HINT_THRESHOLD: usize = 100;
  21
  22 const INSTR_COST: usize = 5;
  23 const CALL_PENALTY: usize = 25;
  24 const LANDINGPAD_PENALTY: usize = 50;
  25 const RESUME_PENALTY: usize = 45;
  26
  27 const UNKNOWN_SIZE_COST: usize = 10;
  28
  29 pub struct Inline;
  30
  31 #[derive(Copy, Clone, Debug)]
  32 struct CallSite<'tcx> {
  33     callee: Instance<'tcx>,
  34     bb: BasicBlock,
  35     source_info: SourceInfo,
  36 }
  37
  38 impl<'tcx> MirPass<'tcx> for Inline {
  39     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
  40         if tcx.sess.opts.debugging_opts.mir_opt_level >= 2 {
  41             if tcx.sess.opts.debugging_opts.instrument_coverage {
  42                 // The current implementation of source code coverage injects code region counters
  43                 // into the MIR, and assumes a 1-to-1 correspondence between MIR and source-code-
  44                 // based function.
  45                 debug!("function inlining is disabled when compiling with `instrument_coverage`");
  46             } else {
  47                 Inliner {
  48                     tcx,
  49                     param_env: tcx.param_env_reveal_all_normalized(body.source.def_id()),
  50                     codegen_fn_attrs: tcx.codegen_fn_attrs(body.source.def_id()),
  51                 }
  52                 .run_pass(body);
  53             }
  54         }
  55     }
  56 }
  57
  58 struct Inliner<'tcx> {
  59     tcx: TyCtxt<'tcx>,
  60     param_env: ParamEnv<'tcx>,
  61     codegen_fn_attrs: &'tcx CodegenFnAttrs,
  62 }
  63
  64 impl Inliner<'tcx> {
  65     fn run_pass(&self, caller_body: &mut Body<'tcx>) {
  66         // Keep a queue of callsites to try inlining on. We take
  67         // advantage of the fact that queries detect cycles here to
  68         // allow us to try and fetch the fully optimized MIR of a
  69         // call; if it succeeds, we can inline it and we know that
  70         // they do not call us.  Otherwise, we just don't try to
  71         // inline.
  72         //
  73         // We use a queue so that we inline "broadly" before we inline
  74         // in depth. It is unclear if this is the best heuristic,
  75         // really, but that's true of all the heuristics in this
  76         // file. =)
  77
  78         let mut callsites = VecDeque::new();
  79
  80         let def_id = caller_body.source.def_id();
  81
  82         // Only do inlining into fn bodies.
  83         let self_hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
  84         if self.tcx.hir().body_owner_kind(self_hir_id).is_fn_or_closure()
  85             && caller_body.source.promoted.is_none()
  86         {
  87             for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated() {
  88                 if let Some(callsite) = self.get_valid_function_call(bb, bb_data, caller_body) {
  89                     callsites.push_back(callsite);
  90                 }
  91             }
  92         } else {
  93             return;
  94         }
  95
  96         let mut changed = false;
  97         while let Some(callsite) = callsites.pop_front() {
  98             debug!("checking whether to inline callsite {:?}", callsite);
  99
 100             if let InstanceDef::Item(_) = callsite.callee.def {
 101                 if !self.tcx.is_mir_available(callsite.callee.def_id()) {
 102                     debug!("checking whether to inline callsite {:?} - MIR unavailable", callsite,);
 103                     continue;
 104                 }
 105             }
 106
 107             let callee_body = if let Some(callee_def_id) = callsite.callee.def_id().as_local() {
 108                 let callee_hir_id = self.tcx.hir().local_def_id_to_hir_id(callee_def_id);
 109                 // Avoid a cycle here by only using `instance_mir` only if we have
 110                 // a lower `HirId` than the callee. This ensures that the callee will
 111                 // not inline us. This trick only works without incremental compilation.
 112                 // So don't do it if that is enabled. Also avoid inlining into generators,
 113                 // since their `optimized_mir` is used for layout computation, which can
 114                 // create a cycle, even when no attempt is made to inline the function
 115                 // in the other direction.
 116                 if !self.tcx.dep_graph.is_fully_enabled()
 117                     && self_hir_id < callee_hir_id
 118                     && caller_body.generator_kind.is_none()
 119                 {
 120                     self.tcx.instance_mir(callsite.callee.def)
 121                 } else {
 122                     continue;
 123                 }
 124             } else {
 125                 // This cannot result in a cycle since the callee MIR is from another crate
 126                 // and is already optimized.
 127                 self.tcx.instance_mir(callsite.callee.def)
 128             };
 129
 130             let callee_body: &Body<'tcx> = &*callee_body;
 131
 132             let callee_body = if self.consider_optimizing(callsite, callee_body) {
 133                 self.tcx.subst_and_normalize_erasing_regions(
 134                     &callsite.callee.substs,
 135                     self.param_env,
 136                     callee_body,
 137                 )
 138             } else {
 139                 continue;
 140             };
 141
 142             // Copy only unevaluated constants from the callee_body into the caller_body.
 143             // Although we are only pushing `ConstKind::Unevaluated` consts to
 144             // `required_consts`, here we may not only have `ConstKind::Unevaluated`
 145             // because we are calling `subst_and_normalize_erasing_regions`.
 146             caller_body.required_consts.extend(callee_body.required_consts.iter().copied().filter(
 147                 |&constant| matches!(constant.literal.val, ConstKind::Unevaluated(_, _, _)),
 148             ));
 149
 150             let start = caller_body.basic_blocks().len();
 151             debug!("attempting to inline callsite {:?} - body={:?}", callsite, callee_body);
 152             if !self.inline_call(callsite, caller_body, callee_body) {
 153                 debug!("attempting to inline callsite {:?} - failure", callsite);
 154                 continue;
 155             }
 156             debug!("attempting to inline callsite {:?} - success", callsite);
 157
 158             // Add callsites from inlined function
 159             for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated().skip(start) {
 160                 if let Some(new_callsite) = self.get_valid_function_call(bb, bb_data, caller_body) {
 161                     // Don't inline the same function multiple times.
 162                     if callsite.callee != new_callsite.callee {
 163                         callsites.push_back(new_callsite);
 164                     }
 165                 }
 166             }
 167
 168             changed = true;
 169         }
 170
 171         // Simplify if we inlined anything.
 172         if changed {
 173             debug!("running simplify cfg on {:?}", caller_body.source);
 174             CfgSimplifier::new(caller_body).simplify();
 175             remove_dead_blocks(caller_body);
 176         }
 177     }
 178
 179     fn get_valid_function_call(
 180         &self,
 181         bb: BasicBlock,
 182         bb_data: &BasicBlockData<'tcx>,
 183         caller_body: &Body<'tcx>,
 184     ) -> Option<CallSite<'tcx>> {
 185         // Don't inline calls that are in cleanup blocks.
 186         if bb_data.is_cleanup {
 187             return None;
 188         }
 189
 190         // Only consider direct calls to functions
 191         let terminator = bb_data.terminator();
 192         if let TerminatorKind::Call { func: ref op, .. } = terminator.kind {
 193             if let ty::FnDef(callee_def_id, substs) = *op.ty(caller_body, self.tcx).kind() {
 194                 // To resolve an instance its substs have to be fully normalized, so
 195                 // we do this here.
 196                 let normalized_substs = self.tcx.normalize_erasing_regions(self.param_env, substs);
 197                 let callee =
 198                     Instance::resolve(self.tcx, self.param_env, callee_def_id, normalized_substs)
 199                         .ok()
 200                         .flatten()?;
 201
 202                 if let InstanceDef::Virtual(..) | InstanceDef::Intrinsic(_) = callee.def {
 203                     return None;
 204                 }
 205
 206                 return Some(CallSite { callee, bb, source_info: terminator.source_info });
 207             }
 208         }
 209
 210         None
 211     }
 212
 213     fn consider_optimizing(&self, callsite: CallSite<'tcx>, callee_body: &Body<'tcx>) -> bool {
 214         debug!("consider_optimizing({:?})", callsite);
 215         self.should_inline(callsite, callee_body)
 216             && self.tcx.consider_optimizing(|| {
 217                 format!("Inline {:?} into {:?}", callee_body.span, callsite)
 218             })
 219     }
 220
 221     fn should_inline(&self, callsite: CallSite<'tcx>, callee_body: &Body<'tcx>) -> bool {
 222         debug!("should_inline({:?})", callsite);
 223         let tcx = self.tcx;
 224
 225         // Cannot inline generators which haven't been transformed yet
 226         if callee_body.yield_ty.is_some() {
 227             debug!("    yield ty present - not inlining");
 228             return false;
 229         }
 230
 231         let codegen_fn_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id());
 232
 233         let self_features = &self.codegen_fn_attrs.target_features;
 234         let callee_features = &codegen_fn_attrs.target_features;
 235         if callee_features.iter().any(|feature| !self_features.contains(feature)) {
 236             debug!("`callee has extra target features - not inlining");
 237             return false;
 238         }
 239
 240         let self_no_sanitize =
 241             self.codegen_fn_attrs.no_sanitize & self.tcx.sess.opts.debugging_opts.sanitizer;
 242         let callee_no_sanitize =
 243             codegen_fn_attrs.no_sanitize & self.tcx.sess.opts.debugging_opts.sanitizer;
 244         if self_no_sanitize != callee_no_sanitize {
 245             debug!("`callee has incompatible no_sanitize attribute - not inlining");
 246             return false;
 247         }
 248
 249         let hinted = match codegen_fn_attrs.inline {
 250             // Just treat inline(always) as a hint for now,
 251             // there are cases that prevent inlining that we
 252             // need to check for first.
 253             attr::InlineAttr::Always => true,
 254             attr::InlineAttr::Never => {
 255                 debug!("`#[inline(never)]` present - not inlining");
 256                 return false;
 257             }
 258             attr::InlineAttr::Hint => true,
 259             attr::InlineAttr::None => false,
 260         };
 261
 262         // Only inline local functions if they would be eligible for cross-crate
 263         // inlining. This is to ensure that the final crate doesn't have MIR that
 264         // reference unexported symbols
 265         if callsite.callee.def_id().is_local() {
 266             if callsite.callee.substs.non_erasable_generics().count() == 0 && !hinted {
 267                 debug!("    callee is an exported function - not inlining");
 268                 return false;
 269             }
 270         }
 271
 272         let mut threshold = if hinted { HINT_THRESHOLD } else { DEFAULT_THRESHOLD };
 273
 274         // Significantly lower the threshold for inlining cold functions
 275         if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) {
 276             threshold /= 5;
 277         }
 278
 279         // Give a bonus functions with a small number of blocks,
 280         // We normally have two or three blocks for even
 281         // very small functions.
 282         if callee_body.basic_blocks().len() <= 3 {
 283             threshold += threshold / 4;
 284         }
 285         debug!("    final inline threshold = {}", threshold);
 286
 287         // FIXME: Give a bonus to functions with only a single caller
 288         let mut first_block = true;
 289         let mut cost = 0;
 290
 291         // Traverse the MIR manually so we can account for the effects of
 292         // inlining on the CFG.
 293         let mut work_list = vec![START_BLOCK];
 294         let mut visited = BitSet::new_empty(callee_body.basic_blocks().len());
 295         while let Some(bb) = work_list.pop() {
 296             if !visited.insert(bb.index()) {
 297                 continue;
 298             }
 299             let blk = &callee_body.basic_blocks()[bb];
 300
 301             for stmt in &blk.statements {
 302                 // Don't count StorageLive/StorageDead in the inlining cost.
 303                 match stmt.kind {
 304                     StatementKind::StorageLive(_)
 305                     | StatementKind::StorageDead(_)
 306                     | StatementKind::Nop => {}
 307                     _ => cost += INSTR_COST,
 308                 }
 309             }
 310             let term = blk.terminator();
 311             let mut is_drop = false;
 312             match term.kind {
 313                 TerminatorKind::Drop { ref place, target, unwind }
 314                 | TerminatorKind::DropAndReplace { ref place, target, unwind, .. } => {
 315                     is_drop = true;
 316                     work_list.push(target);
 317                     // If the place doesn't actually need dropping, treat it like
 318                     // a regular goto.
 319                     let ty = place.ty(callee_body, tcx).subst(tcx, callsite.callee.substs).ty;
 320                     if ty.needs_drop(tcx, self.param_env) {
 321                         cost += CALL_PENALTY;
 322                         if let Some(unwind) = unwind {
 323                             cost += LANDINGPAD_PENALTY;
 324                             work_list.push(unwind);
 325                         }
 326                     } else {
 327                         cost += INSTR_COST;
 328                     }
 329                 }
 330
 331                 TerminatorKind::Unreachable | TerminatorKind::Call { destination: None, .. }
 332                     if first_block =>
 333                 {
 334                     // If the function always diverges, don't inline
 335                     // unless the cost is zero
 336                     threshold = 0;
 337                 }
 338
 339                 TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => {
 340                     if let ty::FnDef(def_id, _) = *f.literal.ty.kind() {
 341                         // Don't give intrinsics the extra penalty for calls
 342                         let f = tcx.fn_sig(def_id);
 343                         if f.abi() == Abi::RustIntrinsic || f.abi() == Abi::PlatformIntrinsic {
 344                             cost += INSTR_COST;
 345                         } else {
 346                             cost += CALL_PENALTY;
 347                         }
 348                     } else {
 349                         cost += CALL_PENALTY;
 350                     }
 351                     if cleanup.is_some() {
 352                         cost += LANDINGPAD_PENALTY;
 353                     }
 354                 }
 355                 TerminatorKind::Assert { cleanup, .. } => {
 356                     cost += CALL_PENALTY;
 357
 358                     if cleanup.is_some() {
 359                         cost += LANDINGPAD_PENALTY;
 360                     }
 361                 }
 362                 TerminatorKind::Resume => cost += RESUME_PENALTY,
 363                 _ => cost += INSTR_COST,
 364             }
 365
 366             if !is_drop {
 367                 for &succ in term.successors() {
 368                     work_list.push(succ);
 369                 }
 370             }
 371
 372             first_block = false;
 373         }
 374
 375         // Count up the cost of local variables and temps, if we know the size
 376         // use that, otherwise we use a moderately-large dummy cost.
 377
 378         let ptr_size = tcx.data_layout.pointer_size.bytes();
 379
 380         for v in callee_body.vars_and_temps_iter() {
 381             let v = &callee_body.local_decls[v];
 382             let ty = v.ty.subst(tcx, callsite.callee.substs);
 383             // Cost of the var is the size in machine-words, if we know
 384             // it.
 385             if let Some(size) = type_size_of(tcx, self.param_env, ty) {
 386                 cost += (size / ptr_size) as usize;
 387             } else {
 388                 cost += UNKNOWN_SIZE_COST;
 389             }
 390         }
 391
 392         if let attr::InlineAttr::Always = codegen_fn_attrs.inline {
 393             debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost);
 394             true
 395         } else {
 396             if cost <= threshold {
 397                 debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold);
 398                 true
 399             } else {
 400                 debug!("NOT inlining {:?} [cost={} > threshold={}]", callsite, cost, threshold);
 401                 false
 402             }
 403         }
 404     }
 405
 406     fn inline_call(
 407         &self,
 408         callsite: CallSite<'tcx>,
 409         caller_body: &mut Body<'tcx>,
 410         mut callee_body: Body<'tcx>,
 411     ) -> bool {
 412         let terminator = caller_body[callsite.bb].terminator.take().unwrap();
 413         match terminator.kind {
 414             // FIXME: Handle inlining of diverging calls
 415             TerminatorKind::Call { args, destination: Some(destination), cleanup, .. } => {
 416                 debug!("inlined {:?} into {:?}", callsite.callee, caller_body.source);
 417
 418                 // If the call is something like `a[*i] = f(i)`, where
 419                 // `i : &mut usize`, then just duplicating the `a[*i]`
 420                 // Place could result in two different locations if `f`
 421                 // writes to `i`. To prevent this we need to create a temporary
 422                 // borrow of the place and pass the destination as `*temp` instead.
 423                 fn dest_needs_borrow(place: Place<'_>) -> bool {
 424                     for elem in place.projection.iter() {
 425                         match elem {
 426                             ProjectionElem::Deref | ProjectionElem::Index(_) => return true,
 427                             _ => {}
 428                         }
 429                     }
 430
 431                     false
 432                 }
 433
 434                 let dest = if dest_needs_borrow(destination.0) {
 435                     debug!("creating temp for return destination");
 436                     let dest = Rvalue::Ref(
 437                         self.tcx.lifetimes.re_erased,
 438                         BorrowKind::Mut { allow_two_phase_borrow: false },
 439                         destination.0,
 440                     );
 441
 442                     let ty = dest.ty(caller_body, self.tcx);
 443
 444                     let temp = LocalDecl::new(ty, callsite.source_info.span);
 445
 446                     let tmp = caller_body.local_decls.push(temp);
 447                     let tmp = Place::from(tmp);
 448
 449                     let stmt = Statement {
 450                         source_info: callsite.source_info,
 451                         kind: StatementKind::Assign(box (tmp, dest)),
 452                     };
 453                     caller_body[callsite.bb].statements.push(stmt);
 454                     self.tcx.mk_place_deref(tmp)
 455                 } else {
 456                     destination.0
 457                 };
 458
 459                 let return_block = destination.1;
 460
 461                 // Copy the arguments if needed.
 462                 let args: Vec<_> = self.make_call_args(args, &callsite, caller_body, return_block);
 463
 464                 let mut integrator = Integrator {
 465                     args: &args,
 466                     new_locals: Local::new(caller_body.local_decls.len())..,
 467                     new_scopes: SourceScope::new(caller_body.source_scopes.len())..,
 468                     new_blocks: BasicBlock::new(caller_body.basic_blocks().len())..,
 469                     destination: dest,
 470                     return_block,
 471                     cleanup_block: cleanup,
 472                     in_cleanup_block: false,
 473                     tcx: self.tcx,
 474                 };
 475
 476                 // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones
 477                 // (or existing ones, in a few special cases) in the caller.
 478                 integrator.visit_body(&mut callee_body);
 479
 480                 for scope in &mut callee_body.source_scopes {
 481                     // FIXME(eddyb) move this into a `fn visit_scope_data` in `Integrator`.
 482                     if scope.parent_scope.is_none() {
 483                         let callsite_scope = &caller_body.source_scopes[callsite.source_info.scope];
 484
 485                         // Attach the outermost callee scope as a child of the callsite
 486                         // scope, via the `parent_scope` and `inlined_parent_scope` chains.
 487                         scope.parent_scope = Some(callsite.source_info.scope);
 488                         assert_eq!(scope.inlined_parent_scope, None);
 489                         scope.inlined_parent_scope = if callsite_scope.inlined.is_some() {
 490                             Some(callsite.source_info.scope)
 491                         } else {
 492                             callsite_scope.inlined_parent_scope
 493                         };
 494
 495                         // Mark the outermost callee scope as an inlined one.
 496                         assert_eq!(scope.inlined, None);
 497                         scope.inlined = Some((callsite.callee, callsite.source_info.span));
 498                     } else if scope.inlined_parent_scope.is_none() {
 499                         // Make it easy to find the scope with `inlined` set above.
 500                         scope.inlined_parent_scope =
 501                             Some(integrator.map_scope(OUTERMOST_SOURCE_SCOPE));
 502                     }
 503                 }
 504
 505                 // Insert all of the (mapped) parts of the callee body into the caller.
 506                 caller_body.local_decls.extend(
 507                     // FIXME(eddyb) make `Range<Local>` iterable so that we can use
 508                     // `callee_body.local_decls.drain(callee_body.vars_and_temps())`
 509                     callee_body
 510                         .vars_and_temps_iter()
 511                         .map(|local| callee_body.local_decls[local].clone()),
 512                 );
 513                 caller_body.source_scopes.extend(callee_body.source_scopes.drain(..));
 514                 caller_body.var_debug_info.extend(callee_body.var_debug_info.drain(..));
 515                 caller_body.basic_blocks_mut().extend(callee_body.basic_blocks_mut().drain(..));
 516
 517                 caller_body[callsite.bb].terminator = Some(Terminator {
 518                     source_info: callsite.source_info,
 519                     kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) },
 520                 });
 521
 522                 true
 523             }
 524             kind => {
 525                 caller_body[callsite.bb].terminator =
 526                     Some(Terminator { source_info: terminator.source_info, kind });
 527                 false
 528             }
 529         }
 530     }
 531
 532     fn make_call_args(
 533         &self,
 534         args: Vec<Operand<'tcx>>,
 535         callsite: &CallSite<'tcx>,
 536         caller_body: &mut Body<'tcx>,
 537         return_block: BasicBlock,
 538     ) -> Vec<Local> {
 539         let tcx = self.tcx;
 540
 541         // There is a bit of a mismatch between the *caller* of a closure and the *callee*.
 542         // The caller provides the arguments wrapped up in a tuple:
 543         //
 544         //     tuple_tmp = (a, b, c)
 545         //     Fn::call(closure_ref, tuple_tmp)
 546         //
 547         // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`)
 548         // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has
 549         // the job of unpacking this tuple. But here, we are codegen. =) So we want to create
 550         // a vector like
 551         //
 552         //     [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2]
 553         //
 554         // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient
 555         // if we "spill" that into *another* temporary, so that we can map the argument
 556         // variable in the callee MIR directly to an argument variable on our side.
 557         // So we introduce temporaries like:
 558         //
 559         //     tmp0 = tuple_tmp.0
 560         //     tmp1 = tuple_tmp.1
 561         //     tmp2 = tuple_tmp.2
 562         //
 563         // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`.
 564         // FIXME(eddyb) make this check for `"rust-call"` ABI combined with
 565         // `callee_body.spread_arg == None`, instead of special-casing closures.
 566         if tcx.is_closure(callsite.callee.def_id()) {
 567             let mut args = args.into_iter();
 568             let self_ = self.create_temp_if_necessary(
 569                 args.next().unwrap(),
 570                 callsite,
 571                 caller_body,
 572                 return_block,
 573             );
 574             let tuple = self.create_temp_if_necessary(
 575                 args.next().unwrap(),
 576                 callsite,
 577                 caller_body,
 578                 return_block,
 579             );
 580             assert!(args.next().is_none());
 581
 582             let tuple = Place::from(tuple);
 583             let tuple_tys = if let ty::Tuple(s) = tuple.ty(caller_body, tcx).ty.kind() {
 584                 s
 585             } else {
 586                 bug!("Closure arguments are not passed as a tuple");
 587             };
 588
 589             // The `closure_ref` in our example above.
 590             let closure_ref_arg = iter::once(self_);
 591
 592             // The `tmp0`, `tmp1`, and `tmp2` in our example abonve.
 593             let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| {
 594                 // This is e.g., `tuple_tmp.0` in our example above.
 595                 let tuple_field =
 596                     Operand::Move(tcx.mk_place_field(tuple, Field::new(i), ty.expect_ty()));
 597
 598                 // Spill to a local to make e.g., `tmp0`.
 599                 self.create_temp_if_necessary(tuple_field, callsite, caller_body, return_block)
 600             });
 601
 602             closure_ref_arg.chain(tuple_tmp_args).collect()
 603         } else {
 604             args.into_iter()
 605                 .map(|a| self.create_temp_if_necessary(a, callsite, caller_body, return_block))
 606                 .collect()
 607         }
 608     }
 609
 610     /// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh
 611     /// temporary `T` and an instruction `T = arg`, and returns `T`.
 612     fn create_temp_if_necessary(
 613         &self,
 614         arg: Operand<'tcx>,
 615         callsite: &CallSite<'tcx>,
 616         caller_body: &mut Body<'tcx>,
 617         return_block: BasicBlock,
 618     ) -> Local {
 619         // FIXME: Analysis of the usage of the arguments to avoid
 620         // unnecessary temporaries.
 621
 622         if let Operand::Move(place) = &arg {
 623             if let Some(local) = place.as_local() {
 624                 if caller_body.local_kind(local) == LocalKind::Temp {
 625                     // Reuse the operand if it's a temporary already
 626                     return local;
 627                 }
 628             }
 629         }
 630
 631         debug!("creating temp for argument {:?}", arg);
 632         // Otherwise, create a temporary for the arg
 633         let arg = Rvalue::Use(arg);
 634
 635         let ty = arg.ty(caller_body, self.tcx);
 636
 637         let arg_tmp = LocalDecl::new(ty, callsite.source_info.span);
 638         let arg_tmp = caller_body.local_decls.push(arg_tmp);
 639
 640         caller_body[callsite.bb].statements.push(Statement {
 641             source_info: callsite.source_info,
 642             kind: StatementKind::StorageLive(arg_tmp),
 643         });
 644         caller_body[callsite.bb].statements.push(Statement {
 645             source_info: callsite.source_info,
 646             kind: StatementKind::Assign(box (Place::from(arg_tmp), arg)),
 647         });
 648         caller_body[return_block].statements.insert(
 649             0,
 650             Statement {
 651                 source_info: callsite.source_info,
 652                 kind: StatementKind::StorageDead(arg_tmp),
 653             },
 654         );
 655
 656         arg_tmp
 657     }
 658 }
 659
 660 fn type_size_of<'tcx>(
 661     tcx: TyCtxt<'tcx>,
 662     param_env: ty::ParamEnv<'tcx>,
 663     ty: Ty<'tcx>,
 664 ) -> Option<u64> {
 665     tcx.layout_of(param_env.and(ty)).ok().map(|layout| layout.size.bytes())
 666 }
 667
 668 /**
 669  * Integrator.
 670  *
 671  * Integrates blocks from the callee function into the calling function.
 672  * Updates block indices, references to locals and other control flow
 673  * stuff.
 674 */
 675 struct Integrator<'a, 'tcx> {
 676     args: &'a [Local],
 677     new_locals: RangeFrom<Local>,
 678     new_scopes: RangeFrom<SourceScope>,
 679     new_blocks: RangeFrom<BasicBlock>,
 680     destination: Place<'tcx>,
 681     return_block: BasicBlock,
 682     cleanup_block: Option<BasicBlock>,
 683     in_cleanup_block: bool,
 684     tcx: TyCtxt<'tcx>,
 685 }
 686
 687 impl<'a, 'tcx> Integrator<'a, 'tcx> {
 688     fn map_local(&self, local: Local) -> Local {
 689         let new = if local == RETURN_PLACE {
 690             self.destination.local
 691         } else {
 692             let idx = local.index() - 1;
 693             if idx < self.args.len() {
 694                 self.args[idx]
 695             } else {
 696                 Local::new(self.new_locals.start.index() + (idx - self.args.len()))
 697             }
 698         };
 699         debug!("mapping local `{:?}` to `{:?}`", local, new);
 700         new
 701     }
 702
 703     fn map_scope(&self, scope: SourceScope) -> SourceScope {
 704         let new = SourceScope::new(self.new_scopes.start.index() + scope.index());
 705         debug!("mapping scope `{:?}` to `{:?}`", scope, new);
 706         new
 707     }
 708
 709     fn map_block(&self, block: BasicBlock) -> BasicBlock {
 710         let new = BasicBlock::new(self.new_blocks.start.index() + block.index());
 711         debug!("mapping block `{:?}` to `{:?}`", block, new);
 712         new
 713     }
 714 }
 715
 716 impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> {
 717     fn tcx(&self) -> TyCtxt<'tcx> {
 718         self.tcx
 719     }
 720
 721     fn visit_local(&mut self, local: &mut Local, _ctxt: PlaceContext, _location: Location) {
 722         *local = self.map_local(*local);
 723     }
 724
 725     fn visit_source_scope(&mut self, scope: &mut SourceScope) {
 726         *scope = self.map_scope(*scope);
 727     }
 728
 729     fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) {
 730         // If this is the `RETURN_PLACE`, we need to rebase any projections onto it.
 731         let dest_proj_len = self.destination.projection.len();
 732         if place.local == RETURN_PLACE && dest_proj_len > 0 {
 733             let mut projs = Vec::with_capacity(dest_proj_len + place.projection.len());
 734             projs.extend(self.destination.projection);
 735             projs.extend(place.projection);
 736
 737             place.projection = self.tcx.intern_place_elems(&*projs);
 738         }
 739         // Handles integrating any locals that occur in the base
 740         // or projections
 741         self.super_place(place, context, location)
 742     }
 743
 744     fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) {
 745         self.in_cleanup_block = data.is_cleanup;
 746         self.super_basic_block_data(block, data);
 747         self.in_cleanup_block = false;
 748     }
 749
 750     fn visit_retag(&mut self, kind: &mut RetagKind, place: &mut Place<'tcx>, loc: Location) {
 751         self.super_retag(kind, place, loc);
 752
 753         // We have to patch all inlined retags to be aware that they are no longer
 754         // happening on function entry.
 755         if *kind == RetagKind::FnEntry {
 756             *kind = RetagKind::Default;
 757         }
 758     }
 759
 760     fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, loc: Location) {
 761         // Don't try to modify the implicit `_0` access on return (`return` terminators are
 762         // replaced down below anyways).
 763         if !matches!(terminator.kind, TerminatorKind::Return) {
 764             self.super_terminator(terminator, loc);
 765         }
 766
 767         match terminator.kind {
 768             TerminatorKind::GeneratorDrop | TerminatorKind::Yield { .. } => bug!(),
 769             TerminatorKind::Goto { ref mut target } => {
 770                 *target = self.map_block(*target);
 771             }
 772             TerminatorKind::SwitchInt { ref mut targets, .. } => {
 773                 for tgt in targets.all_targets_mut() {
 774                     *tgt = self.map_block(*tgt);
 775                 }
 776             }
 777             TerminatorKind::Drop { ref mut target, ref mut unwind, .. }
 778             | TerminatorKind::DropAndReplace { ref mut target, ref mut unwind, .. } => {
 779                 *target = self.map_block(*target);
 780                 if let Some(tgt) = *unwind {
 781                     *unwind = Some(self.map_block(tgt));
 782                 } else if !self.in_cleanup_block {
 783                     // Unless this drop is in a cleanup block, add an unwind edge to
 784                     // the original call's cleanup block
 785                     *unwind = self.cleanup_block;
 786                 }
 787             }
 788             TerminatorKind::Call { ref mut destination, ref mut cleanup, .. } => {
 789                 if let Some((_, ref mut tgt)) = *destination {
 790                     *tgt = self.map_block(*tgt);
 791                 }
 792                 if let Some(tgt) = *cleanup {
 793                     *cleanup = Some(self.map_block(tgt));
 794                 } else if !self.in_cleanup_block {
 795                     // Unless this call is in a cleanup block, add an unwind edge to
 796                     // the original call's cleanup block
 797                     *cleanup = self.cleanup_block;
 798                 }
 799             }
 800             TerminatorKind::Assert { ref mut target, ref mut cleanup, .. } => {
 801                 *target = self.map_block(*target);
 802                 if let Some(tgt) = *cleanup {
 803                     *cleanup = Some(self.map_block(tgt));
 804                 } else if !self.in_cleanup_block {
 805                     // Unless this assert is in a cleanup block, add an unwind edge to
 806                     // the original call's cleanup block
 807                     *cleanup = self.cleanup_block;
 808                 }
 809             }
 810             TerminatorKind::Return => {
 811                 terminator.kind = TerminatorKind::Goto { target: self.return_block };
 812             }
 813             TerminatorKind::Resume => {
 814                 if let Some(tgt) = self.cleanup_block {
 815                     terminator.kind = TerminatorKind::Goto { target: tgt }
 816                 }
 817             }
 818             TerminatorKind::Abort => {}
 819             TerminatorKind::Unreachable => {}
 820             TerminatorKind::FalseEdge { ref mut real_target, ref mut imaginary_target } => {
 821                 *real_target = self.map_block(*real_target);
 822                 *imaginary_target = self.map_block(*imaginary_target);
 823             }
 824             TerminatorKind::FalseUnwind { real_target: _, unwind: _ } =>
 825             // see the ordering of passes in the optimized_mir query.
 826             {
 827                 bug!("False unwinds should have been removed before inlining")
 828             }
 829             TerminatorKind::InlineAsm { ref mut destination, .. } => {
 830                 if let Some(ref mut tgt) = *destination {
 831                     *tgt = self.map_block(*tgt);
 832                 }
 833             }
 834         }
 835     }
 836 }