use rustc_middle::mir::*;
use rustc_middle::ty::subst::Subst;
use rustc_middle::ty::{self, ConstKind, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
+use rustc_span::{hygiene::ExpnKind, ExpnData, Span};
use rustc_target::spec::abi::Abi;
use super::simplify::{remove_dead_blocks, CfgSimplifier};
return;
}
- let mut local_change;
let mut changed = false;
+ while let Some(callsite) = callsites.pop_front() {
+ debug!("checking whether to inline callsite {:?}", callsite);
- loop {
- local_change = false;
- while let Some(callsite) = callsites.pop_front() {
- debug!("checking whether to inline callsite {:?}", callsite);
-
- if let InstanceDef::Item(_) = callsite.callee.def {
- if !self.tcx.is_mir_available(callsite.callee.def_id()) {
- debug!(
- "checking whether to inline callsite {:?} - MIR unavailable",
- callsite,
- );
- continue;
- }
+ if let InstanceDef::Item(_) = callsite.callee.def {
+ if !self.tcx.is_mir_available(callsite.callee.def_id()) {
+ debug!("checking whether to inline callsite {:?} - MIR unavailable", callsite,);
+ continue;
}
+ }
- let callee_body = if let Some(callee_def_id) = callsite.callee.def_id().as_local() {
- let callee_hir_id = self.tcx.hir().local_def_id_to_hir_id(callee_def_id);
- // Avoid a cycle here by only using `instance_mir` only if we have
- // a lower `HirId` than the callee. This ensures that the callee will
- // not inline us. This trick only works without incremental compilation.
- // So don't do it if that is enabled. Also avoid inlining into generators,
- // since their `optimized_mir` is used for layout computation, which can
- // create a cycle, even when no attempt is made to inline the function
- // in the other direction.
- if !self.tcx.dep_graph.is_fully_enabled()
- && self_hir_id < callee_hir_id
- && caller_body.generator_kind.is_none()
- {
- self.tcx.instance_mir(callsite.callee.def)
- } else {
- continue;
- }
- } else {
- // This cannot result in a cycle since the callee MIR is from another crate
- // and is already optimized.
+ let callee_body = if let Some(callee_def_id) = callsite.callee.def_id().as_local() {
+ let callee_hir_id = self.tcx.hir().local_def_id_to_hir_id(callee_def_id);
+ // Avoid a cycle here by only using `instance_mir` only if we have
+ // a lower `HirId` than the callee. This ensures that the callee will
+ // not inline us. This trick only works without incremental compilation.
+ // So don't do it if that is enabled. Also avoid inlining into generators,
+ // since their `optimized_mir` is used for layout computation, which can
+ // create a cycle, even when no attempt is made to inline the function
+ // in the other direction.
+ if !self.tcx.dep_graph.is_fully_enabled()
+ && self_hir_id < callee_hir_id
+ && caller_body.generator_kind.is_none()
+ {
self.tcx.instance_mir(callsite.callee.def)
- };
-
- let callee_body: &Body<'tcx> = &*callee_body;
-
- let callee_body = if self.consider_optimizing(callsite, callee_body) {
- self.tcx.subst_and_normalize_erasing_regions(
- &callsite.callee.substs,
- self.param_env,
- callee_body,
- )
} else {
continue;
- };
+ }
+ } else {
+ // This cannot result in a cycle since the callee MIR is from another crate
+ // and is already optimized.
+ self.tcx.instance_mir(callsite.callee.def)
+ };
- // Copy only unevaluated constants from the callee_body into the caller_body.
- // Although we are only pushing `ConstKind::Unevaluated` consts to
- // `required_consts`, here we may not only have `ConstKind::Unevaluated`
- // because we are calling `subst_and_normalize_erasing_regions`.
- caller_body.required_consts.extend(
- callee_body.required_consts.iter().copied().filter(|&constant| {
- matches!(constant.literal.val, ConstKind::Unevaluated(_, _, _))
- }),
- );
+ let callee_body: &Body<'tcx> = &*callee_body;
- let start = caller_body.basic_blocks().len();
- debug!("attempting to inline callsite {:?} - body={:?}", callsite, callee_body);
- if !self.inline_call(callsite, caller_body, callee_body) {
- debug!("attempting to inline callsite {:?} - failure", callsite);
- continue;
- }
- debug!("attempting to inline callsite {:?} - success", callsite);
-
- // Add callsites from inlined function
- for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated().skip(start) {
- if let Some(new_callsite) =
- self.get_valid_function_call(bb, bb_data, caller_body)
- {
- // Don't inline the same function multiple times.
- if callsite.callee != new_callsite.callee {
- callsites.push_back(new_callsite);
- }
+ let callee_body = if self.consider_optimizing(callsite, callee_body) {
+ self.tcx.subst_and_normalize_erasing_regions(
+ &callsite.callee.substs,
+ self.param_env,
+ callee_body,
+ )
+ } else {
+ continue;
+ };
+
+ let start = caller_body.basic_blocks().len();
+ debug!("attempting to inline callsite {:?} - body={:?}", callsite, callee_body);
+ if !self.inline_call(callsite, caller_body, callee_body) {
+ debug!("attempting to inline callsite {:?} - failure", callsite);
+ continue;
+ }
+ debug!("attempting to inline callsite {:?} - success", callsite);
+
+ // Add callsites from inlined function
+ for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated().skip(start) {
+ if let Some(new_callsite) = self.get_valid_function_call(bb, bb_data, caller_body) {
+ // Don't inline the same function multiple times.
+ if callsite.callee != new_callsite.callee {
+ callsites.push_back(new_callsite);
}
}
-
- local_change = true;
- changed = true;
}
- if !local_change {
- break;
- }
+ changed = true;
}
// Simplify if we inlined anything.
cleanup_block: cleanup,
in_cleanup_block: false,
tcx: self.tcx,
+ callsite_span: callsite.source_info.span,
+ body_span: callee_body.span,
};
// Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones
kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) },
});
+ // Copy only unevaluated constants from the callee_body into the caller_body.
+ // Although we are only pushing `ConstKind::Unevaluated` consts to
+ // `required_consts`, here we may not only have `ConstKind::Unevaluated`
+ // because we are calling `subst_and_normalize_erasing_regions`.
+ caller_body.required_consts.extend(
+ callee_body.required_consts.iter().copied().filter(|&constant| {
+ matches!(constant.literal.val, ConstKind::Unevaluated(_, _, _))
+ }),
+ );
+
true
}
kind => {
cleanup_block: Option<BasicBlock>,
in_cleanup_block: bool,
tcx: TyCtxt<'tcx>,
+ callsite_span: Span,
+ body_span: Span,
}
impl<'a, 'tcx> Integrator<'a, 'tcx> {
*scope = self.map_scope(*scope);
}
+ fn visit_span(&mut self, span: &mut Span) {
+ // Make sure that all spans track the fact that they were inlined.
+ *span = self.callsite_span.fresh_expansion(ExpnData {
+ def_site: self.body_span,
+ ..ExpnData::default(ExpnKind::Inlined, *span, self.tcx.sess.edition(), None)
+ });
+ }
+
fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) {
// If this is the `RETURN_PLACE`, we need to rebase any projections onto it.
let dest_proj_len = self.destination.projection.len();
projects / rust.git / blobdiff