1 use crate::transform::{MirPass, MirSource};
2 use crate::util::patch::MirPatch;
3 use rustc_data_structures::fingerprint::Fingerprint;
4 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
5 use rustc_hir::lang_items;
7 use rustc_middle::ich::StableHashingContext;
8 use rustc_middle::mir::coverage::*;
9 use rustc_middle::mir::interpret::Scalar;
10 use rustc_middle::mir::{
11 self, traversal, BasicBlock, BasicBlockData, CoverageInfo, Operand, Place, SourceInfo,
12 SourceScope, StatementKind, Terminator, TerminatorKind,
15 use rustc_middle::ty::query::Providers;
16 use rustc_middle::ty::FnDef;
17 use rustc_middle::ty::TyCtxt;
18 use rustc_span::def_id::DefId;
19 use rustc_span::{Pos, Span};
21 /// Inserts call to count_code_region() as a placeholder to be replaced during code generation with
22 /// the intrinsic llvm.instrprof.increment.
23 pub struct InstrumentCoverage;
25 /// The `query` provider for `CoverageInfo`, requested by `codegen_intrinsic_call()` when
26 /// constructing the arguments for `llvm.instrprof.increment`.
27 pub(crate) fn provide(providers: &mut Providers) {
28 providers.coverageinfo = |tcx, def_id| coverageinfo_from_mir(tcx, def_id);
31 fn coverageinfo_from_mir<'tcx>(tcx: TyCtxt<'tcx>, mir_def_id: DefId) -> CoverageInfo {
32 let mir_body = tcx.optimized_mir(mir_def_id);
33 // FIXME(richkadel): The current implementation assumes the MIR for the given DefId
34 // represents a single function. Validate and/or correct if inlining (which should be disabled
35 // if -Zinstrument-coverage is enabled) and/or monomorphization invalidates these assumptions.
36 let count_code_region_fn = tcx.require_lang_item(lang_items::CountCodeRegionFnLangItem, None);
37 let coverage_counter_add_fn =
38 tcx.require_lang_item(lang_items::CoverageCounterAddFnLangItem, None);
39 let coverage_counter_subtract_fn =
40 tcx.require_lang_item(lang_items::CoverageCounterSubtractFnLangItem, None);
42 // The `num_counters` argument to `llvm.instrprof.increment` is the number of injected
43 // counters, with each counter having a counter ID from `0..num_counters-1`. MIR optimization
44 // may split and duplicate some BasicBlock sequences. Simply counting the calls may not
45 // work; but computing the num_counters by adding `1` to the highest counter_id (for a given
46 // instrumented function) is valid.
48 // `num_expressions` is the number of counter expressions added to the MIR body. Both
49 // `num_counters` and `num_expressions` are used to initialize new vectors, during backend
50 // code generate, to lookup counters and expressions by simple u32 indexes.
51 let mut num_counters: u32 = 0;
52 let mut num_expressions: u32 = 0;
54 traversal::preorder(mir_body).map(|(_, data)| data).filter_map(call_terminators)
56 if let TerminatorKind::Call { func: Operand::Constant(func), args, .. } = &terminator.kind {
57 match func.literal.ty.kind {
58 FnDef(id, _) if id == count_code_region_fn => {
60 args.get(count_code_region_args::COUNTER_ID).expect("arg found");
61 let counter_id = mir::Operand::scalar_from_const(counter_id_arg)
63 .expect("counter_id arg is u32");
64 num_counters = std::cmp::max(num_counters, counter_id + 1);
67 if id == coverage_counter_add_fn || id == coverage_counter_subtract_fn =>
69 let expression_id_arg = args
70 .get(coverage_counter_expression_args::EXPRESSION_ID)
72 let id_descending_from_max = mir::Operand::scalar_from_const(expression_id_arg)
74 .expect("expression_id arg is u32");
75 // Counter expressions are initially assigned IDs descending from `u32::MAX`, so
76 // the range of expression IDs is disjoint from the range of counter IDs. This
77 // way, both counters and expressions can be operands in other expressions.
78 let expression_index = u32::MAX - id_descending_from_max;
79 num_expressions = std::cmp::max(num_expressions, expression_index + 1);
85 CoverageInfo { num_counters, num_expressions }
88 fn call_terminators(data: &'tcx BasicBlockData<'tcx>) -> Option<&'tcx Terminator<'tcx>> {
89 let terminator = data.terminator();
90 match terminator.kind {
91 TerminatorKind::Call { .. } => Some(terminator),
96 impl<'tcx> MirPass<'tcx> for InstrumentCoverage {
97 fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, mir_body: &mut mir::Body<'tcx>) {
98 // If the InstrumentCoverage pass is called on promoted MIRs, skip them.
99 // See: https://github.com/rust-lang/rust/pull/73011#discussion_r438317601
100 if src.promoted.is_none() {
101 Instrumentor::new(tcx, src, mir_body).inject_counters();
106 /// Distinguishes the expression operators.
112 struct InjectedCall<'tcx> {
114 args: Vec<Operand<'tcx>>,
118 struct Instrumentor<'a, 'tcx> {
121 mir_body: &'a mut mir::Body<'tcx>,
122 hir_body: &'tcx rustc_hir::Body<'tcx>,
123 function_source_hash: Option<u64>,
125 num_expressions: u32,
128 impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
129 fn new(tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, mir_body: &'a mut mir::Body<'tcx>) -> Self {
130 let mir_def_id = src.def_id();
131 let hir_body = hir_body(tcx, mir_def_id);
137 function_source_hash: None,
143 /// Counter IDs start from zero and go up.
144 fn next_counter(&mut self) -> u32 {
145 assert!(self.num_counters < u32::MAX - self.num_expressions);
146 let next = self.num_counters;
147 self.num_counters += 1;
151 /// Expression IDs start from u32::MAX and go down because a CounterExpression can reference
152 /// (add or subtract counts) of both Counter regions and CounterExpression regions. The counter
153 /// expression operand IDs must be unique across both types.
154 fn next_expression(&mut self) -> u32 {
155 assert!(self.num_counters < u32::MAX - self.num_expressions);
156 let next = u32::MAX - self.num_expressions;
157 self.num_expressions += 1;
161 fn function_source_hash(&mut self) -> u64 {
162 match self.function_source_hash {
165 let hash = hash_mir_source(self.tcx, self.hir_body);
166 self.function_source_hash.replace(hash);
172 fn inject_counters(&mut self) {
173 let mir_body = &self.mir_body;
174 let body_span = self.hir_body.value.span;
175 debug!("instrumenting {:?}, span: {:?}", self.mir_def_id, body_span);
177 // FIXME(richkadel): As a first step, counters are only injected at the top of each
178 // function. The complete solution will inject counters at each conditional code branch.
179 let _ignore = mir_body;
180 let id = self.next_counter();
181 let function_source_hash = self.function_source_hash();
182 let code_region = body_span;
183 let scope = rustc_middle::mir::OUTERMOST_SOURCE_SCOPE;
184 let is_cleanup = false;
185 let next_block = rustc_middle::mir::START_BLOCK;
187 self.make_counter(id, function_source_hash, code_region),
193 // FIXME(richkadel): The next step to implement source based coverage analysis will be
194 // instrumenting branches within functions, and some regions will be counted by "counter
195 // expression". The function to inject counter expression is implemented. Replace this
196 // "fake use" with real use.
197 let fake_use = false;
201 let op = if add { Op::Add } else { Op::Subtract };
204 let code_region = body_span;
205 let scope = rustc_middle::mir::OUTERMOST_SOURCE_SCOPE;
206 let is_cleanup = false;
207 let next_block = rustc_middle::mir::START_BLOCK;
209 let id = self.next_expression();
211 self.make_expression(id, code_region, lhs, op, rhs),
222 function_source_hash: u64,
224 ) -> InjectedCall<'tcx> {
225 let inject_at = code_region.shrink_to_lo();
227 let func = function_handle(
229 self.tcx.require_lang_item(lang_items::CountCodeRegionFnLangItem, None),
233 let mut args = Vec::new();
235 use count_code_region_args::*;
236 debug_assert_eq!(FUNCTION_SOURCE_HASH, args.len());
237 args.push(self.const_u64(function_source_hash, inject_at));
239 debug_assert_eq!(COUNTER_ID, args.len());
240 args.push(self.const_u32(id, inject_at));
242 debug_assert_eq!(START_BYTE_POS, args.len());
243 args.push(self.const_u32(code_region.lo().to_u32(), inject_at));
245 debug_assert_eq!(END_BYTE_POS, args.len());
246 args.push(self.const_u32(code_region.hi().to_u32(), inject_at));
248 InjectedCall { func, args, inject_at }
258 ) -> InjectedCall<'tcx> {
259 let inject_at = code_region.shrink_to_lo();
261 let func = function_handle(
263 self.tcx.require_lang_item(
265 Op::Add => lang_items::CoverageCounterAddFnLangItem,
266 Op::Subtract => lang_items::CoverageCounterSubtractFnLangItem,
273 let mut args = Vec::new();
275 use coverage_counter_expression_args::*;
276 debug_assert_eq!(EXPRESSION_ID, args.len());
277 args.push(self.const_u32(id, inject_at));
279 debug_assert_eq!(LEFT_ID, args.len());
280 args.push(self.const_u32(lhs, inject_at));
282 debug_assert_eq!(RIGHT_ID, args.len());
283 args.push(self.const_u32(rhs, inject_at));
285 debug_assert_eq!(START_BYTE_POS, args.len());
286 args.push(self.const_u32(code_region.lo().to_u32(), inject_at));
288 debug_assert_eq!(END_BYTE_POS, args.len());
289 args.push(self.const_u32(code_region.hi().to_u32(), inject_at));
291 InjectedCall { func, args, inject_at }
296 call: InjectedCall<'tcx>,
299 next_block: BasicBlock,
301 let InjectedCall { func, args, inject_at } = call;
303 " injecting {}call to {:?}({:?}) at: {:?}, scope: {:?}",
304 if is_cleanup { "cleanup " } else { "" },
311 let mut patch = MirPatch::new(self.mir_body);
313 let temp = patch.new_temp(self.tcx.mk_unit(), inject_at);
314 let new_block = patch.new_block(placeholder_block(inject_at, scope, is_cleanup));
315 patch.patch_terminator(
317 TerminatorKind::Call {
320 // new_block will swapped with the next_block, after applying patch
321 destination: Some((Place::from(temp), new_block)),
323 from_hir_call: false,
328 patch.add_statement(new_block.start_location(), StatementKind::StorageLive(temp));
329 patch.add_statement(next_block.start_location(), StatementKind::StorageDead(temp));
331 patch.apply(self.mir_body);
333 // To insert the `new_block` in front of the first block in the counted branch (the
334 // `next_block`), just swap the indexes, leaving the rest of the graph unchanged.
335 self.mir_body.basic_blocks_mut().swap(next_block, new_block);
338 fn const_u32(&self, value: u32, span: Span) -> Operand<'tcx> {
339 Operand::const_from_scalar(self.tcx, self.tcx.types.u32, Scalar::from_u32(value), span)
342 fn const_u64(&self, value: u64, span: Span) -> Operand<'tcx> {
343 Operand::const_from_scalar(self.tcx, self.tcx.types.u64, Scalar::from_u64(value), span)
347 fn function_handle<'tcx>(tcx: TyCtxt<'tcx>, fn_def_id: DefId, span: Span) -> Operand<'tcx> {
348 let ret_ty = tcx.fn_sig(fn_def_id).output();
349 let ret_ty = ret_ty.no_bound_vars().unwrap();
350 let substs = tcx.mk_substs(::std::iter::once(ty::subst::GenericArg::from(ret_ty)));
351 Operand::function_handle(tcx, fn_def_id, substs, span)
354 fn placeholder_block(span: Span, scope: SourceScope, is_cleanup: bool) -> BasicBlockData<'tcx> {
357 terminator: Some(Terminator {
358 source_info: SourceInfo { span, scope },
359 // this gets overwritten by the counter Call
360 kind: TerminatorKind::Unreachable,
366 fn hir_body<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx rustc_hir::Body<'tcx> {
367 let hir_node = tcx.hir().get_if_local(def_id).expect("DefId is local");
368 let fn_body_id = hir::map::associated_body(hir_node).expect("HIR node is a function with body");
369 tcx.hir().body(fn_body_id)
372 fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx rustc_hir::Body<'tcx>) -> u64 {
373 let mut hcx = tcx.create_no_span_stable_hashing_context();
374 hash(&mut hcx, &hir_body.value).to_smaller_hash()
378 hcx: &mut StableHashingContext<'tcx>,
379 node: &impl HashStable<StableHashingContext<'tcx>>,
381 let mut stable_hasher = StableHasher::new();
382 node.hash_stable(hcx, &mut stable_hasher);
383 stable_hasher.finish()