11 use counters::CoverageCounters;
12 use graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph};
13 use spans::{CoverageSpan, CoverageSpans};
17 use rustc_data_structures::graph::WithNumNodes;
18 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
19 use rustc_data_structures::sync::Lrc;
20 use rustc_index::vec::IndexVec;
21 use rustc_middle::hir;
22 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
23 use rustc_middle::mir::coverage::*;
24 use rustc_middle::mir::dump_enabled;
25 use rustc_middle::mir::{
26 self, BasicBlock, BasicBlockData, Coverage, SourceInfo, Statement, StatementKind, Terminator,
29 use rustc_middle::ty::TyCtxt;
30 use rustc_span::def_id::DefId;
31 use rustc_span::source_map::SourceMap;
32 use rustc_span::{CharPos, ExpnKind, Pos, SourceFile, Span, Symbol};
34 /// A simple error message wrapper for `coverage::Error`s.
41 pub fn from_string<T>(message: String) -> Result<T, Error> {
46 /// Inserts `StatementKind::Coverage` statements that either instrument the binary with injected
47 /// counters, via intrinsic `llvm.instrprof.increment`, and/or inject metadata used during codegen
48 /// to construct the coverage map.
49 pub struct InstrumentCoverage;
51 impl<'tcx> MirPass<'tcx> for InstrumentCoverage {
52 fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
53 sess.instrument_coverage()
56 fn run_pass(&self, tcx: TyCtxt<'tcx>, mir_body: &mut mir::Body<'tcx>) {
57 let mir_source = mir_body.source;
59 // If the InstrumentCoverage pass is called on promoted MIRs, skip them.
60 // See: https://github.com/rust-lang/rust/pull/73011#discussion_r438317601
61 if mir_source.promoted.is_some() {
63 "InstrumentCoverage skipped for {:?} (already promoted for Miri evaluation)",
70 tcx.hir().get_by_def_id(mir_source.def_id().expect_local()).fn_kind().is_some();
72 // Only instrument functions, methods, and closures (not constants since they are evaluated
73 // at compile time by Miri).
74 // FIXME(#73156): Handle source code coverage in const eval, but note, if and when const
75 // expressions get coverage spans, we will probably have to "carve out" space for const
76 // expressions from coverage spans in enclosing MIR's, like we do for closures. (That might
77 // be tricky if const expressions have no corresponding statements in the enclosing MIR.
78 // Closures are carved out by their initial `Assign` statement.)
80 trace!("InstrumentCoverage skipped for {:?} (not an fn-like)", mir_source.def_id());
84 match mir_body.basic_blocks()[mir::START_BLOCK].terminator().kind {
85 TerminatorKind::Unreachable => {
86 trace!("InstrumentCoverage skipped for unreachable `START_BLOCK`");
92 let codegen_fn_attrs = tcx.codegen_fn_attrs(mir_source.def_id());
93 if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) {
97 trace!("InstrumentCoverage starting for {:?}", mir_source.def_id());
98 Instrumentor::new(&self.name(), tcx, mir_body).inject_counters();
99 trace!("InstrumentCoverage done for {:?}", mir_source.def_id());
103 struct Instrumentor<'a, 'tcx> {
106 mir_body: &'a mut mir::Body<'tcx>,
107 source_file: Lrc<SourceFile>,
110 basic_coverage_blocks: CoverageGraph,
111 coverage_counters: CoverageCounters,
114 impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
115 fn new(pass_name: &'a str, tcx: TyCtxt<'tcx>, mir_body: &'a mut mir::Body<'tcx>) -> Self {
116 let source_map = tcx.sess.source_map();
117 let def_id = mir_body.source.def_id();
118 let (some_fn_sig, hir_body) = fn_sig_and_body(tcx, def_id);
120 let body_span = get_body_span(tcx, hir_body, mir_body);
122 let source_file = source_map.lookup_source_file(body_span.lo());
123 let fn_sig_span = match some_fn_sig.filter(|fn_sig| {
124 fn_sig.span.eq_ctxt(body_span)
125 && Lrc::ptr_eq(&source_file, &source_map.lookup_source_file(fn_sig.span.lo()))
127 Some(fn_sig) => fn_sig.span.with_hi(body_span.lo()),
128 None => body_span.shrink_to_lo(),
132 "instrumenting {}: {:?}, fn sig span: {:?}, body span: {:?}",
133 if tcx.is_closure(def_id) { "closure" } else { "function" },
139 let function_source_hash = hash_mir_source(tcx, hir_body);
140 let basic_coverage_blocks = CoverageGraph::from_mir(mir_body);
148 basic_coverage_blocks,
149 coverage_counters: CoverageCounters::new(function_source_hash),
153 fn inject_counters(&'a mut self) {
155 let mir_source = self.mir_body.source;
156 let def_id = mir_source.def_id();
157 let fn_sig_span = self.fn_sig_span;
158 let body_span = self.body_span;
160 let mut graphviz_data = debug::GraphvizData::new();
161 let mut debug_used_expressions = debug::UsedExpressions::new();
163 let dump_mir = dump_enabled(tcx, self.pass_name, def_id);
164 let dump_graphviz = dump_mir && tcx.sess.opts.debugging_opts.dump_mir_graphviz;
165 let dump_spanview = dump_mir && tcx.sess.opts.debugging_opts.dump_mir_spanview.is_some();
168 graphviz_data.enable();
169 self.coverage_counters.enable_debug();
172 if dump_graphviz || level_enabled!(tracing::Level::DEBUG) {
173 debug_used_expressions.enable();
176 ////////////////////////////////////////////////////
177 // Compute `CoverageSpan`s from the `CoverageGraph`.
178 let coverage_spans = CoverageSpans::generate_coverage_spans(
182 &self.basic_coverage_blocks,
186 debug::dump_coverage_spanview(
189 &self.basic_coverage_blocks,
196 ////////////////////////////////////////////////////
197 // Create an optimized mix of `Counter`s and `Expression`s for the `CoverageGraph`. Ensure
198 // every `CoverageSpan` has a `Counter` or `Expression` assigned to its `BasicCoverageBlock`
199 // and all `Expression` dependencies (operands) are also generated, for any other
200 // `BasicCoverageBlock`s not already associated with a `CoverageSpan`.
202 // Intermediate expressions (used to compute other `Expression` values), which have no
203 // direct associate to any `BasicCoverageBlock`, are returned in the method `Result`.
204 let intermediate_expressions_or_error = self
206 .make_bcb_counters(&mut self.basic_coverage_blocks, &coverage_spans);
208 let (result, intermediate_expressions) = match intermediate_expressions_or_error {
209 Ok(intermediate_expressions) => {
210 // If debugging, add any intermediate expressions (which are not associated with any
211 // BCB) to the `debug_used_expressions` map.
212 if debug_used_expressions.is_enabled() {
213 for intermediate_expression in &intermediate_expressions {
214 debug_used_expressions.add_expression_operands(intermediate_expression);
218 ////////////////////////////////////////////////////
219 // Remove the counter or edge counter from of each `CoverageSpan`s associated
220 // `BasicCoverageBlock`, and inject a `Coverage` statement into the MIR.
222 // `Coverage` statements injected from `CoverageSpan`s will include the code regions
223 // (source code start and end positions) to be counted by the associated counter.
225 // These `CoverageSpan`-associated counters are removed from their associated
226 // `BasicCoverageBlock`s so that the only remaining counters in the `CoverageGraph`
227 // are indirect counters (to be injected next, without associated code regions).
228 self.inject_coverage_span_counters(
231 &mut debug_used_expressions,
234 ////////////////////////////////////////////////////
235 // For any remaining `BasicCoverageBlock` counters (that were not associated with
236 // any `CoverageSpan`), inject `Coverage` statements (_without_ code region `Span`s)
237 // to ensure `BasicCoverageBlock` counters that other `Expression`s may depend on
238 // are in fact counted, even though they don't directly contribute to counting
239 // their own independent code region's coverage.
240 self.inject_indirect_counters(&mut graphviz_data, &mut debug_used_expressions);
242 // Intermediate expressions will be injected as the final step, after generating
243 // debug output, if any.
244 ////////////////////////////////////////////////////
246 (Ok(()), intermediate_expressions)
248 Err(e) => (Err(e), Vec::new()),
251 if graphviz_data.is_enabled() {
252 // Even if there was an error, a partial CoverageGraph can still generate a useful
254 debug::dump_coverage_graphviz(
258 &self.basic_coverage_blocks,
259 &self.coverage_counters.debug_counters,
261 &intermediate_expressions,
262 &debug_used_expressions,
266 if let Err(e) = result {
267 bug!("Error processing: {:?}: {:?}", self.mir_body.source.def_id(), e.message)
270 // Depending on current `debug_options()`, `alert_on_unused_expressions()` could panic, so
271 // this check is performed as late as possible, to allow other debug output (logs and dump
272 // files), which might be helpful in analyzing unused expressions, to still be generated.
273 debug_used_expressions.alert_on_unused_expressions(&self.coverage_counters.debug_counters);
275 ////////////////////////////////////////////////////
276 // Finally, inject the intermediate expressions collected along the way.
277 for intermediate_expression in intermediate_expressions {
278 inject_intermediate_expression(self.mir_body, intermediate_expression);
282 /// Inject a counter for each `CoverageSpan`. There can be multiple `CoverageSpan`s for a given
283 /// BCB, but only one actual counter needs to be incremented per BCB. `bb_counters` maps each
284 /// `bcb` to its `Counter`, when injected. Subsequent `CoverageSpan`s for a BCB that already has
285 /// a `Counter` will inject an `Expression` instead, and compute its value by adding `ZERO` to
286 /// the BCB `Counter` value.
288 /// If debugging, add every BCB `Expression` associated with a `CoverageSpan`s to the
289 /// `used_expression_operands` map.
290 fn inject_coverage_span_counters(
292 coverage_spans: Vec<CoverageSpan>,
293 graphviz_data: &mut debug::GraphvizData,
294 debug_used_expressions: &mut debug::UsedExpressions,
297 let source_map = tcx.sess.source_map();
298 let body_span = self.body_span;
299 let file_name = Symbol::intern(&self.source_file.name.prefer_remapped().to_string_lossy());
301 let mut bcb_counters = IndexVec::from_elem_n(None, self.basic_coverage_blocks.num_nodes());
302 for covspan in coverage_spans {
303 let bcb = covspan.bcb;
304 let span = covspan.span;
305 let counter_kind = if let Some(&counter_operand) = bcb_counters[bcb].as_ref() {
306 self.coverage_counters.make_identity_counter(counter_operand)
307 } else if let Some(counter_kind) = self.bcb_data_mut(bcb).take_counter() {
308 bcb_counters[bcb] = Some(counter_kind.as_operand_id());
309 debug_used_expressions.add_expression_operands(&counter_kind);
312 bug!("Every BasicCoverageBlock should have a Counter or Expression");
314 graphviz_data.add_bcb_coverage_span_with_counter(bcb, &covspan, &counter_kind);
317 "Calling make_code_region(file_name={}, source_file={:?}, span={}, body_span={})",
320 source_map.span_to_diagnostic_string(span),
321 source_map.span_to_diagnostic_string(body_span)
327 self.bcb_leader_bb(bcb),
328 Some(make_code_region(source_map, file_name, &self.source_file, span, body_span)),
333 /// `inject_coverage_span_counters()` looped through the `CoverageSpan`s and injected the
334 /// counter from the `CoverageSpan`s `BasicCoverageBlock`, removing it from the BCB in the
335 /// process (via `take_counter()`).
337 /// Any other counter associated with a `BasicCoverageBlock`, or its incoming edge, but not
338 /// associated with a `CoverageSpan`, should only exist if the counter is an `Expression`
339 /// dependency (one of the expression operands). Collect them, and inject the additional
340 /// counters into the MIR, without a reportable coverage span.
341 fn inject_indirect_counters(
343 graphviz_data: &mut debug::GraphvizData,
344 debug_used_expressions: &mut debug::UsedExpressions,
346 let mut bcb_counters_without_direct_coverage_spans = Vec::new();
347 for (target_bcb, target_bcb_data) in self.basic_coverage_blocks.iter_enumerated_mut() {
348 if let Some(counter_kind) = target_bcb_data.take_counter() {
349 bcb_counters_without_direct_coverage_spans.push((None, target_bcb, counter_kind));
351 if let Some(edge_counters) = target_bcb_data.take_edge_counters() {
352 for (from_bcb, counter_kind) in edge_counters {
353 bcb_counters_without_direct_coverage_spans.push((
362 // If debug is enabled, validate that every BCB or edge counter not directly associated
363 // with a coverage span is at least indirectly associated (it is a dependency of a BCB
364 // counter that _is_ associated with a coverage span).
365 debug_used_expressions.validate(&bcb_counters_without_direct_coverage_spans);
367 for (edge_from_bcb, target_bcb, counter_kind) in bcb_counters_without_direct_coverage_spans
369 debug_used_expressions.add_unused_expression_if_not_found(
376 CoverageKind::Counter { .. } => {
377 let inject_to_bb = if let Some(from_bcb) = edge_from_bcb {
378 // The MIR edge starts `from_bb` (the outgoing / last BasicBlock in
379 // `from_bcb`) and ends at `to_bb` (the incoming / first BasicBlock in the
380 // `target_bcb`; also called the `leader_bb`).
381 let from_bb = self.bcb_last_bb(from_bcb);
382 let to_bb = self.bcb_leader_bb(target_bcb);
384 let new_bb = inject_edge_counter_basic_block(self.mir_body, from_bb, to_bb);
385 graphviz_data.set_edge_counter(from_bcb, new_bb, &counter_kind);
387 "Edge {:?} (last {:?}) -> {:?} (leader {:?}) requires a new MIR \
388 BasicBlock {:?}, for unclaimed edge counter {}",
394 self.format_counter(&counter_kind),
398 let target_bb = self.bcb_last_bb(target_bcb);
399 graphviz_data.add_bcb_dependency_counter(target_bcb, &counter_kind);
401 "{:?} ({:?}) gets a new Coverage statement for unclaimed counter {}",
404 self.format_counter(&counter_kind),
409 inject_statement(self.mir_body, counter_kind, inject_to_bb, None);
411 CoverageKind::Expression { .. } => {
412 inject_intermediate_expression(self.mir_body, counter_kind)
414 _ => bug!("CoverageKind should be a counter"),
420 fn bcb_leader_bb(&self, bcb: BasicCoverageBlock) -> BasicBlock {
421 self.bcb_data(bcb).leader_bb()
425 fn bcb_last_bb(&self, bcb: BasicCoverageBlock) -> BasicBlock {
426 self.bcb_data(bcb).last_bb()
430 fn bcb_data(&self, bcb: BasicCoverageBlock) -> &BasicCoverageBlockData {
431 &self.basic_coverage_blocks[bcb]
435 fn bcb_data_mut(&mut self, bcb: BasicCoverageBlock) -> &mut BasicCoverageBlockData {
436 &mut self.basic_coverage_blocks[bcb]
440 fn format_counter(&self, counter_kind: &CoverageKind) -> String {
441 self.coverage_counters.debug_counters.format_counter(counter_kind)
445 fn inject_edge_counter_basic_block(
446 mir_body: &mut mir::Body<'_>,
450 let span = mir_body[from_bb].terminator().source_info.span.shrink_to_hi();
451 let new_bb = mir_body.basic_blocks_mut().push(BasicBlockData {
452 statements: vec![], // counter will be injected here
453 terminator: Some(Terminator {
454 source_info: SourceInfo::outermost(span),
455 kind: TerminatorKind::Goto { target: to_bb },
459 let edge_ref = mir_body[from_bb]
462 .find(|successor| **successor == to_bb)
463 .expect("from_bb should have a successor for to_bb");
469 mir_body: &mut mir::Body<'_>,
470 counter_kind: CoverageKind,
472 some_code_region: Option<CodeRegion>,
475 " injecting statement {:?} for {:?} at code region: {:?}",
476 counter_kind, bb, some_code_region
478 let data = &mut mir_body[bb];
479 let source_info = data.terminator().source_info;
480 let statement = Statement {
482 kind: StatementKind::Coverage(Box::new(Coverage {
484 code_region: some_code_region,
487 data.statements.insert(0, statement);
490 // Non-code expressions are injected into the coverage map, without generating executable code.
491 fn inject_intermediate_expression(mir_body: &mut mir::Body<'_>, expression: CoverageKind) {
492 debug_assert!(matches!(expression, CoverageKind::Expression { .. }));
493 debug!(" injecting non-code expression {:?}", expression);
494 let inject_in_bb = mir::START_BLOCK;
495 let data = &mut mir_body[inject_in_bb];
496 let source_info = data.terminator().source_info;
497 let statement = Statement {
499 kind: StatementKind::Coverage(Box::new(Coverage { kind: expression, code_region: None })),
501 data.statements.push(statement);
504 /// Convert the Span into its file name, start line and column, and end line and column
506 source_map: &SourceMap,
508 source_file: &Lrc<SourceFile>,
512 let (start_line, mut start_col) = source_file.lookup_file_pos(span.lo());
513 let (end_line, end_col) = if span.hi() == span.lo() {
514 let (end_line, mut end_col) = (start_line, start_col);
515 // Extend an empty span by one character so the region will be counted.
516 let CharPos(char_pos) = start_col;
517 if span.hi() == body_span.hi() {
518 start_col = CharPos(char_pos - 1);
520 end_col = CharPos(char_pos + 1);
524 source_file.lookup_file_pos(span.hi())
526 let start_line = source_map.doctest_offset_line(&source_file.name, start_line);
527 let end_line = source_map.doctest_offset_line(&source_file.name, end_line);
530 start_line: start_line as u32,
531 start_col: start_col.to_u32() + 1,
532 end_line: end_line as u32,
533 end_col: end_col.to_u32() + 1,
537 fn fn_sig_and_body<'tcx>(
540 ) -> (Option<&'tcx rustc_hir::FnSig<'tcx>>, &'tcx rustc_hir::Body<'tcx>) {
541 // FIXME(#79625): Consider improving MIR to provide the information needed, to avoid going back
543 let hir_node = tcx.hir().get_if_local(def_id).expect("expected DefId is local");
544 let fn_body_id = hir::map::associated_body(hir_node).expect("HIR node is a function with body");
545 (hir::map::fn_sig(hir_node), tcx.hir().body(fn_body_id))
548 fn get_body_span<'tcx>(
550 hir_body: &rustc_hir::Body<'tcx>,
551 mir_body: &mut mir::Body<'tcx>,
553 let mut body_span = hir_body.value.span;
554 let def_id = mir_body.source.def_id();
556 if tcx.is_closure(def_id) {
557 // If the MIR function is a closure, and if the closure body span
558 // starts from a macro, but it's content is not in that macro, try
559 // to find a non-macro callsite, and instrument the spans there
562 let expn_data = body_span.ctxt().outer_expn_data();
563 if expn_data.is_root() {
566 if let ExpnKind::Macro { .. } = expn_data.kind {
567 body_span = expn_data.call_site;
577 fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx rustc_hir::Body<'tcx>) -> u64 {
578 // FIXME(cjgillot) Stop hashing HIR manually here.
579 let mut hcx = tcx.create_no_span_stable_hashing_context();
580 let mut stable_hasher = StableHasher::new();
581 let owner = hir_body.id().hir_id.owner;
582 let bodies = &tcx.hir_owner_nodes(owner).unwrap().bodies;
583 hcx.with_hir_bodies(false, owner, bodies, |hcx| {
584 hir_body.value.hash_stable(hcx, &mut stable_hasher)
586 stable_hasher.finish()