compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

   1 use crate::llvm;
   2
   3 use crate::abi::{Abi, FnAbi};
   4 use crate::builder::Builder;
   5 use crate::common::CodegenCx;
   6
   7 use libc::c_uint;
   8 use llvm::coverageinfo::CounterMappingRegion;
   9 use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage};
  10 use rustc_codegen_ssa::traits::{
  11     BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods,
  12     MiscMethods, StaticMethods,
  13 };
  14 use rustc_data_structures::fx::FxHashMap;
  15 use rustc_hir as hir;
  16 use rustc_hir::def_id::DefId;
  17 use rustc_llvm::RustString;
  18 use rustc_middle::bug;
  19 use rustc_middle::mir::coverage::{
  20     CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op,
  21 };
  22 use rustc_middle::ty;
  23 use rustc_middle::ty::layout::FnAbiExt;
  24 use rustc_middle::ty::subst::InternalSubsts;
  25 use rustc_middle::ty::Instance;
  26
  27 use std::cell::RefCell;
  28 use std::ffi::CString;
  29
  30 use std::iter;
  31 use tracing::debug;
  32
  33 pub mod mapgen;
  34
  35 const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START;
  36
  37 const VAR_ALIGN_BYTES: usize = 8;
  38
  39 /// A context object for maintaining all state needed by the coverageinfo module.
  40 pub struct CrateCoverageContext<'ll, 'tcx> {
  41     // Coverage data for each instrumented function identified by DefId.
  42     pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>,
  43     pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>,
  44 }
  45
  46 impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> {
  47     pub fn new() -> Self {
  48         Self {
  49             function_coverage_map: Default::default(),
  50             pgo_func_name_var_map: Default::default(),
  51         }
  52     }
  53
  54     pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> {
  55         self.function_coverage_map.replace(FxHashMap::default())
  56     }
  57 }
  58
  59 impl CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
  60     fn coverageinfo_finalize(&self) {
  61         mapgen::finalize(self)
  62     }
  63
  64     fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value {
  65         if let Some(coverage_context) = self.coverage_context() {
  66             debug!("getting pgo_func_name_var for instance={:?}", instance);
  67             let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut();
  68             pgo_func_name_var_map
  69                 .entry(instance)
  70                 .or_insert_with(|| create_pgo_func_name_var(self, instance))
  71         } else {
  72             bug!("Could not get the `coverage_context`");
  73         }
  74     }
  75
  76     /// Functions with MIR-based coverage are normally codegenned _only_ if
  77     /// called. LLVM coverage tools typically expect every function to be
  78     /// defined (even if unused), with at least one call to LLVM intrinsic
  79     /// `instrprof.increment`.
  80     ///
  81     /// Codegen a small function that will never be called, with one counter
  82     /// that will never be incremented.
  83     ///
  84     /// For used/called functions, the coverageinfo was already added to the
  85     /// `function_coverage_map` (keyed by function `Instance`) during codegen.
  86     /// But in this case, since the unused function was _not_ previously
  87     /// codegenned, collect the coverage `CodeRegion`s from the MIR and add
  88     /// them. The first `CodeRegion` is used to add a single counter, with the
  89     /// same counter ID used in the injected `instrprof.increment` intrinsic
  90     /// call. Since the function is never called, all other `CodeRegion`s can be
  91     /// added as `unreachable_region`s.
  92     fn define_unused_fn(&self, def_id: DefId) {
  93         let instance = declare_unused_fn(self, &def_id);
  94         codegen_unused_fn_and_counter(self, instance);
  95         add_unused_function_coverage(self, instance, def_id);
  96     }
  97 }
  98
  99 impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
 100     fn set_function_source_hash(
 101         &mut self,
 102         instance: Instance<'tcx>,
 103         function_source_hash: u64,
 104     ) -> bool {
 105         if let Some(coverage_context) = self.coverage_context() {
 106             debug!(
 107                 "ensuring function source hash is set for instance={:?}; function_source_hash={}",
 108                 instance, function_source_hash,
 109             );
 110             let mut coverage_map = coverage_context.function_coverage_map.borrow_mut();
 111             coverage_map
 112                 .entry(instance)
 113                 .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
 114                 .set_function_source_hash(function_source_hash);
 115             true
 116         } else {
 117             false
 118         }
 119     }
 120
 121     fn add_coverage_counter(
 122         &mut self,
 123         instance: Instance<'tcx>,
 124         id: CounterValueReference,
 125         region: CodeRegion,
 126     ) -> bool {
 127         if let Some(coverage_context) = self.coverage_context() {
 128             debug!(
 129                 "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}",
 130                 instance, id, region,
 131             );
 132             let mut coverage_map = coverage_context.function_coverage_map.borrow_mut();
 133             coverage_map
 134                 .entry(instance)
 135                 .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
 136                 .add_counter(id, region);
 137             true
 138         } else {
 139             false
 140         }
 141     }
 142
 143     fn add_coverage_counter_expression(
 144         &mut self,
 145         instance: Instance<'tcx>,
 146         id: InjectedExpressionId,
 147         lhs: ExpressionOperandId,
 148         op: Op,
 149         rhs: ExpressionOperandId,
 150         region: Option<CodeRegion>,
 151     ) -> bool {
 152         if let Some(coverage_context) = self.coverage_context() {
 153             debug!(
 154                 "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; \
 155                 region: {:?}",
 156                 instance, id, lhs, op, rhs, region,
 157             );
 158             let mut coverage_map = coverage_context.function_coverage_map.borrow_mut();
 159             coverage_map
 160                 .entry(instance)
 161                 .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
 162                 .add_counter_expression(id, lhs, op, rhs, region);
 163             true
 164         } else {
 165             false
 166         }
 167     }
 168
 169     fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool {
 170         if let Some(coverage_context) = self.coverage_context() {
 171             debug!(
 172                 "adding unreachable code to coverage_map: instance={:?}, at {:?}",
 173                 instance, region,
 174             );
 175             let mut coverage_map = coverage_context.function_coverage_map.borrow_mut();
 176             coverage_map
 177                 .entry(instance)
 178                 .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
 179                 .add_unreachable_region(region);
 180             true
 181         } else {
 182             false
 183         }
 184     }
 185 }
 186
 187 fn declare_unused_fn(cx: &CodegenCx<'ll, 'tcx>, def_id: &DefId) -> Instance<'tcx> {
 188     let tcx = cx.tcx;
 189
 190     let instance = Instance::new(
 191         *def_id,
 192         InternalSubsts::for_item(tcx, *def_id, |param, _| {
 193             if let ty::GenericParamDefKind::Lifetime = param.kind {
 194                 tcx.lifetimes.re_erased.into()
 195             } else {
 196                 tcx.mk_param_from_def(param)
 197             }
 198         }),
 199     );
 200
 201     let llfn = cx.declare_fn(
 202         &tcx.symbol_name(instance).name,
 203         &FnAbi::of_fn_ptr(
 204             cx,
 205             ty::Binder::dummy(tcx.mk_fn_sig(
 206                 iter::once(tcx.mk_unit()),
 207                 tcx.mk_unit(),
 208                 false,
 209                 hir::Unsafety::Unsafe,
 210                 Abi::Rust,
 211             )),
 212             &[],
 213         ),
 214     );
 215
 216     llvm::set_linkage(llfn, llvm::Linkage::WeakAnyLinkage);
 217     llvm::set_visibility(llfn, llvm::Visibility::Hidden);
 218
 219     assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none());
 220
 221     instance
 222 }
 223
 224 fn codegen_unused_fn_and_counter(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) {
 225     let llfn = cx.get_fn(instance);
 226     let mut bx = Builder::new_block(cx, llfn, "unused_function");
 227     let fn_name = bx.get_pgo_func_name_var(instance);
 228     let hash = bx.const_u64(0);
 229     let num_counters = bx.const_u32(1);
 230     let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID));
 231     debug!(
 232         "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?},
 233             index={:?}) for unused function: {:?}",
 234         fn_name, hash, num_counters, index, instance
 235     );
 236     bx.instrprof_increment(fn_name, hash, num_counters, index);
 237     bx.ret_void();
 238 }
 239
 240 fn add_unused_function_coverage(
 241     cx: &CodegenCx<'ll, 'tcx>,
 242     instance: Instance<'tcx>,
 243     def_id: DefId,
 244 ) {
 245     let tcx = cx.tcx;
 246
 247     let mut function_coverage = FunctionCoverage::unused(tcx, instance);
 248     for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() {
 249         if index == 0 {
 250             // Insert at least one real counter so the LLVM CoverageMappingReader will find expected
 251             // definitions.
 252             function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone());
 253         }
 254         // Add a Zero Counter for every code region.
 255         //
 256         // Even though the first coverage region already has an actual Counter, `llvm-cov` will not
 257         // always report it. Re-adding an unreachable region (zero counter) for the same region
 258         // seems to help produce the expected coverage.
 259         function_coverage.add_unreachable_region(code_region.clone());
 260     }
 261
 262     if let Some(coverage_context) = cx.coverage_context() {
 263         coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage);
 264     } else {
 265         bug!("Could not get the `coverage_context`");
 266     }
 267 }
 268
 269 /// Calls llvm::createPGOFuncNameVar() with the given function instance's
 270 /// mangled function name. The LLVM API returns an llvm::GlobalVariable
 271 /// containing the function name, with the specific variable name and linkage
 272 /// required by LLVM InstrProf source-based coverage instrumentation. Use
 273 /// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per
 274 /// `Instance`.
 275 fn create_pgo_func_name_var(
 276     cx: &CodegenCx<'ll, 'tcx>,
 277     instance: Instance<'tcx>,
 278 ) -> &'ll llvm::Value {
 279     let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name)
 280         .expect("error converting function name to C string");
 281     let llfn = cx.get_fn(instance);
 282     unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) }
 283 }
 284
 285 pub(crate) fn write_filenames_section_to_buffer<'a>(
 286     filenames: impl IntoIterator<Item = &'a CString>,
 287     buffer: &RustString,
 288 ) {
 289     let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::<Vec<_>>();
 290     unsafe {
 291         llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer(
 292             c_str_vec.as_ptr(),
 293             c_str_vec.len(),
 294             buffer,
 295         );
 296     }
 297 }
 298
 299 pub(crate) fn write_mapping_to_buffer(
 300     virtual_file_mapping: Vec<u32>,
 301     expressions: Vec<CounterExpression>,
 302     mapping_regions: Vec<CounterMappingRegion>,
 303     buffer: &RustString,
 304 ) {
 305     unsafe {
 306         llvm::LLVMRustCoverageWriteMappingToBuffer(
 307             virtual_file_mapping.as_ptr(),
 308             virtual_file_mapping.len() as c_uint,
 309             expressions.as_ptr(),
 310             expressions.len() as c_uint,
 311             mapping_regions.as_ptr(),
 312             mapping_regions.len() as c_uint,
 313             buffer,
 314         );
 315     }
 316 }
 317
 318 pub(crate) fn hash_str(strval: &str) -> u64 {
 319     let strval = CString::new(strval).expect("null error converting hashable str to C string");
 320     unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) }
 321 }
 322
 323 pub(crate) fn hash_bytes(bytes: Vec<u8>) -> u64 {
 324     unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) }
 325 }
 326
 327 pub(crate) fn mapping_version() -> u32 {
 328     unsafe { llvm::LLVMRustCoverageMappingVersion() }
 329 }
 330
 331 pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>(
 332     cx: &CodegenCx<'ll, 'tcx>,
 333     cov_data_val: &'ll llvm::Value,
 334 ) {
 335     let covmap_var_name = llvm::build_string(|s| unsafe {
 336         llvm::LLVMRustCoverageWriteMappingVarNameToString(s);
 337     })
 338     .expect("Rust Coverage Mapping var name failed UTF-8 conversion");
 339     debug!("covmap var name: {:?}", covmap_var_name);
 340
 341     let covmap_section_name = llvm::build_string(|s| unsafe {
 342         llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s);
 343     })
 344     .expect("Rust Coverage section name failed UTF-8 conversion");
 345     debug!("covmap section name: {:?}", covmap_section_name);
 346
 347     let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name);
 348     llvm::set_initializer(llglobal, cov_data_val);
 349     llvm::set_global_constant(llglobal, true);
 350     llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
 351     llvm::set_section(llglobal, &covmap_section_name);
 352     llvm::set_alignment(llglobal, VAR_ALIGN_BYTES);
 353     cx.add_used_global(llglobal);
 354 }
 355
 356 pub(crate) fn save_func_record_to_mod<'ll, 'tcx>(
 357     cx: &CodegenCx<'ll, 'tcx>,
 358     func_name_hash: u64,
 359     func_record_val: &'ll llvm::Value,
 360     is_used: bool,
 361 ) {
 362     // Assign a name to the function record. This is used to merge duplicates.
 363     //
 364     // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that
 365     // are included-but-not-used. If (or when) Rust generates functions that are
 366     // included-but-not-used, note that a dummy description for a function included-but-not-used
 367     // in a Crate can be replaced by full description provided by a different Crate. The two kinds
 368     // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by
 369     // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging.
 370     let func_record_var_name =
 371         format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" });
 372     debug!("function record var name: {:?}", func_record_var_name);
 373
 374     let func_record_section_name = llvm::build_string(|s| unsafe {
 375         llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s);
 376     })
 377     .expect("Rust Coverage function record section name failed UTF-8 conversion");
 378     debug!("function record section name: {:?}", func_record_section_name);
 379
 380     let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name);
 381     llvm::set_initializer(llglobal, func_record_val);
 382     llvm::set_global_constant(llglobal, true);
 383     llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage);
 384     llvm::set_visibility(llglobal, llvm::Visibility::Hidden);
 385     llvm::set_section(llglobal, &func_record_section_name);
 386     llvm::set_alignment(llglobal, VAR_ALIGN_BYTES);
 387     llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name);
 388     cx.add_used_global(llglobal);
 389 }