compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

   1 use crate::llvm;
   2
   3 use crate::abi::Abi;
   4 use crate::builder::Builder;
   5 use crate::common::CodegenCx;
   6
   7 use libc::c_uint;
   8 use llvm::coverageinfo::CounterMappingRegion;
   9 use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage};
  10 use rustc_codegen_ssa::traits::{
  11     BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods,
  12     MiscMethods, StaticMethods,
  13 };
  14 use rustc_data_structures::fx::FxHashMap;
  15 use rustc_hir as hir;
  16 use rustc_hir::def_id::DefId;
  17 use rustc_llvm::RustString;
  18 use rustc_middle::bug;
  19 use rustc_middle::mir::coverage::{
  20     CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op,
  21 };
  22 use rustc_middle::ty;
  23 use rustc_middle::ty::layout::FnAbiOf;
  24 use rustc_middle::ty::subst::InternalSubsts;
  25 use rustc_middle::ty::Instance;
  26
  27 use std::cell::RefCell;
  28 use std::ffi::CString;
  29
  30 use std::iter;
  31
  32 pub mod mapgen;
  33
  34 const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START;
  35
  36 const VAR_ALIGN_BYTES: usize = 8;
  37
  38 /// A context object for maintaining all state needed by the coverageinfo module.
  39 pub struct CrateCoverageContext<'ll, 'tcx> {
  40     /// Coverage data for each instrumented function identified by DefId.
  41     pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>,
  42     pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>,
  43 }
  44
  45 impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> {
  46     pub fn new() -> Self {
  47         Self {
  48             function_coverage_map: Default::default(),
  49             pgo_func_name_var_map: Default::default(),
  50         }
  51     }
  52
  53     pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> {
  54         self.function_coverage_map.replace(FxHashMap::default())
  55     }
  56 }
  57
  58 impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
  59     fn coverageinfo_finalize(&self) {
  60         mapgen::finalize(self)
  61     }
  62
  63     fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value {
  64         if let Some(coverage_context) = self.coverage_context() {
  65             debug!("getting pgo_func_name_var for instance={:?}", instance);
  66             let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut();
  67             pgo_func_name_var_map
  68                 .entry(instance)
  69                 .or_insert_with(|| create_pgo_func_name_var(self, instance))
  70         } else {
  71             bug!("Could not get the `coverage_context`");
  72         }
  73     }
  74
  75     /// Functions with MIR-based coverage are normally codegenned _only_ if
  76     /// called. LLVM coverage tools typically expect every function to be
  77     /// defined (even if unused), with at least one call to LLVM intrinsic
  78     /// `instrprof.increment`.
  79     ///
  80     /// Codegen a small function that will never be called, with one counter
  81     /// that will never be incremented.
  82     ///
  83     /// For used/called functions, the coverageinfo was already added to the
  84     /// `function_coverage_map` (keyed by function `Instance`) during codegen.
  85     /// But in this case, since the unused function was _not_ previously
  86     /// codegenned, collect the coverage `CodeRegion`s from the MIR and add
  87     /// them. The first `CodeRegion` is used to add a single counter, with the
  88     /// same counter ID used in the injected `instrprof.increment` intrinsic
  89     /// call. Since the function is never called, all other `CodeRegion`s can be
  90     /// added as `unreachable_region`s.
  91     fn define_unused_fn(&self, def_id: DefId) {
  92         let instance = declare_unused_fn(self, def_id);
  93         codegen_unused_fn_and_counter(self, instance);
  94         add_unused_function_coverage(self, instance, def_id);
  95     }
  96 }
  97
  98 impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
  99     fn set_function_source_hash(
 100         &mut self,
 101         instance: Instance<'tcx>,
 102         function_source_hash: u64,
 103     ) -> bool {
 104         if let Some(coverage_context) = self.coverage_context() {
 105             debug!(
 106                 "ensuring function source hash is set for instance={:?}; function_source_hash={}",
 107                 instance, function_source_hash,
 108             );
 109             let mut coverage_map = coverage_context.function_coverage_map.borrow_mut();
 110             coverage_map
 111                 .entry(instance)
 112                 .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
 113                 .set_function_source_hash(function_source_hash);
 114             true
 115         } else {
 116             false
 117         }
 118     }
 119
 120     fn add_coverage_counter(
 121         &mut self,
 122         instance: Instance<'tcx>,
 123         id: CounterValueReference,
 124         region: CodeRegion,
 125     ) -> bool {
 126         if let Some(coverage_context) = self.coverage_context() {
 127             debug!(
 128                 "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}",
 129                 instance, id, region,
 130             );
 131             let mut coverage_map = coverage_context.function_coverage_map.borrow_mut();
 132             coverage_map
 133                 .entry(instance)
 134                 .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
 135                 .add_counter(id, region);
 136             true
 137         } else {
 138             false
 139         }
 140     }
 141
 142     fn add_coverage_counter_expression(
 143         &mut self,
 144         instance: Instance<'tcx>,
 145         id: InjectedExpressionId,
 146         lhs: ExpressionOperandId,
 147         op: Op,
 148         rhs: ExpressionOperandId,
 149         region: Option<CodeRegion>,
 150     ) -> bool {
 151         if let Some(coverage_context) = self.coverage_context() {
 152             debug!(
 153                 "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; \
 154                 region: {:?}",
 155                 instance, id, lhs, op, rhs, region,
 156             );
 157             let mut coverage_map = coverage_context.function_coverage_map.borrow_mut();
 158             coverage_map
 159                 .entry(instance)
 160                 .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
 161                 .add_counter_expression(id, lhs, op, rhs, region);
 162             true
 163         } else {
 164             false
 165         }
 166     }
 167
 168     fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool {
 169         if let Some(coverage_context) = self.coverage_context() {
 170             debug!(
 171                 "adding unreachable code to coverage_map: instance={:?}, at {:?}",
 172                 instance, region,
 173             );
 174             let mut coverage_map = coverage_context.function_coverage_map.borrow_mut();
 175             coverage_map
 176                 .entry(instance)
 177                 .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
 178                 .add_unreachable_region(region);
 179             true
 180         } else {
 181             false
 182         }
 183     }
 184 }
 185
 186 fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<'tcx> {
 187     let tcx = cx.tcx;
 188
 189     let instance = Instance::new(
 190         def_id,
 191         InternalSubsts::for_item(tcx, def_id, |param, _| {
 192             if let ty::GenericParamDefKind::Lifetime = param.kind {
 193                 tcx.lifetimes.re_erased.into()
 194             } else {
 195                 tcx.mk_param_from_def(param)
 196             }
 197         }),
 198     );
 199
 200     let llfn = cx.declare_fn(
 201         tcx.symbol_name(instance).name,
 202         cx.fn_abi_of_fn_ptr(
 203             ty::Binder::dummy(tcx.mk_fn_sig(
 204                 iter::once(tcx.mk_unit()),
 205                 tcx.mk_unit(),
 206                 false,
 207                 hir::Unsafety::Unsafe,
 208                 Abi::Rust,
 209             )),
 210             ty::List::empty(),
 211         ),
 212     );
 213
 214     llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage);
 215     llvm::set_visibility(llfn, llvm::Visibility::Default);
 216
 217     assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none());
 218
 219     instance
 220 }
 221
 222 fn codegen_unused_fn_and_counter<'tcx>(cx: &CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) {
 223     let llfn = cx.get_fn(instance);
 224     let llbb = Builder::append_block(cx, llfn, "unused_function");
 225     let mut bx = Builder::build(cx, llbb);
 226     let fn_name = bx.get_pgo_func_name_var(instance);
 227     let hash = bx.const_u64(0);
 228     let num_counters = bx.const_u32(1);
 229     let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID));
 230     debug!(
 231         "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?},
 232             index={:?}) for unused function: {:?}",
 233         fn_name, hash, num_counters, index, instance
 234     );
 235     bx.instrprof_increment(fn_name, hash, num_counters, index);
 236     bx.ret_void();
 237 }
 238
 239 fn add_unused_function_coverage<'tcx>(
 240     cx: &CodegenCx<'_, 'tcx>,
 241     instance: Instance<'tcx>,
 242     def_id: DefId,
 243 ) {
 244     let tcx = cx.tcx;
 245
 246     let mut function_coverage = FunctionCoverage::unused(tcx, instance);
 247     for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() {
 248         if index == 0 {
 249             // Insert at least one real counter so the LLVM CoverageMappingReader will find expected
 250             // definitions.
 251             function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone());
 252         } else {
 253             function_coverage.add_unreachable_region(code_region.clone());
 254         }
 255     }
 256
 257     if let Some(coverage_context) = cx.coverage_context() {
 258         coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage);
 259     } else {
 260         bug!("Could not get the `coverage_context`");
 261     }
 262 }
 263
 264 /// Calls llvm::createPGOFuncNameVar() with the given function instance's
 265 /// mangled function name. The LLVM API returns an llvm::GlobalVariable
 266 /// containing the function name, with the specific variable name and linkage
 267 /// required by LLVM InstrProf source-based coverage instrumentation. Use
 268 /// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per
 269 /// `Instance`.
 270 fn create_pgo_func_name_var<'ll, 'tcx>(
 271     cx: &CodegenCx<'ll, 'tcx>,
 272     instance: Instance<'tcx>,
 273 ) -> &'ll llvm::Value {
 274     let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name)
 275         .expect("error converting function name to C string");
 276     let llfn = cx.get_fn(instance);
 277     unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) }
 278 }
 279
 280 pub(crate) fn write_filenames_section_to_buffer<'a>(
 281     filenames: impl IntoIterator<Item = &'a CString>,
 282     buffer: &RustString,
 283 ) {
 284     let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::<Vec<_>>();
 285     unsafe {
 286         llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer(
 287             c_str_vec.as_ptr(),
 288             c_str_vec.len(),
 289             buffer,
 290         );
 291     }
 292 }
 293
 294 pub(crate) fn write_mapping_to_buffer(
 295     virtual_file_mapping: Vec<u32>,
 296     expressions: Vec<CounterExpression>,
 297     mapping_regions: Vec<CounterMappingRegion>,
 298     buffer: &RustString,
 299 ) {
 300     unsafe {
 301         llvm::LLVMRustCoverageWriteMappingToBuffer(
 302             virtual_file_mapping.as_ptr(),
 303             virtual_file_mapping.len() as c_uint,
 304             expressions.as_ptr(),
 305             expressions.len() as c_uint,
 306             mapping_regions.as_ptr(),
 307             mapping_regions.len() as c_uint,
 308             buffer,
 309         );
 310     }
 311 }
 312
 313 pub(crate) fn hash_str(strval: &str) -> u64 {
 314     let strval = CString::new(strval).expect("null error converting hashable str to C string");
 315     unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) }
 316 }
 317
 318 pub(crate) fn hash_bytes(bytes: Vec<u8>) -> u64 {
 319     unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) }
 320 }
 321
 322 pub(crate) fn mapping_version() -> u32 {
 323     unsafe { llvm::LLVMRustCoverageMappingVersion() }
 324 }
 325
 326 pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>(
 327     cx: &CodegenCx<'ll, 'tcx>,
 328     cov_data_val: &'ll llvm::Value,
 329 ) {
 330     let covmap_var_name = llvm::build_string(|s| unsafe {
 331         llvm::LLVMRustCoverageWriteMappingVarNameToString(s);
 332     })
 333     .expect("Rust Coverage Mapping var name failed UTF-8 conversion");
 334     debug!("covmap var name: {:?}", covmap_var_name);
 335
 336     let covmap_section_name = llvm::build_string(|s| unsafe {
 337         llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s);
 338     })
 339     .expect("Rust Coverage section name failed UTF-8 conversion");
 340     debug!("covmap section name: {:?}", covmap_section_name);
 341
 342     let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name);
 343     llvm::set_initializer(llglobal, cov_data_val);
 344     llvm::set_global_constant(llglobal, true);
 345     llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
 346     llvm::set_section(llglobal, &covmap_section_name);
 347     llvm::set_alignment(llglobal, VAR_ALIGN_BYTES);
 348     cx.add_used_global(llglobal);
 349 }
 350
 351 pub(crate) fn save_func_record_to_mod<'ll, 'tcx>(
 352     cx: &CodegenCx<'ll, 'tcx>,
 353     func_name_hash: u64,
 354     func_record_val: &'ll llvm::Value,
 355     is_used: bool,
 356 ) {
 357     // Assign a name to the function record. This is used to merge duplicates.
 358     //
 359     // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that
 360     // are included-but-not-used. If (or when) Rust generates functions that are
 361     // included-but-not-used, note that a dummy description for a function included-but-not-used
 362     // in a Crate can be replaced by full description provided by a different Crate. The two kinds
 363     // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by
 364     // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging.
 365     let func_record_var_name =
 366         format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" });
 367     debug!("function record var name: {:?}", func_record_var_name);
 368
 369     let func_record_section_name = llvm::build_string(|s| unsafe {
 370         llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s);
 371     })
 372     .expect("Rust Coverage function record section name failed UTF-8 conversion");
 373     debug!("function record section name: {:?}", func_record_section_name);
 374
 375     let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name);
 376     llvm::set_initializer(llglobal, func_record_val);
 377     llvm::set_global_constant(llglobal, true);
 378     llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage);
 379     llvm::set_visibility(llglobal, llvm::Visibility::Hidden);
 380     llvm::set_section(llglobal, &func_record_section_name);
 381     llvm::set_alignment(llglobal, VAR_ALIGN_BYTES);
 382     llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name);
 383     cx.add_used_global(llglobal);
 384 }