1 use crate::back::write::{
2 self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers,
4 use crate::llvm::archive_ro::ArchiveRO;
5 use crate::llvm::{self, build_string, False, True};
6 use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm};
7 use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared};
8 use rustc_codegen_ssa::back::symbol_export;
9 use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig};
10 use rustc_codegen_ssa::traits::*;
11 use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind};
12 use rustc_data_structures::fx::FxHashMap;
13 use rustc_errors::{FatalError, Handler};
14 use rustc_hir::def_id::LOCAL_CRATE;
15 use rustc_middle::bug;
16 use rustc_middle::dep_graph::WorkProduct;
17 use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
18 use rustc_session::cgu_reuse_tracker::CguReuse;
19 use rustc_session::config::{self, CrateType, Lto};
20 use tracing::{debug, info};
22 use std::ffi::{CStr, CString};
31 /// We keep track of the computed LTO cache keys from the previous
32 /// session to determine which CGUs we can reuse.
33 pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin";
35 pub fn crate_type_allows_lto(crate_type: CrateType) -> bool {
37 CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => true,
38 CrateType::Dylib | CrateType::Rlib | CrateType::ProcMacro => false,
43 cgcx: &CodegenContext<LlvmCodegenBackend>,
44 diag_handler: &Handler,
45 ) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> {
46 let export_threshold = match cgcx.lto {
47 // We're just doing LTO for our one crate
48 Lto::ThinLocal => SymbolExportLevel::Rust,
50 // We're doing LTO for the entire crate graph
51 Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types),
53 Lto::No => panic!("didn't request LTO but we're doing LTO"),
56 let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| {
57 if info.level.is_below_threshold(export_threshold) || info.used {
58 Some(CString::new(name.as_str()).unwrap())
63 let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
64 let mut symbols_below_threshold = {
65 let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold");
66 exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<CString>>()
68 info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
70 // If we're performing LTO for the entire crate graph, then for each of our
71 // upstream dependencies, find the corresponding rlib and load the bitcode
74 // We save off all the bytecode and LLVM module ids for later processing
75 // with either fat or thin LTO
76 let mut upstream_modules = Vec::new();
77 if cgcx.lto != Lto::ThinLocal {
78 if cgcx.opts.cg.prefer_dynamic {
80 .struct_err("cannot prefer dynamic linking when performing LTO")
82 "only 'staticlib', 'bin', and 'cdylib' outputs are \
86 return Err(FatalError);
89 // Make sure we actually can run LTO
90 for crate_type in cgcx.crate_types.iter() {
91 if !crate_type_allows_lto(*crate_type) {
92 let e = diag_handler.fatal(
93 "lto can only be run for executables, cdylibs and \
94 static library outputs",
100 for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
101 let exported_symbols =
102 cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
105 cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold");
106 symbols_below_threshold
107 .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
110 let archive = ArchiveRO::open(path).expect("wanted an rlib");
111 let obj_files = archive
113 .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c))))
114 .filter(|&(name, _)| looks_like_rust_object_file(name));
115 for (name, child) in obj_files {
116 info!("adding bitcode from {}", name);
117 match get_bitcode_slice_from_object_data(child.data()) {
119 let module = SerializedModule::FromRlib(data.to_vec());
120 upstream_modules.push((module, CString::new(name).unwrap()));
122 Err(msg) => return Err(diag_handler.fatal(&msg)),
128 Ok((symbols_below_threshold, upstream_modules))
131 fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> {
134 unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) };
137 let bc = unsafe { slice::from_raw_parts(data, len) };
139 // `bc` must be a sub-slice of `obj`.
140 assert!(obj.as_ptr() <= bc.as_ptr());
141 assert!(bc[bc.len()..bc.len()].as_ptr() <= obj[obj.len()..obj.len()].as_ptr());
146 let msg = llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string());
147 Err(format!("failed to get bitcode from object file for LTO ({})", msg))
151 /// Performs fat LTO by merging all modules into a single one and returning it
152 /// for further optimization.
153 pub(crate) fn run_fat(
154 cgcx: &CodegenContext<LlvmCodegenBackend>,
155 modules: Vec<FatLTOInput<LlvmCodegenBackend>>,
156 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
157 ) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> {
158 let diag_handler = cgcx.create_diag_handler();
159 let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?;
160 let symbols_below_threshold =
161 symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
168 &symbols_below_threshold,
172 /// Performs thin LTO by performing necessary global analysis and returning two
173 /// lists, one of the modules that need optimization and another for modules that
174 /// can simply be copied over from the incr. comp. cache.
175 pub(crate) fn run_thin(
176 cgcx: &CodegenContext<LlvmCodegenBackend>,
177 modules: Vec<(String, ThinBuffer)>,
178 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
179 ) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
180 let diag_handler = cgcx.create_diag_handler();
181 let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?;
182 let symbols_below_threshold =
183 symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
184 if cgcx.opts.cg.linker_plugin_lto.enabled() {
186 "We should never reach this case if the LTO step \
187 is deferred to the linker"
196 &symbols_below_threshold,
200 pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) {
201 let name = module.name.clone();
202 let buffer = ThinBuffer::new(module.module_llvm.llmod());
207 cgcx: &CodegenContext<LlvmCodegenBackend>,
208 diag_handler: &Handler,
209 modules: Vec<FatLTOInput<LlvmCodegenBackend>>,
210 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
211 mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
212 symbols_below_threshold: &[*const libc::c_char],
213 ) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> {
214 let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module");
215 info!("going for a fat lto");
217 // Sort out all our lists of incoming modules into two lists.
219 // * `serialized_modules` (also and argument to this function) contains all
220 // modules that are serialized in-memory.
221 // * `in_memory` contains modules which are already parsed and in-memory,
222 // such as from multi-CGU builds.
224 // All of `cached_modules` (cached from previous incremental builds) can
225 // immediately go onto the `serialized_modules` modules list and then we can
226 // split the `modules` array into these two lists.
227 let mut in_memory = Vec::new();
228 serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
229 info!("pushing cached module {:?}", wp.cgu_name);
230 (buffer, CString::new(wp.cgu_name).unwrap())
232 for module in modules {
234 FatLTOInput::InMemory(m) => in_memory.push(m),
235 FatLTOInput::Serialized { name, buffer } => {
236 info!("pushing serialized module {:?}", name);
237 let buffer = SerializedModule::Local(buffer);
238 serialized_modules.push((buffer, CString::new(name).unwrap()));
243 // Find the "costliest" module and merge everything into that codegen unit.
244 // All the other modules will be serialized and reparsed into the new
245 // context, so this hopefully avoids serializing and parsing the largest
248 // Additionally use a regular module as the base here to ensure that various
249 // file copy operations in the backend work correctly. The only other kind
250 // of module here should be an allocator one, and if your crate is smaller
251 // than the allocator module then the size doesn't really matter anyway.
252 let costliest_module = in_memory
255 .filter(|&(_, module)| module.kind == ModuleKind::Regular)
257 let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) };
262 // If we found a costliest module, we're good to go. Otherwise all our
263 // inputs were serialized which could happen in the case, for example, that
264 // all our inputs were incrementally reread from the cache and we're just
265 // re-executing the LTO passes. If that's the case deserialize the first
266 // module and create a linker with it.
267 let module: ModuleCodegen<ModuleLlvm> = match costliest_module {
268 Some((_cost, i)) => in_memory.remove(i),
270 assert!(!serialized_modules.is_empty(), "must have at least one serialized module");
271 let (buffer, name) = serialized_modules.remove(0);
272 info!("no in-memory regular modules to choose from, parsing {:?}", name);
274 module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), diag_handler)?,
275 name: name.into_string().unwrap(),
276 kind: ModuleKind::Regular,
280 let mut serialized_bitcode = Vec::new();
282 let (llcx, llmod) = {
283 let llvm = &module.module_llvm;
284 (&llvm.llcx, llvm.llmod())
286 info!("using {:?} as a base module", module.name);
288 // The linking steps below may produce errors and diagnostics within LLVM
289 // which we'd like to handle and print, so set up our diagnostic handlers
290 // (which get unregistered when they go out of scope below).
291 let _handler = DiagnosticHandlers::new(cgcx, diag_handler, llcx);
293 // For all other modules we codegened we'll need to link them into our own
294 // bitcode. All modules were codegened in their own LLVM context, however,
295 // and we want to move everything to the same LLVM context. Currently the
296 // way we know of to do that is to serialize them to a string and them parse
297 // them later. Not great but hey, that's why it's "fat" LTO, right?
298 for module in in_memory {
299 let buffer = ModuleBuffer::new(module.module_llvm.llmod());
300 let llmod_id = CString::new(&module.name[..]).unwrap();
301 serialized_modules.push((SerializedModule::Local(buffer), llmod_id));
303 // Sort the modules to ensure we produce deterministic results.
304 serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1));
306 // For all serialized bitcode files we parse them and link them in as we did
307 // above, this is all mostly handled in C++. Like above, though, we don't
308 // know much about the memory management here so we err on the side of being
309 // save and persist everything with the original module.
310 let mut linker = Linker::new(llmod);
311 for (bc_decoded, name) in serialized_modules {
314 .generic_activity_with_arg_recorder("LLVM_fat_lto_link_module", |recorder| {
315 recorder.record_arg(format!("{:?}", name))
317 info!("linking {:?}", name);
318 let data = bc_decoded.data();
319 linker.add(data).map_err(|()| {
320 let msg = format!("failed to load bitcode of module {:?}", name);
321 write::llvm_err(diag_handler, &msg)
323 serialized_bitcode.push(bc_decoded);
326 save_temp_bitcode(cgcx, &module, "lto.input");
328 // Fat LTO also suffers from the invalid DWARF issue similar to Thin LTO.
329 // Here we rewrite all `DICompileUnit` pointers if there is only one `DICompileUnit`.
330 // This only works around the problem when codegen-units = 1.
331 // Refer to the comments in the `optimize_thin_module` function for more details.
332 let mut cu1 = ptr::null_mut();
333 let mut cu2 = ptr::null_mut();
334 unsafe { llvm::LLVMRustLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2) };
337 cgcx.prof.generic_activity_with_arg("LLVM_fat_lto_patch_debuginfo", &*module.name);
338 unsafe { llvm::LLVMRustLTOPatchDICompileUnit(llmod, cu1) };
339 save_temp_bitcode(cgcx, &module, "fat-lto-after-patch");
342 // Internalize everything below threshold to help strip out more modules and such.
344 let ptr = symbols_below_threshold.as_ptr();
345 llvm::LLVMRustRunRestrictionPass(
347 ptr as *const *const libc::c_char,
348 symbols_below_threshold.len() as libc::size_t,
350 save_temp_bitcode(cgcx, &module, "lto.after-restriction");
354 Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode })
357 pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>);
359 impl<'a> Linker<'a> {
360 pub(crate) fn new(llmod: &'a llvm::Module) -> Self {
361 unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) }
364 pub(crate) fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> {
366 if llvm::LLVMRustLinkerAdd(
368 bytecode.as_ptr() as *const libc::c_char,
379 impl Drop for Linker<'_> {
382 llvm::LLVMRustLinkerFree(&mut *(self.0 as *mut _));
387 /// Prepare "thin" LTO to get run on these modules.
389 /// The general structure of ThinLTO is quite different from the structure of
390 /// "fat" LTO above. With "fat" LTO all LLVM modules in question are merged into
391 /// one giant LLVM module, and then we run more optimization passes over this
392 /// big module after internalizing most symbols. Thin LTO, on the other hand,
393 /// avoid this large bottleneck through more targeted optimization.
395 /// At a high level Thin LTO looks like:
397 /// 1. Prepare a "summary" of each LLVM module in question which describes
398 /// the values inside, cost of the values, etc.
399 /// 2. Merge the summaries of all modules in question into one "index"
400 /// 3. Perform some global analysis on this index
401 /// 4. For each module, use the index and analysis calculated previously to
402 /// perform local transformations on the module, for example inlining
403 /// small functions from other modules.
404 /// 5. Run thin-specific optimization passes over each module, and then code
405 /// generate everything at the end.
407 /// The summary for each module is intended to be quite cheap, and the global
408 /// index is relatively quite cheap to create as well. As a result, the goal of
409 /// ThinLTO is to reduce the bottleneck on LTO and enable LTO to be used in more
410 /// situations. For example one cheap optimization is that we can parallelize
411 /// all codegen modules, easily making use of all the cores on a machine.
413 /// With all that in mind, the function here is designed at specifically just
414 /// calculating the *index* for ThinLTO. This index will then be shared amongst
415 /// all of the `LtoModuleCodegen` units returned below and destroyed once
416 /// they all go out of scope.
418 cgcx: &CodegenContext<LlvmCodegenBackend>,
419 diag_handler: &Handler,
420 modules: Vec<(String, ThinBuffer)>,
421 serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
422 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
423 symbols_below_threshold: &[*const libc::c_char],
424 ) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
425 let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis");
427 info!("going for that thin, thin LTO");
429 let green_modules: FxHashMap<_, _> =
430 cached_modules.iter().map(|&(_, ref wp)| (wp.cgu_name.clone(), wp.clone())).collect();
432 let full_scope_len = modules.len() + serialized_modules.len() + cached_modules.len();
433 let mut thin_buffers = Vec::with_capacity(modules.len());
434 let mut module_names = Vec::with_capacity(full_scope_len);
435 let mut thin_modules = Vec::with_capacity(full_scope_len);
437 for (i, (name, buffer)) in modules.into_iter().enumerate() {
438 info!("local module: {} - {}", i, name);
439 let cname = CString::new(name.clone()).unwrap();
440 thin_modules.push(llvm::ThinLTOModule {
441 identifier: cname.as_ptr(),
442 data: buffer.data().as_ptr(),
443 len: buffer.data().len(),
445 thin_buffers.push(buffer);
446 module_names.push(cname);
449 // FIXME: All upstream crates are deserialized internally in the
450 // function below to extract their summary and modules. Note that
451 // unlike the loop above we *must* decode and/or read something
452 // here as these are all just serialized files on disk. An
453 // improvement, however, to make here would be to store the
454 // module summary separately from the actual module itself. Right
455 // now this is store in one large bitcode file, and the entire
456 // file is deflate-compressed. We could try to bypass some of the
457 // decompression by storing the index uncompressed and only
458 // lazily decompressing the bytecode if necessary.
460 // Note that truly taking advantage of this optimization will
461 // likely be further down the road. We'd have to implement
462 // incremental ThinLTO first where we could actually avoid
463 // looking at upstream modules entirely sometimes (the contents,
464 // we must always unconditionally look at the index).
465 let mut serialized = Vec::with_capacity(serialized_modules.len() + cached_modules.len());
468 cached_modules.into_iter().map(|(sm, wp)| (sm, CString::new(wp.cgu_name).unwrap()));
470 for (module, name) in serialized_modules.into_iter().chain(cached_modules) {
471 info!("upstream or cached module {:?}", name);
472 thin_modules.push(llvm::ThinLTOModule {
473 identifier: name.as_ptr(),
474 data: module.data().as_ptr(),
475 len: module.data().len(),
477 serialized.push(module);
478 module_names.push(name);
482 assert_eq!(thin_modules.len(), module_names.len());
484 // Delegate to the C++ bindings to create some data here. Once this is a
485 // tried-and-true interface we may wish to try to upstream some of this
486 // to LLVM itself, right now we reimplement a lot of what they do
488 let data = llvm::LLVMRustCreateThinLTOData(
489 thin_modules.as_ptr(),
490 thin_modules.len() as u32,
491 symbols_below_threshold.as_ptr(),
492 symbols_below_threshold.len() as u32,
494 .ok_or_else(|| write::llvm_err(diag_handler, "failed to prepare thin LTO context"))?;
496 let data = ThinData(data);
498 info!("thin LTO data created");
500 let (key_map_path, prev_key_map, curr_key_map) = if let Some(ref incr_comp_session_dir) =
501 cgcx.incr_comp_session_dir
503 let path = incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME);
504 // If the previous file was deleted, or we get an IO error
505 // reading the file, then we'll just use `None` as the
506 // prev_key_map, which will force the code to be recompiled.
508 if path.exists() { ThinLTOKeysMap::load_from_file(&path).ok() } else { None };
509 let curr = ThinLTOKeysMap::from_thin_lto_modules(&data, &thin_modules, &module_names);
510 (Some(path), prev, curr)
512 // If we don't compile incrementally, we don't need to load the
513 // import data from LLVM.
514 assert!(green_modules.is_empty());
515 let curr = ThinLTOKeysMap::default();
518 info!("thin LTO cache key map loaded");
519 info!("prev_key_map: {:#?}", prev_key_map);
520 info!("curr_key_map: {:#?}", curr_key_map);
522 // Throw our data in an `Arc` as we'll be sharing it across threads. We
523 // also put all memory referenced by the C++ data (buffers, ids, etc)
524 // into the arc as well. After this we'll create a thin module
525 // codegen per module in this data.
526 let shared = Arc::new(ThinShared {
529 serialized_modules: serialized,
533 let mut copy_jobs = vec![];
534 let mut opt_jobs = vec![];
536 info!("checking which modules can be-reused and which have to be re-optimized.");
537 for (module_index, module_name) in shared.module_names.iter().enumerate() {
538 let module_name = module_name_to_str(module_name);
539 if let (Some(prev_key_map), true) =
540 (prev_key_map.as_ref(), green_modules.contains_key(module_name))
542 assert!(cgcx.incr_comp_session_dir.is_some());
544 // If a module exists in both the current and the previous session,
545 // and has the same LTO cache key in both sessions, then we can re-use it
546 if prev_key_map.keys.get(module_name) == curr_key_map.keys.get(module_name) {
547 let work_product = green_modules[module_name].clone();
548 copy_jobs.push(work_product);
549 info!(" - {}: re-used", module_name);
550 assert!(cgcx.incr_comp_session_dir.is_some());
551 cgcx.cgu_reuse_tracker.set_actual_reuse(module_name, CguReuse::PostLto);
556 info!(" - {}: re-compiled", module_name);
557 opt_jobs.push(LtoModuleCodegen::Thin(ThinModule {
558 shared: shared.clone(),
563 // Save the current ThinLTO import information for the next compilation
564 // session, overwriting the previous serialized data (if any).
565 if let Some(path) = key_map_path {
566 if let Err(err) = curr_key_map.save_to_file(&path) {
567 let msg = format!("Error while writing ThinLTO key data: {}", err);
568 return Err(write::llvm_err(diag_handler, &msg));
572 Ok((opt_jobs, copy_jobs))
576 pub(crate) fn run_pass_manager(
577 cgcx: &CodegenContext<LlvmCodegenBackend>,
578 diag_handler: &Handler,
579 module: &mut ModuleCodegen<ModuleLlvm>,
581 ) -> Result<(), FatalError> {
582 let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &*module.name);
583 let config = cgcx.config(module.kind);
585 // Now we have one massive module inside of llmod. Time to run the
586 // LTO-specific optimization passes that LLVM provides.
588 // This code is based off the code found in llvm's LTO code generator:
589 // llvm/lib/LTO/LTOCodeGenerator.cpp
590 debug!("running the pass manager");
592 if !llvm::LLVMRustHasModuleFlag(
593 module.module_llvm.llmod(),
594 "LTOPostLink".as_ptr().cast(),
597 llvm::LLVMRustAddModuleFlag(
598 module.module_llvm.llmod(),
599 llvm::LLVMModFlagBehavior::Error,
600 "LTOPostLink\0".as_ptr().cast(),
604 if llvm_util::should_use_new_llvm_pass_manager(
605 &config.new_llvm_pass_manager,
608 let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO };
609 let opt_level = config.opt_level.unwrap_or(config::OptLevel::No);
610 write::optimize_with_new_llvm_pass_manager(
622 let pm = llvm::LLVMCreatePassManager();
623 llvm::LLVMAddAnalysisPasses(module.module_llvm.tm, pm);
625 if config.verify_llvm_ir {
626 let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast());
627 llvm::LLVMRustAddPass(pm, pass.unwrap());
630 let opt_level = config
632 .map(|x| to_llvm_opt_settings(x).0)
633 .unwrap_or(llvm::CodeGenOptLevel::None);
634 with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| {
636 llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm);
638 llvm::LLVMRustPassManagerBuilderPopulateLTOPassManager(
639 b, pm, /* Internalize = */ False, /* RunInliner = */ True,
644 // We always generate bitcode through ThinLTOBuffers,
645 // which do not support anonymous globals
646 if config.bitcode_needed() {
647 let pass = llvm::LLVMRustFindAndCreatePass("name-anon-globals\0".as_ptr().cast());
648 llvm::LLVMRustAddPass(pm, pass.unwrap());
651 if config.verify_llvm_ir {
652 let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast());
653 llvm::LLVMRustAddPass(pm, pass.unwrap());
656 llvm::LLVMRunPassManager(pm, module.module_llvm.llmod());
658 llvm::LLVMDisposePassManager(pm);
664 pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer);
666 unsafe impl Send for ModuleBuffer {}
667 unsafe impl Sync for ModuleBuffer {}
670 pub fn new(m: &llvm::Module) -> ModuleBuffer {
671 ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) })
675 impl ModuleBufferMethods for ModuleBuffer {
676 fn data(&self) -> &[u8] {
678 let ptr = llvm::LLVMRustModuleBufferPtr(self.0);
679 let len = llvm::LLVMRustModuleBufferLen(self.0);
680 slice::from_raw_parts(ptr, len)
685 impl Drop for ModuleBuffer {
688 llvm::LLVMRustModuleBufferFree(&mut *(self.0 as *mut _));
693 pub struct ThinData(&'static mut llvm::ThinLTOData);
695 unsafe impl Send for ThinData {}
696 unsafe impl Sync for ThinData {}
698 impl Drop for ThinData {
701 llvm::LLVMRustFreeThinLTOData(&mut *(self.0 as *mut _));
706 pub struct ThinBuffer(&'static mut llvm::ThinLTOBuffer);
708 unsafe impl Send for ThinBuffer {}
709 unsafe impl Sync for ThinBuffer {}
712 pub fn new(m: &llvm::Module) -> ThinBuffer {
714 let buffer = llvm::LLVMRustThinLTOBufferCreate(m);
720 impl ThinBufferMethods for ThinBuffer {
721 fn data(&self) -> &[u8] {
723 let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _;
724 let len = llvm::LLVMRustThinLTOBufferLen(self.0);
725 slice::from_raw_parts(ptr, len)
730 impl Drop for ThinBuffer {
733 llvm::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _));
738 pub unsafe fn optimize_thin_module(
739 thin_module: ThinModule<LlvmCodegenBackend>,
740 cgcx: &CodegenContext<LlvmCodegenBackend>,
741 ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
742 let diag_handler = cgcx.create_diag_handler();
744 let module_name = &thin_module.shared.module_names[thin_module.idx];
745 let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap());
747 (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, &e))?;
749 // Right now the implementation we've got only works over serialized
750 // modules, so we create a fresh new LLVM context and parse the module
751 // into that context. One day, however, we may do this for upstream
752 // crates but for locally codegened modules we may be able to reuse
753 // that LLVM Context and Module.
754 let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
755 let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _;
756 let mut module = ModuleCodegen {
757 module_llvm: ModuleLlvm { llmod_raw, llcx, tm },
758 name: thin_module.name().to_string(),
759 kind: ModuleKind::Regular,
762 let target = &*module.module_llvm.tm;
763 let llmod = module.module_llvm.llmod();
764 save_temp_bitcode(cgcx, &module, "thin-lto-input");
766 // Before we do much else find the "main" `DICompileUnit` that we'll be
767 // using below. If we find more than one though then rustc has changed
768 // in a way we're not ready for, so generate an ICE by returning
770 let mut cu1 = ptr::null_mut();
771 let mut cu2 = ptr::null_mut();
772 llvm::LLVMRustLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2);
774 let msg = "multiple source DICompileUnits found";
775 return Err(write::llvm_err(&diag_handler, msg));
778 // Up next comes the per-module local analyses that we do for Thin LTO.
779 // Each of these functions is basically copied from the LLVM
780 // implementation and then tailored to suit this implementation. Ideally
781 // each of these would be supported by upstream LLVM but that's perhaps
782 // a patch for another day!
784 // You can find some more comments about these functions in the LLVM
785 // bindings we've got (currently `PassWrapper.cpp`)
788 cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name());
789 if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) {
790 let msg = "failed to prepare thin LTO module";
791 return Err(write::llvm_err(&diag_handler, msg));
793 save_temp_bitcode(cgcx, &module, "thin-lto-after-rename");
799 .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name());
800 if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) {
801 let msg = "failed to prepare thin LTO module";
802 return Err(write::llvm_err(&diag_handler, msg));
804 save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve");
810 .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name());
811 if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) {
812 let msg = "failed to prepare thin LTO module";
813 return Err(write::llvm_err(&diag_handler, msg));
815 save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize");
820 cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name());
821 if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) {
822 let msg = "failed to prepare thin LTO module";
823 return Err(write::llvm_err(&diag_handler, msg));
825 save_temp_bitcode(cgcx, &module, "thin-lto-after-import");
828 // Ok now this is a bit unfortunate. This is also something you won't
829 // find upstream in LLVM's ThinLTO passes! This is a hack for now to
830 // work around bugs in LLVM.
832 // First discovered in #45511 it was found that as part of ThinLTO
833 // importing passes LLVM will import `DICompileUnit` metadata
834 // information across modules. This means that we'll be working with one
835 // LLVM module that has multiple `DICompileUnit` instances in it (a
836 // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of
837 // bugs in LLVM's backend which generates invalid DWARF in a situation
840 // https://bugs.llvm.org/show_bug.cgi?id=35212
841 // https://bugs.llvm.org/show_bug.cgi?id=35562
843 // While the first bug there is fixed the second ended up causing #46346
844 // which was basically a resurgence of #45511 after LLVM's bug 35212 was
847 // This function below is a huge hack around this problem. The function
848 // below is defined in `PassWrapper.cpp` and will basically "merge"
849 // all `DICompileUnit` instances in a module. Basically it'll take all
850 // the objects, rewrite all pointers of `DISubprogram` to point to the
851 // first `DICompileUnit`, and then delete all the other units.
853 // This is probably mangling to the debug info slightly (but hopefully
854 // not too much) but for now at least gets LLVM to emit valid DWARF (or
855 // so it appears). Hopefully we can remove this once upstream bugs are
860 .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name());
861 llvm::LLVMRustLTOPatchDICompileUnit(llmod, cu1);
862 save_temp_bitcode(cgcx, &module, "thin-lto-after-patch");
865 // Alright now that we've done everything related to the ThinLTO
866 // analysis it's time to run some optimizations! Here we use the same
867 // `run_pass_manager` as the "fat" LTO above except that we tell it to
868 // populate a thin-specific pass manager, which presumably LLVM treats a
869 // little differently.
871 info!("running thin lto passes over {}", module.name);
872 run_pass_manager(cgcx, &diag_handler, &mut module, true)?;
873 save_temp_bitcode(cgcx, &module, "thin-lto-after-pm");
879 /// Maps LLVM module identifiers to their corresponding LLVM LTO cache keys
880 #[derive(Debug, Default)]
881 pub struct ThinLTOKeysMap {
882 // key = llvm name of importing module, value = LLVM cache key
883 keys: FxHashMap<String, String>,
886 impl ThinLTOKeysMap {
887 fn save_to_file(&self, path: &Path) -> io::Result<()> {
889 let file = File::create(path)?;
890 let mut writer = io::BufWriter::new(file);
891 for (module, key) in &self.keys {
892 writeln!(writer, "{} {}", module, key)?;
897 fn load_from_file(path: &Path) -> io::Result<Self> {
898 use std::io::BufRead;
899 let mut keys = FxHashMap::default();
900 let file = File::open(path)?;
901 for line in io::BufReader::new(file).lines() {
903 let mut split = line.split(' ');
904 let module = split.next().unwrap();
905 let key = split.next().unwrap();
906 assert_eq!(split.next(), None, "Expected two space-separated values, found {:?}", line);
907 keys.insert(module.to_string(), key.to_string());
912 fn from_thin_lto_modules(
914 modules: &[llvm::ThinLTOModule],
917 let keys = iter::zip(modules, names)
918 .map(|(module, name)| {
919 let key = build_string(|rust_str| unsafe {
920 llvm::LLVMRustComputeLTOCacheKey(rust_str, module.identifier, data.0);
922 .expect("Invalid ThinLTO module key");
923 (name.clone().into_string().unwrap(), key)
930 fn module_name_to_str(c_str: &CStr) -> &str {
931 c_str.to_str().unwrap_or_else(|e| {
932 bug!("Encountered non-utf8 LLVM module name `{}`: {}", c_str.to_string_lossy(), e)
936 pub fn parse_module<'a>(
937 cx: &'a llvm::Context,
940 diag_handler: &Handler,
941 ) -> Result<&'a llvm::Module, FatalError> {
943 llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else(
945 let msg = "failed to parse bitcode for LTO module";
946 write::llvm_err(diag_handler, msg)