1 use crate::back::bytecode::DecodedBytecode;
2 use crate::back::write::{
3 self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers,
5 use crate::llvm::archive_ro::ArchiveRO;
6 use crate::llvm::{self, False, True};
7 use crate::{LlvmCodegenBackend, ModuleLlvm};
8 use log::{debug, info};
10 use rustc::dep_graph::WorkProduct;
11 use rustc::middle::exported_symbols::SymbolExportLevel;
12 use rustc::session::config::{self, Lto};
13 use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared};
14 use rustc_codegen_ssa::back::symbol_export;
15 use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, ModuleConfig};
16 use rustc_codegen_ssa::traits::*;
17 use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, RLIB_BYTECODE_EXTENSION};
18 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
19 use rustc_errors::{FatalError, Handler};
20 use rustc_hir::def_id::LOCAL_CRATE;
21 use rustc_session::cgu_reuse_tracker::CguReuse;
23 use std::ffi::{CStr, CString};
32 /// We keep track of past LTO imports that were used to produce the current set
33 /// of compiled object files that we might choose to reuse during this
34 /// compilation session.
35 pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-imports.bin";
37 pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool {
39 config::CrateType::Executable
40 | config::CrateType::Staticlib
41 | config::CrateType::Cdylib => true,
43 config::CrateType::Dylib | config::CrateType::Rlib | config::CrateType::ProcMacro => false,
48 cgcx: &CodegenContext<LlvmCodegenBackend>,
49 diag_handler: &Handler,
50 ) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> {
51 let export_threshold = match cgcx.lto {
52 // We're just doing LTO for our one crate
53 Lto::ThinLocal => SymbolExportLevel::Rust,
55 // We're doing LTO for the entire crate graph
56 Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types),
58 Lto::No => panic!("didn't request LTO but we're doing LTO"),
61 let symbol_filter = &|&(ref name, level): &(String, SymbolExportLevel)| {
62 if level.is_below_threshold(export_threshold) {
63 Some(CString::new(name.as_str()).unwrap())
68 let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
69 let mut symbol_white_list = {
70 let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbol_white_list");
71 exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<CString>>()
73 info!("{} symbols to preserve in this crate", symbol_white_list.len());
75 // If we're performing LTO for the entire crate graph, then for each of our
76 // upstream dependencies, find the corresponding rlib and load the bitcode
79 // We save off all the bytecode and LLVM module ids for later processing
80 // with either fat or thin LTO
81 let mut upstream_modules = Vec::new();
82 if cgcx.lto != Lto::ThinLocal {
83 if cgcx.opts.cg.prefer_dynamic {
85 .struct_err("cannot prefer dynamic linking when performing LTO")
87 "only 'staticlib', 'bin', and 'cdylib' outputs are \
91 return Err(FatalError);
94 // Make sure we actually can run LTO
95 for crate_type in cgcx.crate_types.iter() {
96 if !crate_type_allows_lto(*crate_type) {
97 let e = diag_handler.fatal(
98 "lto can only be run for executables, cdylibs and \
99 static library outputs",
105 for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
106 let exported_symbols =
107 cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
109 let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbol_white_list");
110 symbol_white_list.extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
113 let _timer = cgcx.prof.generic_activity("LLVM_lto_load_upstream_bitcode");
114 let archive = ArchiveRO::open(&path).expect("wanted an rlib");
115 let bytecodes = archive
117 .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c))))
118 .filter(|&(name, _)| name.ends_with(RLIB_BYTECODE_EXTENSION));
119 for (name, data) in bytecodes {
120 info!("adding bytecode {}", name);
121 let bc_encoded = data.data();
125 .extra_verbose_generic_activity(&format!("decode {}", name))
126 .run(|| match DecodedBytecode::new(bc_encoded) {
127 Ok(b) => Ok((b.bytecode(), b.identifier().to_string())),
128 Err(e) => Err(diag_handler.fatal(&e)),
130 let bc = SerializedModule::FromRlib(bc);
131 upstream_modules.push((bc, CString::new(id).unwrap()));
136 Ok((symbol_white_list, upstream_modules))
139 /// Performs fat LTO by merging all modules into a single one and returning it
140 /// for further optimization.
141 pub(crate) fn run_fat(
142 cgcx: &CodegenContext<LlvmCodegenBackend>,
143 modules: Vec<FatLTOInput<LlvmCodegenBackend>>,
144 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
145 ) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> {
146 let diag_handler = cgcx.create_diag_handler();
147 let (symbol_white_list, upstream_modules) = prepare_lto(cgcx, &diag_handler)?;
148 let symbol_white_list = symbol_white_list.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
149 fat_lto(cgcx, &diag_handler, modules, cached_modules, upstream_modules, &symbol_white_list)
152 /// Performs thin LTO by performing necessary global analysis and returning two
153 /// lists, one of the modules that need optimization and another for modules that
154 /// can simply be copied over from the incr. comp. cache.
155 pub(crate) fn run_thin(
156 cgcx: &CodegenContext<LlvmCodegenBackend>,
157 modules: Vec<(String, ThinBuffer)>,
158 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
159 ) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
160 let diag_handler = cgcx.create_diag_handler();
161 let (symbol_white_list, upstream_modules) = prepare_lto(cgcx, &diag_handler)?;
162 let symbol_white_list = symbol_white_list.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
163 if cgcx.opts.cg.linker_plugin_lto.enabled() {
165 "We should never reach this case if the LTO step \
166 is deferred to the linker"
169 thin_lto(cgcx, &diag_handler, modules, upstream_modules, cached_modules, &symbol_white_list)
172 pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) {
173 let name = module.name.clone();
174 let buffer = ThinBuffer::new(module.module_llvm.llmod());
179 cgcx: &CodegenContext<LlvmCodegenBackend>,
180 diag_handler: &Handler,
181 modules: Vec<FatLTOInput<LlvmCodegenBackend>>,
182 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
183 mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
184 symbol_white_list: &[*const libc::c_char],
185 ) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> {
186 let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module");
187 info!("going for a fat lto");
189 // Sort out all our lists of incoming modules into two lists.
191 // * `serialized_modules` (also and argument to this function) contains all
192 // modules that are serialized in-memory.
193 // * `in_memory` contains modules which are already parsed and in-memory,
194 // such as from multi-CGU builds.
196 // All of `cached_modules` (cached from previous incremental builds) can
197 // immediately go onto the `serialized_modules` modules list and then we can
198 // split the `modules` array into these two lists.
199 let mut in_memory = Vec::new();
200 serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
201 info!("pushing cached module {:?}", wp.cgu_name);
202 (buffer, CString::new(wp.cgu_name).unwrap())
204 for module in modules {
206 FatLTOInput::InMemory(m) => in_memory.push(m),
207 FatLTOInput::Serialized { name, buffer } => {
208 info!("pushing serialized module {:?}", name);
209 let buffer = SerializedModule::Local(buffer);
210 serialized_modules.push((buffer, CString::new(name).unwrap()));
215 // Find the "costliest" module and merge everything into that codegen unit.
216 // All the other modules will be serialized and reparsed into the new
217 // context, so this hopefully avoids serializing and parsing the largest
220 // Additionally use a regular module as the base here to ensure that various
221 // file copy operations in the backend work correctly. The only other kind
222 // of module here should be an allocator one, and if your crate is smaller
223 // than the allocator module then the size doesn't really matter anyway.
224 let costliest_module = in_memory
227 .filter(|&(_, module)| module.kind == ModuleKind::Regular)
229 let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) };
234 // If we found a costliest module, we're good to go. Otherwise all our
235 // inputs were serialized which could happen in the case, for example, that
236 // all our inputs were incrementally reread from the cache and we're just
237 // re-executing the LTO passes. If that's the case deserialize the first
238 // module and create a linker with it.
239 let module: ModuleCodegen<ModuleLlvm> = match costliest_module {
240 Some((_cost, i)) => in_memory.remove(i),
242 assert!(serialized_modules.len() > 0, "must have at least one serialized module");
243 let (buffer, name) = serialized_modules.remove(0);
244 info!("no in-memory regular modules to choose from, parsing {:?}", name);
246 module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), diag_handler)?,
247 name: name.into_string().unwrap(),
248 kind: ModuleKind::Regular,
252 let mut serialized_bitcode = Vec::new();
254 let (llcx, llmod) = {
255 let llvm = &module.module_llvm;
256 (&llvm.llcx, llvm.llmod())
258 info!("using {:?} as a base module", module.name);
260 // The linking steps below may produce errors and diagnostics within LLVM
261 // which we'd like to handle and print, so set up our diagnostic handlers
262 // (which get unregistered when they go out of scope below).
263 let _handler = DiagnosticHandlers::new(cgcx, diag_handler, llcx);
265 // For all other modules we codegened we'll need to link them into our own
266 // bitcode. All modules were codegened in their own LLVM context, however,
267 // and we want to move everything to the same LLVM context. Currently the
268 // way we know of to do that is to serialize them to a string and them parse
269 // them later. Not great but hey, that's why it's "fat" LTO, right?
270 for module in in_memory {
271 let buffer = ModuleBuffer::new(module.module_llvm.llmod());
272 let llmod_id = CString::new(&module.name[..]).unwrap();
273 serialized_modules.push((SerializedModule::Local(buffer), llmod_id));
275 // Sort the modules to ensure we produce deterministic results.
276 serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1));
278 // For all serialized bitcode files we parse them and link them in as we did
279 // above, this is all mostly handled in C++. Like above, though, we don't
280 // know much about the memory management here so we err on the side of being
281 // save and persist everything with the original module.
282 let mut linker = Linker::new(llmod);
283 for (bc_decoded, name) in serialized_modules {
284 let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_link_module");
285 info!("linking {:?}", name);
286 cgcx.prof.extra_verbose_generic_activity(&format!("ll link {:?}", name)).run(|| {
287 let data = bc_decoded.data();
288 linker.add(&data).map_err(|()| {
289 let msg = format!("failed to load bc of {:?}", name);
290 write::llvm_err(&diag_handler, &msg)
293 serialized_bitcode.push(bc_decoded);
296 save_temp_bitcode(&cgcx, &module, "lto.input");
298 // Internalize everything that *isn't* in our whitelist to help strip out
299 // more modules and such
301 let ptr = symbol_white_list.as_ptr();
302 llvm::LLVMRustRunRestrictionPass(
304 ptr as *const *const libc::c_char,
305 symbol_white_list.len() as libc::size_t,
307 save_temp_bitcode(&cgcx, &module, "lto.after-restriction");
310 if cgcx.no_landing_pads {
312 llvm::LLVMRustMarkAllFunctionsNounwind(llmod);
314 save_temp_bitcode(&cgcx, &module, "lto.after-nounwind");
318 Ok(LtoModuleCodegen::Fat { module: Some(module), _serialized_bitcode: serialized_bitcode })
321 struct Linker<'a>(&'a mut llvm::Linker<'a>);
324 fn new(llmod: &'a llvm::Module) -> Self {
325 unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) }
328 fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> {
330 if llvm::LLVMRustLinkerAdd(
332 bytecode.as_ptr() as *const libc::c_char,
343 impl Drop for Linker<'a> {
346 llvm::LLVMRustLinkerFree(&mut *(self.0 as *mut _));
351 /// Prepare "thin" LTO to get run on these modules.
353 /// The general structure of ThinLTO is quite different from the structure of
354 /// "fat" LTO above. With "fat" LTO all LLVM modules in question are merged into
355 /// one giant LLVM module, and then we run more optimization passes over this
356 /// big module after internalizing most symbols. Thin LTO, on the other hand,
357 /// avoid this large bottleneck through more targeted optimization.
359 /// At a high level Thin LTO looks like:
361 /// 1. Prepare a "summary" of each LLVM module in question which describes
362 /// the values inside, cost of the values, etc.
363 /// 2. Merge the summaries of all modules in question into one "index"
364 /// 3. Perform some global analysis on this index
365 /// 4. For each module, use the index and analysis calculated previously to
366 /// perform local transformations on the module, for example inlining
367 /// small functions from other modules.
368 /// 5. Run thin-specific optimization passes over each module, and then code
369 /// generate everything at the end.
371 /// The summary for each module is intended to be quite cheap, and the global
372 /// index is relatively quite cheap to create as well. As a result, the goal of
373 /// ThinLTO is to reduce the bottleneck on LTO and enable LTO to be used in more
374 /// situations. For example one cheap optimization is that we can parallelize
375 /// all codegen modules, easily making use of all the cores on a machine.
377 /// With all that in mind, the function here is designed at specifically just
378 /// calculating the *index* for ThinLTO. This index will then be shared amongst
379 /// all of the `LtoModuleCodegen` units returned below and destroyed once
380 /// they all go out of scope.
382 cgcx: &CodegenContext<LlvmCodegenBackend>,
383 diag_handler: &Handler,
384 modules: Vec<(String, ThinBuffer)>,
385 serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
386 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
387 symbol_white_list: &[*const libc::c_char],
388 ) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
389 let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis");
391 info!("going for that thin, thin LTO");
393 let green_modules: FxHashMap<_, _> =
394 cached_modules.iter().map(|&(_, ref wp)| (wp.cgu_name.clone(), wp.clone())).collect();
396 let full_scope_len = modules.len() + serialized_modules.len() + cached_modules.len();
397 let mut thin_buffers = Vec::with_capacity(modules.len());
398 let mut module_names = Vec::with_capacity(full_scope_len);
399 let mut thin_modules = Vec::with_capacity(full_scope_len);
401 for (i, (name, buffer)) in modules.into_iter().enumerate() {
402 info!("local module: {} - {}", i, name);
403 let cname = CString::new(name.clone()).unwrap();
404 thin_modules.push(llvm::ThinLTOModule {
405 identifier: cname.as_ptr(),
406 data: buffer.data().as_ptr(),
407 len: buffer.data().len(),
409 thin_buffers.push(buffer);
410 module_names.push(cname);
413 // FIXME: All upstream crates are deserialized internally in the
414 // function below to extract their summary and modules. Note that
415 // unlike the loop above we *must* decode and/or read something
416 // here as these are all just serialized files on disk. An
417 // improvement, however, to make here would be to store the
418 // module summary separately from the actual module itself. Right
419 // now this is store in one large bitcode file, and the entire
420 // file is deflate-compressed. We could try to bypass some of the
421 // decompression by storing the index uncompressed and only
422 // lazily decompressing the bytecode if necessary.
424 // Note that truly taking advantage of this optimization will
425 // likely be further down the road. We'd have to implement
426 // incremental ThinLTO first where we could actually avoid
427 // looking at upstream modules entirely sometimes (the contents,
428 // we must always unconditionally look at the index).
429 let mut serialized = Vec::with_capacity(serialized_modules.len() + cached_modules.len());
432 cached_modules.into_iter().map(|(sm, wp)| (sm, CString::new(wp.cgu_name).unwrap()));
434 for (module, name) in serialized_modules.into_iter().chain(cached_modules) {
435 info!("upstream or cached module {:?}", name);
436 thin_modules.push(llvm::ThinLTOModule {
437 identifier: name.as_ptr(),
438 data: module.data().as_ptr(),
439 len: module.data().len(),
441 serialized.push(module);
442 module_names.push(name);
446 assert_eq!(thin_modules.len(), module_names.len());
448 // Delegate to the C++ bindings to create some data here. Once this is a
449 // tried-and-true interface we may wish to try to upstream some of this
450 // to LLVM itself, right now we reimplement a lot of what they do
452 let data = llvm::LLVMRustCreateThinLTOData(
453 thin_modules.as_ptr(),
454 thin_modules.len() as u32,
455 symbol_white_list.as_ptr(),
456 symbol_white_list.len() as u32,
458 .ok_or_else(|| write::llvm_err(&diag_handler, "failed to prepare thin LTO context"))?;
460 info!("thin LTO data created");
462 let (import_map_path, prev_import_map, curr_import_map) =
463 if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir {
464 let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME);
465 // If previous imports have been deleted, or we get an IO error
466 // reading the file storing them, then we'll just use `None` as the
467 // prev_import_map, which will force the code to be recompiled.
469 if path.exists() { ThinLTOImports::load_from_file(&path).ok() } else { None };
470 let curr = ThinLTOImports::from_thin_lto_data(data);
471 (Some(path), prev, curr)
473 // If we don't compile incrementally, we don't need to load the
474 // import data from LLVM.
475 assert!(green_modules.is_empty());
476 let curr = ThinLTOImports::default();
479 info!("thin LTO import map loaded");
481 let data = ThinData(data);
483 // Throw our data in an `Arc` as we'll be sharing it across threads. We
484 // also put all memory referenced by the C++ data (buffers, ids, etc)
485 // into the arc as well. After this we'll create a thin module
486 // codegen per module in this data.
487 let shared = Arc::new(ThinShared {
490 serialized_modules: serialized,
494 let mut copy_jobs = vec![];
495 let mut opt_jobs = vec![];
497 info!("checking which modules can be-reused and which have to be re-optimized.");
498 for (module_index, module_name) in shared.module_names.iter().enumerate() {
499 let module_name = module_name_to_str(module_name);
501 // If (1.) the module hasn't changed, and (2.) none of the modules
502 // it imports from has changed, *and* (3.) the import-set itself has
503 // not changed from the previous compile when it was last
504 // ThinLTO'ed, then we can re-use the post-ThinLTO version of the
505 // module. Otherwise, freshly perform LTO optimization.
507 // This strategy means we can always save the computed imports as
508 // canon: when we reuse the post-ThinLTO version, condition (3.)
509 // ensures that the curent import set is the same as the previous
510 // one. (And of course, when we don't reuse the post-ThinLTO
511 // version, the current import set *is* the correct one, since we
512 // are doing the ThinLTO in this current compilation cycle.)
514 // See rust-lang/rust#59535.
515 if let (Some(prev_import_map), true) =
516 (prev_import_map.as_ref(), green_modules.contains_key(module_name))
518 assert!(cgcx.incr_comp_session_dir.is_some());
520 let prev_imports = prev_import_map.modules_imported_by(module_name);
521 let curr_imports = curr_import_map.modules_imported_by(module_name);
522 let imports_all_green = curr_imports
524 .all(|imported_module| green_modules.contains_key(imported_module));
526 if imports_all_green && equivalent_as_sets(prev_imports, curr_imports) {
527 let work_product = green_modules[module_name].clone();
528 copy_jobs.push(work_product);
529 info!(" - {}: re-used", module_name);
530 assert!(cgcx.incr_comp_session_dir.is_some());
531 cgcx.cgu_reuse_tracker.set_actual_reuse(module_name, CguReuse::PostLto);
536 info!(" - {}: re-compiled", module_name);
537 opt_jobs.push(LtoModuleCodegen::Thin(ThinModule {
538 shared: shared.clone(),
543 // Save the curent ThinLTO import information for the next compilation
544 // session, overwriting the previous serialized imports (if any).
545 if let Some(path) = import_map_path {
546 if let Err(err) = curr_import_map.save_to_file(&path) {
547 let msg = format!("Error while writing ThinLTO import data: {}", err);
548 return Err(write::llvm_err(&diag_handler, &msg));
552 Ok((opt_jobs, copy_jobs))
556 /// Given two slices, each with no repeat elements. returns true if and only if
557 /// the two slices have the same contents when considered as sets (i.e. when
558 /// element order is disregarded).
559 fn equivalent_as_sets(a: &[String], b: &[String]) -> bool {
560 // cheap path: unequal lengths means cannot possibly be set equivalent.
561 if a.len() != b.len() {
564 // fast path: before building new things, check if inputs are equivalent as is.
568 // slow path: general set comparison.
569 let a: FxHashSet<&str> = a.iter().map(|s| s.as_str()).collect();
570 let b: FxHashSet<&str> = b.iter().map(|s| s.as_str()).collect();
574 pub(crate) fn run_pass_manager(
575 cgcx: &CodegenContext<LlvmCodegenBackend>,
576 module: &ModuleCodegen<ModuleLlvm>,
577 config: &ModuleConfig,
580 // Now we have one massive module inside of llmod. Time to run the
581 // LTO-specific optimization passes that LLVM provides.
583 // This code is based off the code found in llvm's LTO code generator:
584 // tools/lto/LTOCodeGenerator.cpp
585 debug!("running the pass manager");
587 let pm = llvm::LLVMCreatePassManager();
588 llvm::LLVMAddAnalysisPasses(module.module_llvm.tm, pm);
590 if config.verify_llvm_ir {
591 let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast());
592 llvm::LLVMRustAddPass(pm, pass.unwrap());
595 // When optimizing for LTO we don't actually pass in `-O0`, but we force
596 // it to always happen at least with `-O1`.
598 // With ThinLTO we mess around a lot with symbol visibility in a way
599 // that will actually cause linking failures if we optimize at O0 which
600 // notable is lacking in dead code elimination. To ensure we at least
601 // get some optimizations and correctly link we forcibly switch to `-O1`
602 // to get dead code elimination.
604 // Note that in general this shouldn't matter too much as you typically
605 // only turn on ThinLTO when you're compiling with optimizations
607 let opt_level = config
609 .map(|x| to_llvm_opt_settings(x).0)
610 .unwrap_or(llvm::CodeGenOptLevel::None);
611 let opt_level = match opt_level {
612 llvm::CodeGenOptLevel::None => llvm::CodeGenOptLevel::Less,
615 with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| {
617 llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm);
619 llvm::LLVMPassManagerBuilderPopulateLTOPassManager(
620 b, pm, /* Internalize = */ False, /* RunInliner = */ True,
625 // We always generate bitcode through ThinLTOBuffers,
626 // which do not support anonymous globals
627 if config.bitcode_needed() {
628 let pass = llvm::LLVMRustFindAndCreatePass("name-anon-globals\0".as_ptr().cast());
629 llvm::LLVMRustAddPass(pm, pass.unwrap());
632 if config.verify_llvm_ir {
633 let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast());
634 llvm::LLVMRustAddPass(pm, pass.unwrap());
638 .extra_verbose_generic_activity("LTO_passes")
639 .run(|| llvm::LLVMRunPassManager(pm, module.module_llvm.llmod()));
641 llvm::LLVMDisposePassManager(pm);
646 pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer);
648 unsafe impl Send for ModuleBuffer {}
649 unsafe impl Sync for ModuleBuffer {}
652 pub fn new(m: &llvm::Module) -> ModuleBuffer {
653 ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) })
657 impl ModuleBufferMethods for ModuleBuffer {
658 fn data(&self) -> &[u8] {
660 let ptr = llvm::LLVMRustModuleBufferPtr(self.0);
661 let len = llvm::LLVMRustModuleBufferLen(self.0);
662 slice::from_raw_parts(ptr, len)
667 impl Drop for ModuleBuffer {
670 llvm::LLVMRustModuleBufferFree(&mut *(self.0 as *mut _));
675 pub struct ThinData(&'static mut llvm::ThinLTOData);
677 unsafe impl Send for ThinData {}
678 unsafe impl Sync for ThinData {}
680 impl Drop for ThinData {
683 llvm::LLVMRustFreeThinLTOData(&mut *(self.0 as *mut _));
688 pub struct ThinBuffer(&'static mut llvm::ThinLTOBuffer);
690 unsafe impl Send for ThinBuffer {}
691 unsafe impl Sync for ThinBuffer {}
694 pub fn new(m: &llvm::Module) -> ThinBuffer {
696 let buffer = llvm::LLVMRustThinLTOBufferCreate(m);
702 impl ThinBufferMethods for ThinBuffer {
703 fn data(&self) -> &[u8] {
705 let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _;
706 let len = llvm::LLVMRustThinLTOBufferLen(self.0);
707 slice::from_raw_parts(ptr, len)
712 impl Drop for ThinBuffer {
715 llvm::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _));
720 pub unsafe fn optimize_thin_module(
721 thin_module: &mut ThinModule<LlvmCodegenBackend>,
722 cgcx: &CodegenContext<LlvmCodegenBackend>,
723 ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
724 let diag_handler = cgcx.create_diag_handler();
725 let tm = (cgcx.tm_factory.0)().map_err(|e| write::llvm_err(&diag_handler, &e))?;
727 // Right now the implementation we've got only works over serialized
728 // modules, so we create a fresh new LLVM context and parse the module
729 // into that context. One day, however, we may do this for upstream
730 // crates but for locally codegened modules we may be able to reuse
731 // that LLVM Context and Module.
732 let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
733 let llmod_raw = parse_module(
735 &thin_module.shared.module_names[thin_module.idx],
739 let module = ModuleCodegen {
740 module_llvm: ModuleLlvm { llmod_raw, llcx, tm },
741 name: thin_module.name().to_string(),
742 kind: ModuleKind::Regular,
745 let llmod = module.module_llvm.llmod();
746 save_temp_bitcode(&cgcx, &module, "thin-lto-input");
748 // Before we do much else find the "main" `DICompileUnit` that we'll be
749 // using below. If we find more than one though then rustc has changed
750 // in a way we're not ready for, so generate an ICE by returning
752 let mut cu1 = ptr::null_mut();
753 let mut cu2 = ptr::null_mut();
754 llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2);
756 let msg = "multiple source DICompileUnits found";
757 return Err(write::llvm_err(&diag_handler, msg));
760 // Like with "fat" LTO, get some better optimizations if landing pads
761 // are disabled by removing all landing pads.
762 if cgcx.no_landing_pads {
763 let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_remove_landing_pads");
764 llvm::LLVMRustMarkAllFunctionsNounwind(llmod);
765 save_temp_bitcode(&cgcx, &module, "thin-lto-after-nounwind");
768 // Up next comes the per-module local analyses that we do for Thin LTO.
769 // Each of these functions is basically copied from the LLVM
770 // implementation and then tailored to suit this implementation. Ideally
771 // each of these would be supported by upstream LLVM but that's perhaps
772 // a patch for another day!
774 // You can find some more comments about these functions in the LLVM
775 // bindings we've got (currently `PassWrapper.cpp`)
777 let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_rename");
778 if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod) {
779 let msg = "failed to prepare thin LTO module";
780 return Err(write::llvm_err(&diag_handler, msg));
782 save_temp_bitcode(cgcx, &module, "thin-lto-after-rename");
786 let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_resolve_weak");
787 if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) {
788 let msg = "failed to prepare thin LTO module";
789 return Err(write::llvm_err(&diag_handler, msg));
791 save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve");
795 let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_internalize");
796 if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) {
797 let msg = "failed to prepare thin LTO module";
798 return Err(write::llvm_err(&diag_handler, msg));
800 save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize");
804 let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_import");
805 if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod) {
806 let msg = "failed to prepare thin LTO module";
807 return Err(write::llvm_err(&diag_handler, msg));
809 save_temp_bitcode(cgcx, &module, "thin-lto-after-import");
812 // Ok now this is a bit unfortunate. This is also something you won't
813 // find upstream in LLVM's ThinLTO passes! This is a hack for now to
814 // work around bugs in LLVM.
816 // First discovered in #45511 it was found that as part of ThinLTO
817 // importing passes LLVM will import `DICompileUnit` metadata
818 // information across modules. This means that we'll be working with one
819 // LLVM module that has multiple `DICompileUnit` instances in it (a
820 // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of
821 // bugs in LLVM's backend which generates invalid DWARF in a situation
824 // https://bugs.llvm.org/show_bug.cgi?id=35212
825 // https://bugs.llvm.org/show_bug.cgi?id=35562
827 // While the first bug there is fixed the second ended up causing #46346
828 // which was basically a resurgence of #45511 after LLVM's bug 35212 was
831 // This function below is a huge hack around this problem. The function
832 // below is defined in `PassWrapper.cpp` and will basically "merge"
833 // all `DICompileUnit` instances in a module. Basically it'll take all
834 // the objects, rewrite all pointers of `DISubprogram` to point to the
835 // first `DICompileUnit`, and then delete all the other units.
837 // This is probably mangling to the debug info slightly (but hopefully
838 // not too much) but for now at least gets LLVM to emit valid DWARF (or
839 // so it appears). Hopefully we can remove this once upstream bugs are
842 let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_patch_debuginfo");
843 llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1);
844 save_temp_bitcode(cgcx, &module, "thin-lto-after-patch");
847 // Alright now that we've done everything related to the ThinLTO
848 // analysis it's time to run some optimizations! Here we use the same
849 // `run_pass_manager` as the "fat" LTO above except that we tell it to
850 // populate a thin-specific pass manager, which presumably LLVM treats a
851 // little differently.
853 let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_optimize");
854 info!("running thin lto passes over {}", module.name);
855 let config = cgcx.config(module.kind);
856 run_pass_manager(cgcx, &module, config, true);
857 save_temp_bitcode(cgcx, &module, "thin-lto-after-pm");
863 #[derive(Debug, Default)]
864 pub struct ThinLTOImports {
865 // key = llvm name of importing module, value = list of modules it imports from
866 imports: FxHashMap<String, Vec<String>>,
869 impl ThinLTOImports {
870 fn modules_imported_by(&self, llvm_module_name: &str) -> &[String] {
871 self.imports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[])
874 fn save_to_file(&self, path: &Path) -> io::Result<()> {
876 let file = File::create(path)?;
877 let mut writer = io::BufWriter::new(file);
878 for (importing_module_name, imported_modules) in &self.imports {
879 writeln!(writer, "{}", importing_module_name)?;
880 for imported_module in imported_modules {
881 writeln!(writer, " {}", imported_module)?;
888 fn load_from_file(path: &Path) -> io::Result<ThinLTOImports> {
889 use std::io::BufRead;
890 let mut imports = FxHashMap::default();
891 let mut current_module = None;
892 let mut current_imports = vec![];
893 let file = File::open(path)?;
894 for line in io::BufReader::new(file).lines() {
897 let importing_module = current_module.take().expect("Importing module not set");
898 imports.insert(importing_module, mem::replace(&mut current_imports, vec![]));
899 } else if line.starts_with(" ") {
900 // Space marks an imported module
901 assert_ne!(current_module, None);
902 current_imports.push(line.trim().to_string());
904 // Otherwise, beginning of a new module (must be start or follow empty line)
905 assert_eq!(current_module, None);
906 current_module = Some(line.trim().to_string());
909 Ok(ThinLTOImports { imports })
912 /// Loads the ThinLTO import map from ThinLTOData.
913 unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports {
914 unsafe extern "C" fn imported_module_callback(
915 payload: *mut libc::c_void,
916 importing_module_name: *const libc::c_char,
917 imported_module_name: *const libc::c_char,
919 let map = &mut *(payload as *mut ThinLTOImports);
920 let importing_module_name = CStr::from_ptr(importing_module_name);
921 let importing_module_name = module_name_to_str(&importing_module_name);
922 let imported_module_name = CStr::from_ptr(imported_module_name);
923 let imported_module_name = module_name_to_str(&imported_module_name);
925 if !map.imports.contains_key(importing_module_name) {
926 map.imports.insert(importing_module_name.to_owned(), vec![]);
930 .get_mut(importing_module_name)
932 .push(imported_module_name.to_owned());
934 let mut map = ThinLTOImports::default();
935 llvm::LLVMRustGetThinLTOModuleImports(
937 imported_module_callback,
938 &mut map as *mut _ as *mut libc::c_void,
944 fn module_name_to_str(c_str: &CStr) -> &str {
945 c_str.to_str().unwrap_or_else(|e| {
946 bug!("Encountered non-utf8 LLVM module name `{}`: {}", c_str.to_string_lossy(), e)
950 pub fn parse_module<'a>(
951 cx: &'a llvm::Context,
954 diag_handler: &Handler,
955 ) -> Result<&'a llvm::Module, FatalError> {
957 llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else(
959 let msg = "failed to parse bitcode for LTO module";
960 write::llvm_err(&diag_handler, msg)