6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/StandardInstrumentations.h"
21 #include "llvm/Support/CBindingWrapping.h"
22 #include "llvm/Support/FileSystem.h"
23 #include "llvm/Support/Host.h"
24 #include "llvm/Target/TargetMachine.h"
25 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
26 #include "llvm/Transforms/IPO/AlwaysInliner.h"
27 #include "llvm/Transforms/IPO/FunctionImport.h"
28 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
29 #include "llvm/LTO/LTO.h"
30 #include "llvm-c/Transforms/PassManagerBuilder.h"
32 #include "llvm/Transforms/Instrumentation.h"
33 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
34 #include "llvm/Support/TimeProfiler.h"
35 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
36 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
37 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
38 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
39 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
43 typedef struct LLVMOpaquePass *LLVMPassRef;
44 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
46 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
47 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
48 #if LLVM_VERSION_LT(11, 0)
49 DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
50 LLVMPassManagerBuilderRef)
53 extern "C" void LLVMInitializePasses() {
54 PassRegistry &Registry = *PassRegistry::getPassRegistry();
55 initializeCore(Registry);
56 initializeCodeGen(Registry);
57 initializeScalarOpts(Registry);
58 initializeVectorization(Registry);
59 initializeIPO(Registry);
60 initializeAnalysis(Registry);
61 initializeTransformUtils(Registry);
62 initializeInstCombine(Registry);
63 initializeInstrumentation(Registry);
64 initializeTarget(Registry);
67 extern "C" void LLVMTimeTraceProfilerInitialize() {
68 #if LLVM_VERSION_GE(10, 0)
69 timeTraceProfilerInitialize(
70 /* TimeTraceGranularity */ 0,
71 /* ProcName */ "rustc");
73 timeTraceProfilerInitialize();
77 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
78 StringRef FN(FileName);
80 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
82 timeTraceProfilerWrite(OS);
83 timeTraceProfilerCleanup();
86 enum class LLVMRustPassKind {
92 static LLVMRustPassKind toRust(PassKind Kind) {
95 return LLVMRustPassKind::Function;
97 return LLVMRustPassKind::Module;
99 return LLVMRustPassKind::Other;
103 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
104 StringRef SR(PassName);
105 PassRegistry *PR = PassRegistry::getPassRegistry();
107 const PassInfo *PI = PR->getPassInfo(SR);
109 return wrap(PI->createPass());
114 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
115 const bool CompileKernel = false;
116 const bool UseAfterScope = true;
118 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
121 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
122 const bool CompileKernel = false;
124 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
127 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
128 const bool CompileKernel = false;
130 return wrap(createMemorySanitizerLegacyPassPass(
131 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
134 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
135 return wrap(createThreadSanitizerLegacyPassPass());
138 extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) {
139 const bool CompileKernel = false;
141 return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover));
144 extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
146 Pass *Pass = unwrap(RustPass);
147 return toRust(Pass->getPassKind());
150 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
152 Pass *Pass = unwrap(RustPass);
153 PassManagerBase *PMB = unwrap(PMR);
158 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
159 LLVMPassManagerBuilderRef PMBR,
160 LLVMPassManagerRef PMR
162 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
166 void LLVMRustAddLastExtensionPasses(
167 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
168 auto AddExtensionPasses = [Passes, NumPasses](
169 const PassManagerBuilder &Builder, PassManagerBase &PM) {
170 for (size_t I = 0; I < NumPasses; I++) {
171 PM.add(unwrap(Passes[I]));
174 // Add the passes to both of the pre-finalization extension points,
175 // so they are run for optimized and non-optimized builds.
176 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
178 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
182 #ifdef LLVM_COMPONENT_X86
183 #define SUBTARGET_X86 SUBTARGET(X86)
185 #define SUBTARGET_X86
188 #ifdef LLVM_COMPONENT_ARM
189 #define SUBTARGET_ARM SUBTARGET(ARM)
191 #define SUBTARGET_ARM
194 #ifdef LLVM_COMPONENT_AARCH64
195 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
197 #define SUBTARGET_AARCH64
200 #ifdef LLVM_COMPONENT_AVR
201 #define SUBTARGET_AVR SUBTARGET(AVR)
203 #define SUBTARGET_AVR
206 #ifdef LLVM_COMPONENT_MIPS
207 #define SUBTARGET_MIPS SUBTARGET(Mips)
209 #define SUBTARGET_MIPS
212 #ifdef LLVM_COMPONENT_POWERPC
213 #define SUBTARGET_PPC SUBTARGET(PPC)
215 #define SUBTARGET_PPC
218 #ifdef LLVM_COMPONENT_SYSTEMZ
219 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
221 #define SUBTARGET_SYSTEMZ
224 #ifdef LLVM_COMPONENT_MSP430
225 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
227 #define SUBTARGET_MSP430
230 #ifdef LLVM_COMPONENT_RISCV
231 #define SUBTARGET_RISCV SUBTARGET(RISCV)
233 #define SUBTARGET_RISCV
236 #ifdef LLVM_COMPONENT_SPARC
237 #define SUBTARGET_SPARC SUBTARGET(Sparc)
239 #define SUBTARGET_SPARC
242 #ifdef LLVM_COMPONENT_HEXAGON
243 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
245 #define SUBTARGET_HEXAGON
248 #define GEN_SUBTARGETS \
261 #define SUBTARGET(x) \
263 extern const SubtargetFeatureKV x##FeatureKV[]; \
264 extern const SubtargetFeatureKV x##SubTypeKV[]; \
270 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
271 const char *Feature) {
272 TargetMachine *Target = unwrap(TM);
273 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
274 return MCInfo->checkFeatures(std::string("+") + Feature);
277 enum class LLVMRustCodeModel {
286 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
288 case LLVMRustCodeModel::Tiny:
289 return CodeModel::Tiny;
290 case LLVMRustCodeModel::Small:
291 return CodeModel::Small;
292 case LLVMRustCodeModel::Kernel:
293 return CodeModel::Kernel;
294 case LLVMRustCodeModel::Medium:
295 return CodeModel::Medium;
296 case LLVMRustCodeModel::Large:
297 return CodeModel::Large;
298 case LLVMRustCodeModel::None:
301 report_fatal_error("Bad CodeModel.");
305 enum class LLVMRustCodeGenOptLevel {
312 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
314 case LLVMRustCodeGenOptLevel::None:
315 return CodeGenOpt::None;
316 case LLVMRustCodeGenOptLevel::Less:
317 return CodeGenOpt::Less;
318 case LLVMRustCodeGenOptLevel::Default:
319 return CodeGenOpt::Default;
320 case LLVMRustCodeGenOptLevel::Aggressive:
321 return CodeGenOpt::Aggressive;
323 report_fatal_error("Bad CodeGenOptLevel.");
327 enum class LLVMRustPassBuilderOptLevel {
336 static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
338 case LLVMRustPassBuilderOptLevel::O0:
339 return PassBuilder::OptimizationLevel::O0;
340 case LLVMRustPassBuilderOptLevel::O1:
341 return PassBuilder::OptimizationLevel::O1;
342 case LLVMRustPassBuilderOptLevel::O2:
343 return PassBuilder::OptimizationLevel::O2;
344 case LLVMRustPassBuilderOptLevel::O3:
345 return PassBuilder::OptimizationLevel::O3;
346 case LLVMRustPassBuilderOptLevel::Os:
347 return PassBuilder::OptimizationLevel::Os;
348 case LLVMRustPassBuilderOptLevel::Oz:
349 return PassBuilder::OptimizationLevel::Oz;
351 report_fatal_error("Bad PassBuilderOptLevel.");
355 enum class LLVMRustRelocModel {
364 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
366 case LLVMRustRelocModel::Static:
367 return Reloc::Static;
368 case LLVMRustRelocModel::PIC:
370 case LLVMRustRelocModel::DynamicNoPic:
371 return Reloc::DynamicNoPIC;
372 case LLVMRustRelocModel::ROPI:
374 case LLVMRustRelocModel::RWPI:
376 case LLVMRustRelocModel::ROPIRWPI:
377 return Reloc::ROPI_RWPI;
379 report_fatal_error("Bad RelocModel.");
383 /// getLongestEntryLength - Return the length of the longest entry in the table.
384 template<typename KV>
385 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
387 for (auto &I : Table)
388 MaxLen = std::max(MaxLen, std::strlen(I.Key));
392 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
393 const TargetMachine *Target = unwrap(TM);
394 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
395 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
396 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
397 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
398 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
400 printf("Available CPUs for this target:\n");
401 if (HostArch == TargetArch) {
402 const StringRef HostCPU = sys::getHostCPUName();
403 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
404 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
406 for (auto &CPU : CPUTable)
407 printf(" %-*s\n", MaxCPULen, CPU.Key);
411 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
412 const TargetMachine *Target = unwrap(TM);
413 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
414 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
415 unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
417 printf("Available features for this target:\n");
418 for (auto &Feature : FeatTable)
419 printf(" %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
420 printf("\nRust-specific features:\n");
421 printf(" %-*s - %s.\n",
424 "Enables libraries with C Run-time Libraries(CRT) to be statically linked"
428 printf("Use +feature to enable a feature, or -feature to disable it.\n"
429 "For example, rustc -C -target-cpu=mycpu -C "
430 "target-feature=+feature1,-feature2\n\n");
435 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
436 printf("Target CPU help is not supported by this LLVM version.\n\n");
439 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
440 printf("Target features help is not supported by this LLVM version.\n\n");
444 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
445 StringRef Name = sys::getHostCPUName();
450 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
451 const char *TripleStr, const char *CPU, const char *Feature,
452 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
453 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
454 bool FunctionSections,
456 bool TrapUnreachable,
459 bool EmitStackSizeSection,
460 bool RelaxELFRelocations,
462 const char *SplitDwarfFile) {
464 auto OptLevel = fromRust(RustOptLevel);
465 auto RM = fromRust(RustReloc);
466 auto CM = fromRust(RustCM);
469 Triple Trip(Triple::normalize(TripleStr));
470 const llvm::Target *TheTarget =
471 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
472 if (TheTarget == nullptr) {
473 LLVMRustSetLastError(Error.c_str());
477 TargetOptions Options;
479 Options.FloatABIType = FloatABI::Default;
481 Options.FloatABIType = FloatABI::Soft;
483 Options.DataSections = DataSections;
484 Options.FunctionSections = FunctionSections;
485 Options.MCOptions.AsmVerbose = AsmComments;
486 Options.MCOptions.PreserveAsmComments = AsmComments;
487 Options.MCOptions.ABIName = ABIStr;
488 if (SplitDwarfFile) {
489 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
491 Options.RelaxELFRelocations = RelaxELFRelocations;
492 Options.UseInitArray = UseInitArray;
494 if (TrapUnreachable) {
495 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
496 // This limits the extent of possible undefined behavior in some cases, as
497 // it prevents control flow from "falling through" into whatever code
498 // happens to be laid out next in memory.
499 Options.TrapUnreachable = true;
503 Options.ThreadModel = ThreadModel::Single;
506 Options.EmitStackSizeSection = EmitStackSizeSection;
508 TargetMachine *TM = TheTarget->createTargetMachine(
509 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
513 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
517 extern "C" void LLVMRustConfigurePassManagerBuilder(
518 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
519 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
520 const char* PGOGenPath, const char* PGOUsePath) {
521 unwrap(PMBR)->MergeFunctions = MergeFunctions;
522 unwrap(PMBR)->SLPVectorize = SLPVectorize;
523 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
524 unwrap(PMBR)->LoopVectorize = LoopVectorize;
525 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
529 unwrap(PMBR)->EnablePGOInstrGen = true;
530 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
534 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
538 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
539 // field of a PassManagerBuilder, we expose our own method of doing so.
540 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
542 bool DisableSimplifyLibCalls) {
543 Triple TargetTriple(unwrap(M)->getTargetTriple());
544 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
545 if (DisableSimplifyLibCalls)
546 TLI->disableAllFunctions();
547 unwrap(PMBR)->LibraryInfo = TLI;
550 // Unfortunately, the LLVM C API doesn't provide a way to create the
551 // TargetLibraryInfo pass, so we use this method to do so.
552 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
553 bool DisableSimplifyLibCalls) {
554 Triple TargetTriple(unwrap(M)->getTargetTriple());
555 TargetLibraryInfoImpl TLII(TargetTriple);
556 if (DisableSimplifyLibCalls)
557 TLII.disableAllFunctions();
558 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
561 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
562 // all the functions in a module, so we do that manually here. You'll find
563 // similar code in clang's BackendUtil.cpp file.
564 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
566 llvm::legacy::FunctionPassManager *P =
567 unwrap<llvm::legacy::FunctionPassManager>(PMR);
568 P->doInitialization();
570 // Upgrade all calls to old intrinsics first.
571 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
572 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
574 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
576 if (!I->isDeclaration())
582 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
583 // Initializing the command-line options more than once is not allowed. So,
584 // check if they've already been initialized. (This could happen if we're
585 // being called from rustpkg, for example). If the arguments change, then
586 // that's just kinda unfortunate.
587 static bool Initialized = false;
591 cl::ParseCommandLineOptions(Argc, Argv);
594 enum class LLVMRustFileType {
599 #if LLVM_VERSION_GE(10, 0)
600 static CodeGenFileType fromRust(LLVMRustFileType Type) {
602 case LLVMRustFileType::AssemblyFile:
603 return CGFT_AssemblyFile;
604 case LLVMRustFileType::ObjectFile:
605 return CGFT_ObjectFile;
607 report_fatal_error("Bad FileType.");
611 static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
613 case LLVMRustFileType::AssemblyFile:
614 return TargetMachine::CGFT_AssemblyFile;
615 case LLVMRustFileType::ObjectFile:
616 return TargetMachine::CGFT_ObjectFile;
618 report_fatal_error("Bad FileType.");
623 extern "C" LLVMRustResult
624 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
625 LLVMModuleRef M, const char *Path, const char *DwoPath,
626 LLVMRustFileType RustFileType) {
627 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
628 auto FileType = fromRust(RustFileType);
630 std::string ErrorInfo;
632 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
634 ErrorInfo = EC.message();
635 if (ErrorInfo != "") {
636 LLVMRustSetLastError(ErrorInfo.c_str());
637 return LLVMRustResult::Failure;
640 buffer_ostream BOS(OS);
642 raw_fd_ostream DOS(DwoPath, EC, sys::fs::F_None);
645 ErrorInfo = EC.message();
646 if (ErrorInfo != "") {
647 LLVMRustSetLastError(ErrorInfo.c_str());
648 return LLVMRustResult::Failure;
650 buffer_ostream DBOS(DOS);
651 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
654 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
658 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
659 // stream (OS), so the only real safe place to delete this is here? Don't we
660 // wish this was written in Rust?
661 LLVMDisposePassManager(PMR);
662 return LLVMRustResult::Success;
665 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
666 const char*, // pass name
667 const char*); // IR name
668 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
670 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
671 if (any_isa<const Module *>(WrappedIr))
672 return any_cast<const Module *>(WrappedIr)->getName().str();
673 if (any_isa<const Function *>(WrappedIr))
674 return any_cast<const Function *>(WrappedIr)->getName().str();
675 if (any_isa<const Loop *>(WrappedIr))
676 return any_cast<const Loop *>(WrappedIr)->getName().str();
677 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
678 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
683 void LLVMSelfProfileInitializeCallbacks(
684 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
685 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
686 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
687 #if LLVM_VERSION_GE(12, 0)
688 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
689 StringRef Pass, llvm::Any Ir) {
690 std::string PassName = Pass.str();
691 std::string IrName = LLVMRustwrappedIrGetName(Ir);
692 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
695 PIC.registerAfterPassCallback(
696 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
697 const PreservedAnalyses &Preserved) {
698 AfterPassCallback(LlvmSelfProfiler);
701 PIC.registerAfterPassInvalidatedCallback(
702 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
703 AfterPassCallback(LlvmSelfProfiler);
706 PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback](
707 StringRef Pass, llvm::Any Ir) {
708 std::string PassName = Pass.str();
709 std::string IrName = LLVMRustwrappedIrGetName(Ir);
710 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
714 PIC.registerAfterPassCallback(
715 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
716 AfterPassCallback(LlvmSelfProfiler);
719 PIC.registerAfterPassInvalidatedCallback(
720 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) {
721 AfterPassCallback(LlvmSelfProfiler);
725 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
726 StringRef Pass, llvm::Any Ir) {
727 std::string PassName = Pass.str();
728 std::string IrName = LLVMRustwrappedIrGetName(Ir);
729 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
732 PIC.registerAfterAnalysisCallback(
733 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
734 AfterPassCallback(LlvmSelfProfiler);
738 enum class LLVMRustOptStage {
746 struct LLVMRustSanitizerOptions {
747 bool SanitizeAddress;
748 bool SanitizeAddressRecover;
750 bool SanitizeMemoryRecover;
751 int SanitizeMemoryTrackOrigins;
753 bool SanitizeHWAddress;
754 bool SanitizeHWAddressRecover;
758 LLVMRustOptimizeWithNewPassManager(
759 LLVMModuleRef ModuleRef,
760 LLVMTargetMachineRef TMRef,
761 LLVMRustPassBuilderOptLevel OptLevelRust,
762 LLVMRustOptStage OptStage,
763 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
764 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
765 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
766 LLVMRustSanitizerOptions *SanitizerOptions,
767 const char *PGOGenPath, const char *PGOUsePath,
768 void* LlvmSelfProfiler,
769 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
770 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
771 Module *TheModule = unwrap(ModuleRef);
772 TargetMachine *TM = unwrap(TMRef);
773 PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust);
776 PipelineTuningOptions PTO;
777 PTO.LoopUnrolling = UnrollLoops;
778 PTO.LoopInterleaving = UnrollLoops;
779 PTO.LoopVectorization = LoopVectorize;
780 PTO.SLPVectorization = SLPVectorize;
781 #if LLVM_VERSION_GE(12, 0)
782 PTO.MergeFunctions = MergeFunctions;
784 // MergeFunctions is not supported by NewPM in older LLVM versions.
785 (void) MergeFunctions;
788 // FIXME: We may want to expose this as an option.
789 bool DebugPassManager = false;
791 PassInstrumentationCallbacks PIC;
792 #if LLVM_VERSION_GE(12, 0)
793 StandardInstrumentations SI(DebugPassManager);
795 StandardInstrumentations SI;
797 SI.registerCallbacks(PIC);
799 if (LlvmSelfProfiler){
800 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
803 Optional<PGOOptions> PGOOpt;
806 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr);
807 } else if (PGOUsePath) {
809 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse);
812 #if LLVM_VERSION_GE(12, 0)
813 PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC);
815 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
818 LoopAnalysisManager LAM(DebugPassManager);
819 FunctionAnalysisManager FAM(DebugPassManager);
820 CGSCCAnalysisManager CGAM(DebugPassManager);
821 ModuleAnalysisManager MAM(DebugPassManager);
823 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
825 Triple TargetTriple(TheModule->getTargetTriple());
826 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
827 if (DisableSimplifyLibCalls)
828 TLII->disableAllFunctions();
829 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
831 PB.registerModuleAnalyses(MAM);
832 PB.registerCGSCCAnalyses(CGAM);
833 PB.registerFunctionAnalyses(FAM);
834 PB.registerLoopAnalyses(LAM);
835 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
837 // We manually collect pipeline callbacks so we can apply them at O0, where the
838 // PassBuilder does not create a pipeline.
839 std::vector<std::function<void(ModulePassManager &, PassBuilder::OptimizationLevel)>>
840 PipelineStartEPCallbacks;
841 #if LLVM_VERSION_GE(11, 0)
842 std::vector<std::function<void(ModulePassManager &, PassBuilder::OptimizationLevel)>>
843 OptimizerLastEPCallbacks;
845 std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>>
846 OptimizerLastEPCallbacks;
850 PipelineStartEPCallbacks.push_back(
851 [VerifyIR](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
852 MPM.addPass(VerifierPass());
857 if (SanitizerOptions) {
858 if (SanitizerOptions->SanitizeMemory) {
859 MemorySanitizerOptions Options(
860 SanitizerOptions->SanitizeMemoryTrackOrigins,
861 SanitizerOptions->SanitizeMemoryRecover,
862 /*CompileKernel=*/false);
863 #if LLVM_VERSION_GE(11, 0)
864 OptimizerLastEPCallbacks.push_back(
865 [Options](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
866 MPM.addPass(MemorySanitizerPass(Options));
867 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
871 #if LLVM_VERSION_GE(10, 0)
872 PipelineStartEPCallbacks.push_back(
873 [Options](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
874 MPM.addPass(MemorySanitizerPass(Options));
878 OptimizerLastEPCallbacks.push_back(
879 [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
880 FPM.addPass(MemorySanitizerPass(Options));
886 if (SanitizerOptions->SanitizeThread) {
887 #if LLVM_VERSION_GE(11, 0)
888 OptimizerLastEPCallbacks.push_back(
889 [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
890 MPM.addPass(ThreadSanitizerPass());
891 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
895 #if LLVM_VERSION_GE(10, 0)
896 PipelineStartEPCallbacks.push_back(
897 [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
898 MPM.addPass(ThreadSanitizerPass());
902 OptimizerLastEPCallbacks.push_back(
903 [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
904 FPM.addPass(ThreadSanitizerPass());
910 if (SanitizerOptions->SanitizeAddress) {
911 #if LLVM_VERSION_GE(11, 0)
912 OptimizerLastEPCallbacks.push_back(
913 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
914 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
915 MPM.addPass(ModuleAddressSanitizerPass(
916 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
917 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
918 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
919 /*UseAfterScope=*/true)));
923 PipelineStartEPCallbacks.push_back(
924 [&](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
925 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
928 OptimizerLastEPCallbacks.push_back(
929 [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
930 FPM.addPass(AddressSanitizerPass(
931 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
932 /*UseAfterScope=*/true));
935 PipelineStartEPCallbacks.push_back(
936 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
937 MPM.addPass(ModuleAddressSanitizerPass(
938 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
943 if (SanitizerOptions->SanitizeHWAddress) {
944 #if LLVM_VERSION_GE(11, 0)
945 OptimizerLastEPCallbacks.push_back(
946 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
947 MPM.addPass(HWAddressSanitizerPass(
948 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
952 PipelineStartEPCallbacks.push_back(
953 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
954 MPM.addPass(HWAddressSanitizerPass(
955 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
962 ModulePassManager MPM(DebugPassManager);
963 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
964 if (!NoPrepopulatePasses) {
965 if (OptLevel == PassBuilder::OptimizationLevel::O0) {
966 #if LLVM_VERSION_GE(12, 0)
967 for (const auto &C : PipelineStartEPCallbacks)
968 PB.registerPipelineStartEPCallback(C);
969 for (const auto &C : OptimizerLastEPCallbacks)
970 PB.registerOptimizerLastEPCallback(C);
972 // Pass false as we manually schedule ThinLTOBufferPasses below.
973 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
975 for (const auto &C : PipelineStartEPCallbacks)
978 # if LLVM_VERSION_GE(11, 0)
979 for (const auto &C : OptimizerLastEPCallbacks)
982 if (!OptimizerLastEPCallbacks.empty()) {
983 FunctionPassManager FPM(DebugPassManager);
984 for (const auto &C : OptimizerLastEPCallbacks)
986 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
990 MPM.addPass(AlwaysInlinerPass(EmitLifetimeMarkers));
992 # if LLVM_VERSION_GE(10, 0)
994 PB.addPGOInstrPassesForO0(
995 MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr,
996 /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
1001 #if LLVM_VERSION_GE(12, 0)
1002 for (const auto &C : PipelineStartEPCallbacks)
1003 PB.registerPipelineStartEPCallback(C);
1005 for (const auto &C : PipelineStartEPCallbacks)
1006 PB.registerPipelineStartEPCallback([C, OptLevel](ModulePassManager &MPM) {
1010 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
1011 for (const auto &C : OptimizerLastEPCallbacks)
1012 PB.registerOptimizerLastEPCallback(C);
1016 case LLVMRustOptStage::PreLinkNoLTO:
1017 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
1019 case LLVMRustOptStage::PreLinkThinLTO:
1020 #if LLVM_VERSION_GE(12, 0)
1021 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
1022 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
1023 // passes may still run afterwards. This means we need to run the buffer passes again.
1024 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
1025 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
1026 if (OptimizerLastEPCallbacks.empty())
1027 NeedThinLTOBufferPasses = false;
1029 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
1031 #if LLVM_VERSION_GE(11, 0)
1032 for (const auto &C : OptimizerLastEPCallbacks)
1035 if (!OptimizerLastEPCallbacks.empty()) {
1036 FunctionPassManager FPM(DebugPassManager);
1037 for (const auto &C : OptimizerLastEPCallbacks)
1039 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1043 case LLVMRustOptStage::PreLinkFatLTO:
1044 #if LLVM_VERSION_GE(12, 0)
1045 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
1046 NeedThinLTOBufferPasses = false;
1048 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
1051 case LLVMRustOptStage::ThinLTO:
1052 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
1053 // It only seems to be needed for C++ specific optimizations.
1054 #if LLVM_VERSION_GE(12, 0)
1055 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
1057 MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
1060 case LLVMRustOptStage::FatLTO:
1061 #if LLVM_VERSION_GE(12, 0)
1062 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
1064 MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
1071 if (NeedThinLTOBufferPasses) {
1072 MPM.addPass(CanonicalizeAliasesPass());
1073 MPM.addPass(NameAnonGlobalPass());
1076 // Upgrade all calls to old intrinsics first.
1077 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
1078 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
1080 MPM.run(*TheModule, MAM);
1083 // Callback to demangle function name
1085 // * name to be demangled
1088 // * output buffer len
1089 // Returns len of demangled string, or 0 if demangle failed.
1090 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1095 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1096 DemangleFn Demangle;
1097 std::vector<char> Buf;
1100 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1102 // Return empty string if demangle failed
1103 // or if name does not need to be demangled
1104 StringRef CallDemangle(StringRef name) {
1109 if (Buf.size() < name.size() * 2) {
1110 // Semangled name usually shorter than mangled,
1111 // but allocate twice as much memory just in case
1112 Buf.resize(name.size() * 2);
1115 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1121 auto Demangled = StringRef(Buf.data(), R);
1122 if (Demangled == name) {
1123 // Do not print anything if demangled name is equal to mangled.
1130 void emitFunctionAnnot(const Function *F,
1131 formatted_raw_ostream &OS) override {
1132 StringRef Demangled = CallDemangle(F->getName());
1133 if (Demangled.empty()) {
1137 OS << "; " << Demangled << "\n";
1140 void emitInstructionAnnot(const Instruction *I,
1141 formatted_raw_ostream &OS) override {
1144 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1146 Value = CI->getCalledOperand();
1147 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1149 Value = II->getCalledOperand();
1151 // Could demangle more operations, e. g.
1152 // `store %place, @function`.
1156 if (!Value->hasName()) {
1160 StringRef Demangled = CallDemangle(Value->getName());
1161 if (Demangled.empty()) {
1165 OS << "; " << Name << " " << Demangled << "\n";
1171 extern "C" LLVMRustResult
1172 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1173 std::string ErrorInfo;
1175 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
1177 ErrorInfo = EC.message();
1178 if (ErrorInfo != "") {
1179 LLVMRustSetLastError(ErrorInfo.c_str());
1180 return LLVMRustResult::Failure;
1183 RustAssemblyAnnotationWriter AAW(Demangle);
1184 formatted_raw_ostream FOS(OS);
1185 unwrap(M)->print(FOS, &AAW);
1187 return LLVMRustResult::Success;
1190 extern "C" void LLVMRustPrintPasses() {
1191 LLVMInitializePasses();
1192 struct MyListener : PassRegistrationListener {
1193 void passEnumerate(const PassInfo *Info) {
1194 StringRef PassArg = Info->getPassArgument();
1195 StringRef PassName = Info->getPassName();
1196 if (!PassArg.empty()) {
1197 // These unsigned->signed casts could theoretically overflow, but
1198 // realistically never will (and even if, the result is implementation
1199 // defined rather plain UB).
1200 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1201 (int)PassName.size(), PassName.data());
1206 PassRegistry *PR = PassRegistry::getPassRegistry();
1207 PR->enumerateWith(&Listener);
1210 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1211 bool AddLifetimes) {
1212 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1215 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1217 llvm::legacy::PassManager passes;
1219 auto PreserveFunctions = [=](const GlobalValue &GV) {
1220 for (size_t I = 0; I < Len; I++) {
1221 if (GV.getName() == Symbols[I]) {
1228 passes.add(llvm::createInternalizePass(PreserveFunctions));
1230 passes.run(*unwrap(M));
1233 extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
1234 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
1236 GV->setDoesNotThrow();
1237 Function *F = dyn_cast<Function>(GV);
1241 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
1242 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
1243 if (isa<InvokeInst>(I)) {
1244 InvokeInst *CI = cast<InvokeInst>(I);
1245 CI->setDoesNotThrow();
1253 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1254 LLVMTargetMachineRef TMR) {
1255 TargetMachine *Target = unwrap(TMR);
1256 unwrap(Module)->setDataLayout(Target->createDataLayout());
1259 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1260 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1263 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1264 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1267 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1268 LLVMRustCodeModel Model) {
1269 auto CM = fromRust(Model);
1272 unwrap(M)->setCodeModel(*CM);
1275 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1276 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1277 // LLVM, and otherwise it's just blanket rejected from other compilers.
1279 // Most of this implementation is straight copied from LLVM. At the time of
1280 // this writing it wasn't *quite* suitable to reuse more code from upstream
1281 // for our purposes, but we should strive to upstream this support once it's
1282 // ready to go! I figure we may want a bit of testing locally first before
1283 // sending this upstream to LLVM. I hear though they're quite eager to receive
1284 // feedback like this!
1286 // If you're reading this code and wondering "what in the world" or you're
1287 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1288 // then fear not! (ok maybe fear a little). All code here is mostly based
1289 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1291 // You'll find that the general layout here roughly corresponds to the `run`
1292 // method in that file as well as `ProcessThinLTOModule`. Functions are
1293 // specifically commented below as well, but if you're updating this code
1294 // or otherwise trying to understand it, the LLVM source will be useful in
1295 // interpreting the mysteries within.
1297 // Otherwise I'll apologize in advance, it probably requires a relatively
1298 // significant investment on your part to "truly understand" what's going on
1299 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1300 // and various online resources about ThinLTO to make heads or tails of all
1303 // This is a shared data structure which *must* be threadsafe to share
1304 // read-only amongst threads. This also corresponds basically to the arguments
1305 // of the `ProcessThinLTOModule` function in the LLVM source.
1306 struct LLVMRustThinLTOData {
1307 // The combined index that is the global analysis over all modules we're
1308 // performing ThinLTO for. This is mostly managed by LLVM.
1309 ModuleSummaryIndex Index;
1311 // All modules we may look at, stored as in-memory serialized versions. This
1312 // is later used when inlining to ensure we can extract any module to inline
1314 StringMap<MemoryBufferRef> ModuleMap;
1316 // A set that we manage of everything we *don't* want internalized. Note that
1317 // this includes all transitive references right now as well, but it may not
1319 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1321 // Not 100% sure what these are, but they impact what's internalized and
1322 // what's inlined across modules, I believe.
1323 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1324 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1325 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1326 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1328 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1331 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1332 struct LLVMRustThinLTOModule {
1333 const char *identifier;
1338 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1340 static const GlobalValueSummary *
1341 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1342 auto StrongDefForLinker = llvm::find_if(
1343 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1344 auto Linkage = Summary->linkage();
1345 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1346 !GlobalValue::isWeakForLinker(Linkage);
1348 if (StrongDefForLinker != GVSummaryList.end())
1349 return StrongDefForLinker->get();
1351 auto FirstDefForLinker = llvm::find_if(
1352 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1353 auto Linkage = Summary->linkage();
1354 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1356 if (FirstDefForLinker == GVSummaryList.end())
1358 return FirstDefForLinker->get();
1361 // The main entry point for creating the global ThinLTO analysis. The structure
1362 // here is basically the same as before threads are spawned in the `run`
1363 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1364 extern "C" LLVMRustThinLTOData*
1365 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1367 const char **preserved_symbols,
1369 #if LLVM_VERSION_GE(10, 0)
1370 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1372 auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
1375 // Load each module's summary and merge it into one combined index
1376 for (int i = 0; i < num_modules; i++) {
1377 auto module = &modules[i];
1378 StringRef buffer(module->data, module->len);
1379 MemoryBufferRef mem_buffer(buffer, module->identifier);
1381 Ret->ModuleMap[module->identifier] = mem_buffer;
1383 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1384 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1389 // Collect for each module the list of function it defines (GUID -> Summary)
1390 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1392 // Convert the preserved symbols set from string to GUID, this is then needed
1393 // for internalization.
1394 for (int i = 0; i < num_symbols; i++) {
1395 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1396 Ret->GUIDPreservedSymbols.insert(GUID);
1399 // Collect the import/export lists for all modules from the call-graph in the
1402 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1403 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1404 return PrevailingType::Unknown;
1406 // We don't have a complete picture in our use of ThinLTO, just our immediate
1407 // crate, so we need `ImportEnabled = false` to limit internalization.
1408 // Otherwise, we sometimes lose `static` values -- see #60184.
1409 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1410 deadIsPrevailing, /* ImportEnabled = */ false);
1411 ComputeCrossModuleImport(
1413 Ret->ModuleToDefinedGVSummaries,
1418 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1419 // impacts the caching.
1421 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1422 // being lifted from `lib/LTO/LTO.cpp` as well
1423 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1424 for (auto &I : Ret->Index) {
1425 if (I.second.SummaryList.size() > 1)
1426 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1428 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1429 const auto &Prevailing = PrevailingCopy.find(GUID);
1430 if (Prevailing == PrevailingCopy.end())
1432 return Prevailing->second == S;
1434 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1435 GlobalValue::GUID GUID,
1436 GlobalValue::LinkageTypes NewLinkage) {
1437 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1440 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
1441 Ret->GUIDPreservedSymbols);
1443 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1444 // callback below. This callback below will dictate the linkage for all
1445 // summaries in the index, and we basically just only want to ensure that dead
1446 // symbols are internalized. Otherwise everything that's already external
1447 // linkage will stay as external, and internal will stay as internal.
1448 std::set<GlobalValue::GUID> ExportedGUIDs;
1449 for (auto &List : Ret->Index) {
1450 for (auto &GVS: List.second.SummaryList) {
1451 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1453 auto GUID = GVS->getOriginalName();
1454 if (GVS->flags().Live)
1455 ExportedGUIDs.insert(GUID);
1458 #if LLVM_VERSION_GE(10, 0)
1459 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1460 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1461 return (ExportList != Ret->ExportLists.end() &&
1462 ExportList->second.count(VI)) ||
1463 ExportedGUIDs.count(VI.getGUID());
1465 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1467 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
1468 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1469 return (ExportList != Ret->ExportLists.end() &&
1470 ExportList->second.count(GUID)) ||
1471 ExportedGUIDs.count(GUID);
1473 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
1476 return Ret.release();
1480 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1484 // Below are the various passes that happen *per module* when doing ThinLTO.
1486 // In other words, these are the functions that are all run concurrently
1487 // with one another, one per module. The passes here correspond to the analysis
1488 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1489 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1490 // so rustc can save off the intermediate bytecode between each step.
1492 #if LLVM_VERSION_GE(11, 0)
1494 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1495 // When linking an ELF shared object, dso_local should be dropped. We
1496 // conservatively do this for -fpic.
1497 bool ClearDSOLocalOnDeclarations =
1498 TM.getTargetTriple().isOSBinFormatELF() &&
1499 TM.getRelocationModel() != Reloc::Static &&
1500 Mod.getPIELevel() == PIELevel::Default;
1501 return ClearDSOLocalOnDeclarations;
1506 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1507 LLVMTargetMachineRef TM) {
1508 Module &Mod = *unwrap(M);
1509 TargetMachine &Target = *unwrap(TM);
1511 #if LLVM_VERSION_GE(11, 0)
1512 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1513 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1515 bool error = renameModuleForThinLTO(Mod, Data->Index);
1519 LLVMRustSetLastError("renameModuleForThinLTO failed");
1526 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1527 Module &Mod = *unwrap(M);
1528 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1529 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1534 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1535 Module &Mod = *unwrap(M);
1536 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1537 thinLTOInternalizeModule(Mod, DefinedGlobals);
1542 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1543 LLVMTargetMachineRef TM) {
1544 Module &Mod = *unwrap(M);
1545 TargetMachine &Target = *unwrap(TM);
1547 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1548 auto Loader = [&](StringRef Identifier) {
1549 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1550 auto &Context = Mod.getContext();
1551 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1556 // The rest of this closure is a workaround for
1557 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1558 // we accidentally import wasm custom sections into different modules,
1559 // duplicating them by in the final output artifact.
1561 // The issue is worked around here by manually removing the
1562 // `wasm.custom_sections` named metadata node from any imported module. This
1563 // we know isn't used by any optimization pass so there's no need for it to
1566 // Note that the metadata is currently lazily loaded, so we materialize it
1567 // here before looking up if there's metadata inside. The `FunctionImporter`
1568 // will immediately materialize metadata anyway after an import, so this
1569 // shouldn't be a perf hit.
1570 if (Error Err = (*MOrErr)->materializeMetadata()) {
1571 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1575 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1576 if (WasmCustomSections)
1577 WasmCustomSections->eraseFromParent();
1581 #if LLVM_VERSION_GE(11, 0)
1582 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1583 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1585 FunctionImporter Importer(Data->Index, Loader);
1587 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1589 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1595 extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1596 const char*, // importing module name
1597 const char*); // imported module name
1599 // Calls `module_name_callback` for each module import done by ThinLTO.
1600 // The callback is provided with regular null-terminated C strings.
1602 LLVMRustGetThinLTOModules(const LLVMRustThinLTOData *data,
1603 LLVMRustModuleNameCallback module_name_callback,
1604 void* callback_payload) {
1605 for (const auto& importing_module : data->ImportLists) {
1606 const std::string importing_module_id = importing_module.getKey().str();
1607 const auto& imports = importing_module.getValue();
1608 for (const auto& imported_module : imports) {
1609 const std::string imported_module_id = imported_module.getKey().str();
1610 module_name_callback(callback_payload,
1611 importing_module_id.c_str(),
1612 imported_module_id.c_str());
1617 // This struct and various functions are sort of a hack right now, but the
1618 // problem is that we've got in-memory LLVM modules after we generate and
1619 // optimize all codegen-units for one compilation in rustc. To be compatible
1620 // with the LTO support above we need to serialize the modules plus their
1621 // ThinLTO summary into memory.
1623 // This structure is basically an owned version of a serialize module, with
1624 // a ThinLTO summary attached.
1625 struct LLVMRustThinLTOBuffer {
1629 extern "C" LLVMRustThinLTOBuffer*
1630 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1631 #if LLVM_VERSION_GE(10, 0)
1632 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1634 auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
1637 raw_string_ostream OS(Ret->data);
1639 legacy::PassManager PM;
1640 PM.add(createWriteThinLTOBitcodePass(OS));
1644 return Ret.release();
1648 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1652 extern "C" const void*
1653 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1654 return Buffer->data.data();
1658 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1659 return Buffer->data.length();
1662 // This is what we used to parse upstream bitcode for actual ThinLTO
1663 // processing. We'll call this once per module optimized through ThinLTO, and
1664 // it'll be called concurrently on many threads.
1665 extern "C" LLVMModuleRef
1666 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1669 const char *identifier) {
1670 StringRef Data(data, len);
1671 MemoryBufferRef Buffer(Data, identifier);
1672 unwrap(Context)->enableDebugTypeODRUniquing();
1673 Expected<std::unique_ptr<Module>> SrcOrError =
1674 parseBitcodeFile(Buffer, *unwrap(Context));
1676 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1679 return wrap(std::move(*SrcOrError).release());
1682 // Find the bitcode section in the object file data and return it as a slice.
1683 // Fail if the bitcode section is present but empty.
1685 // On success, the return value is the pointer to the start of the slice and
1686 // `out_len` is filled with the (non-zero) length. On failure, the return value
1687 // is `nullptr` and `out_len` is set to zero.
1688 extern "C" const char*
1689 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1694 StringRef Data(data, len);
1695 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1697 Expected<MemoryBufferRef> BitcodeOrError =
1698 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1699 if (!BitcodeOrError) {
1700 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1704 *out_len = BitcodeOrError->getBufferSize();
1705 return BitcodeOrError->getBufferStart();
1708 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1709 // the comment in `back/lto.rs` for why this exists.
1711 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1713 DICompileUnit **B) {
1714 Module *M = unwrap(Mod);
1715 DICompileUnit **Cur = A;
1716 DICompileUnit **Next = B;
1717 for (DICompileUnit *CU : M->debug_compile_units()) {
1726 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1727 // the comment in `back/lto.rs` for why this exists.
1729 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1730 Module *M = unwrap(Mod);
1732 // If the original source module didn't have a `DICompileUnit` then try to
1733 // merge all the existing compile units. If there aren't actually any though
1734 // then there's not much for us to do so return.
1735 if (Unit == nullptr) {
1736 for (DICompileUnit *CU : M->debug_compile_units()) {
1740 if (Unit == nullptr)
1744 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1745 // process it recursively. Note that we used to specifically iterate over
1746 // instructions to ensure we feed everything into it, but `processModule`
1747 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1748 DebugInfoFinder Finder;
1749 Finder.processModule(*M);
1751 // After we've found all our debuginfo, rewrite all subprograms to point to
1752 // the same `DICompileUnit`.
1753 for (auto &F : Finder.subprograms()) {
1754 F->replaceUnit(Unit);
1757 // Erase any other references to other `DICompileUnit` instances, the verifier
1758 // will later ensure that we don't actually have any other stale references to
1760 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1761 MD->clearOperands();
1762 MD->addOperand(Unit);
1765 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1766 // storing the result in 'KeyOut'.
1767 // Currently, this cache key is a SHA-1 hash of anything that could affect
1768 // the result of optimizing this module (e.g. module imports, exports, liveness
1769 // of access globals, etc).
1770 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1771 // used during the normal linker-plugin incremental thin-LTO process.
1773 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1774 SmallString<40> Key;
1775 llvm::lto::Config conf;
1776 const auto &ImportList = Data->ImportLists.lookup(ModId);
1777 const auto &ExportList = Data->ExportLists.lookup(ModId);
1778 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1779 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1780 std::set<GlobalValue::GUID> CfiFunctionDefs;
1781 std::set<GlobalValue::GUID> CfiFunctionDecls;
1783 // Based on the 'InProcessThinBackend' constructor in LLVM
1784 for (auto &Name : Data->Index.cfiFunctionDefs())
1785 CfiFunctionDefs.insert(
1786 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1787 for (auto &Name : Data->Index.cfiFunctionDecls())
1788 CfiFunctionDecls.insert(
1789 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1791 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1792 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1795 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());