6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/StandardInstrumentations.h"
21 #include "llvm/Support/CBindingWrapping.h"
22 #include "llvm/Support/FileSystem.h"
23 #include "llvm/Support/Host.h"
24 #include "llvm/Target/TargetMachine.h"
25 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
26 #include "llvm/Transforms/IPO/AlwaysInliner.h"
27 #include "llvm/Transforms/IPO/FunctionImport.h"
28 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
29 #include "llvm/LTO/LTO.h"
30 #include "llvm-c/Transforms/PassManagerBuilder.h"
32 #include "llvm/Transforms/Instrumentation.h"
33 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
34 #include "llvm/Support/TimeProfiler.h"
35 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
36 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
37 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
38 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
39 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
43 typedef struct LLVMOpaquePass *LLVMPassRef;
44 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
46 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
47 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
48 #if LLVM_VERSION_LT(11, 0)
49 DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
50 LLVMPassManagerBuilderRef)
53 extern "C" void LLVMInitializePasses() {
54 PassRegistry &Registry = *PassRegistry::getPassRegistry();
55 initializeCore(Registry);
56 initializeCodeGen(Registry);
57 initializeScalarOpts(Registry);
58 initializeVectorization(Registry);
59 initializeIPO(Registry);
60 initializeAnalysis(Registry);
61 initializeTransformUtils(Registry);
62 initializeInstCombine(Registry);
63 initializeInstrumentation(Registry);
64 initializeTarget(Registry);
67 extern "C" void LLVMTimeTraceProfilerInitialize() {
68 #if LLVM_VERSION_GE(10, 0)
69 timeTraceProfilerInitialize(
70 /* TimeTraceGranularity */ 0,
71 /* ProcName */ "rustc");
73 timeTraceProfilerInitialize();
77 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
78 StringRef FN(FileName);
80 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
82 timeTraceProfilerWrite(OS);
83 timeTraceProfilerCleanup();
86 enum class LLVMRustPassKind {
92 static LLVMRustPassKind toRust(PassKind Kind) {
95 return LLVMRustPassKind::Function;
97 return LLVMRustPassKind::Module;
99 return LLVMRustPassKind::Other;
103 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
104 StringRef SR(PassName);
105 PassRegistry *PR = PassRegistry::getPassRegistry();
107 const PassInfo *PI = PR->getPassInfo(SR);
109 return wrap(PI->createPass());
114 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
115 const bool CompileKernel = false;
116 const bool UseAfterScope = true;
118 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
121 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
122 const bool CompileKernel = false;
124 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
127 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
128 const bool CompileKernel = false;
130 return wrap(createMemorySanitizerLegacyPassPass(
131 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
134 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
135 return wrap(createThreadSanitizerLegacyPassPass());
138 extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) {
139 const bool CompileKernel = false;
141 return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover));
144 extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
146 Pass *Pass = unwrap(RustPass);
147 return toRust(Pass->getPassKind());
150 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
152 Pass *Pass = unwrap(RustPass);
153 PassManagerBase *PMB = unwrap(PMR);
158 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
159 LLVMPassManagerBuilderRef PMBR,
160 LLVMPassManagerRef PMR
162 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
166 void LLVMRustAddLastExtensionPasses(
167 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
168 auto AddExtensionPasses = [Passes, NumPasses](
169 const PassManagerBuilder &Builder, PassManagerBase &PM) {
170 for (size_t I = 0; I < NumPasses; I++) {
171 PM.add(unwrap(Passes[I]));
174 // Add the passes to both of the pre-finalization extension points,
175 // so they are run for optimized and non-optimized builds.
176 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
178 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
182 #ifdef LLVM_COMPONENT_X86
183 #define SUBTARGET_X86 SUBTARGET(X86)
185 #define SUBTARGET_X86
188 #ifdef LLVM_COMPONENT_ARM
189 #define SUBTARGET_ARM SUBTARGET(ARM)
191 #define SUBTARGET_ARM
194 #ifdef LLVM_COMPONENT_AARCH64
195 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
197 #define SUBTARGET_AARCH64
200 #ifdef LLVM_COMPONENT_AVR
201 #define SUBTARGET_AVR SUBTARGET(AVR)
203 #define SUBTARGET_AVR
206 #ifdef LLVM_COMPONENT_MIPS
207 #define SUBTARGET_MIPS SUBTARGET(Mips)
209 #define SUBTARGET_MIPS
212 #ifdef LLVM_COMPONENT_POWERPC
213 #define SUBTARGET_PPC SUBTARGET(PPC)
215 #define SUBTARGET_PPC
218 #ifdef LLVM_COMPONENT_SYSTEMZ
219 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
221 #define SUBTARGET_SYSTEMZ
224 #ifdef LLVM_COMPONENT_MSP430
225 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
227 #define SUBTARGET_MSP430
230 #ifdef LLVM_COMPONENT_RISCV
231 #define SUBTARGET_RISCV SUBTARGET(RISCV)
233 #define SUBTARGET_RISCV
236 #ifdef LLVM_COMPONENT_SPARC
237 #define SUBTARGET_SPARC SUBTARGET(Sparc)
239 #define SUBTARGET_SPARC
242 #ifdef LLVM_COMPONENT_HEXAGON
243 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
245 #define SUBTARGET_HEXAGON
248 #define GEN_SUBTARGETS \
261 #define SUBTARGET(x) \
263 extern const SubtargetFeatureKV x##FeatureKV[]; \
264 extern const SubtargetFeatureKV x##SubTypeKV[]; \
270 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
271 const char *Feature) {
272 TargetMachine *Target = unwrap(TM);
273 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
274 return MCInfo->checkFeatures(std::string("+") + Feature);
277 enum class LLVMRustCodeModel {
286 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
288 case LLVMRustCodeModel::Tiny:
289 return CodeModel::Tiny;
290 case LLVMRustCodeModel::Small:
291 return CodeModel::Small;
292 case LLVMRustCodeModel::Kernel:
293 return CodeModel::Kernel;
294 case LLVMRustCodeModel::Medium:
295 return CodeModel::Medium;
296 case LLVMRustCodeModel::Large:
297 return CodeModel::Large;
298 case LLVMRustCodeModel::None:
301 report_fatal_error("Bad CodeModel.");
305 enum class LLVMRustCodeGenOptLevel {
312 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
314 case LLVMRustCodeGenOptLevel::None:
315 return CodeGenOpt::None;
316 case LLVMRustCodeGenOptLevel::Less:
317 return CodeGenOpt::Less;
318 case LLVMRustCodeGenOptLevel::Default:
319 return CodeGenOpt::Default;
320 case LLVMRustCodeGenOptLevel::Aggressive:
321 return CodeGenOpt::Aggressive;
323 report_fatal_error("Bad CodeGenOptLevel.");
327 enum class LLVMRustPassBuilderOptLevel {
336 static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
338 case LLVMRustPassBuilderOptLevel::O0:
339 return PassBuilder::OptimizationLevel::O0;
340 case LLVMRustPassBuilderOptLevel::O1:
341 return PassBuilder::OptimizationLevel::O1;
342 case LLVMRustPassBuilderOptLevel::O2:
343 return PassBuilder::OptimizationLevel::O2;
344 case LLVMRustPassBuilderOptLevel::O3:
345 return PassBuilder::OptimizationLevel::O3;
346 case LLVMRustPassBuilderOptLevel::Os:
347 return PassBuilder::OptimizationLevel::Os;
348 case LLVMRustPassBuilderOptLevel::Oz:
349 return PassBuilder::OptimizationLevel::Oz;
351 report_fatal_error("Bad PassBuilderOptLevel.");
355 enum class LLVMRustRelocModel {
364 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
366 case LLVMRustRelocModel::Static:
367 return Reloc::Static;
368 case LLVMRustRelocModel::PIC:
370 case LLVMRustRelocModel::DynamicNoPic:
371 return Reloc::DynamicNoPIC;
372 case LLVMRustRelocModel::ROPI:
374 case LLVMRustRelocModel::RWPI:
376 case LLVMRustRelocModel::ROPIRWPI:
377 return Reloc::ROPI_RWPI;
379 report_fatal_error("Bad RelocModel.");
383 /// getLongestEntryLength - Return the length of the longest entry in the table.
384 template<typename KV>
385 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
387 for (auto &I : Table)
388 MaxLen = std::max(MaxLen, std::strlen(I.Key));
392 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
393 const TargetMachine *Target = unwrap(TM);
394 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
395 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
396 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
397 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
398 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
400 printf("Available CPUs for this target:\n");
401 if (HostArch == TargetArch) {
402 const StringRef HostCPU = sys::getHostCPUName();
403 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
404 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
406 for (auto &CPU : CPUTable)
407 printf(" %-*s\n", MaxCPULen, CPU.Key);
411 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
412 const TargetMachine *Target = unwrap(TM);
413 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
414 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
415 unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
417 printf("Available features for this target:\n");
418 for (auto &Feature : FeatTable)
419 printf(" %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
420 printf("\nRust-specific features:\n");
421 printf(" %-*s - %s.\n",
424 "Enables libraries with C Run-time Libraries(CRT) to be statically linked"
428 printf("Use +feature to enable a feature, or -feature to disable it.\n"
429 "For example, rustc -C -target-cpu=mycpu -C "
430 "target-feature=+feature1,-feature2\n\n");
435 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
436 printf("Target CPU help is not supported by this LLVM version.\n\n");
439 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
440 printf("Target features help is not supported by this LLVM version.\n\n");
444 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
445 StringRef Name = sys::getHostCPUName();
450 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
451 const char *TripleStr, const char *CPU, const char *Feature,
452 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
453 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
454 bool FunctionSections,
456 bool TrapUnreachable,
459 bool EmitStackSizeSection,
460 bool RelaxELFRelocations,
462 const char *SplitDwarfFile) {
464 auto OptLevel = fromRust(RustOptLevel);
465 auto RM = fromRust(RustReloc);
466 auto CM = fromRust(RustCM);
469 Triple Trip(Triple::normalize(TripleStr));
470 const llvm::Target *TheTarget =
471 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
472 if (TheTarget == nullptr) {
473 LLVMRustSetLastError(Error.c_str());
477 TargetOptions Options;
479 Options.FloatABIType = FloatABI::Default;
481 Options.FloatABIType = FloatABI::Soft;
483 Options.DataSections = DataSections;
484 Options.FunctionSections = FunctionSections;
485 Options.MCOptions.AsmVerbose = AsmComments;
486 Options.MCOptions.PreserveAsmComments = AsmComments;
487 Options.MCOptions.ABIName = ABIStr;
488 if (SplitDwarfFile) {
489 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
491 Options.RelaxELFRelocations = RelaxELFRelocations;
492 Options.UseInitArray = UseInitArray;
494 if (TrapUnreachable) {
495 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
496 // This limits the extent of possible undefined behavior in some cases, as
497 // it prevents control flow from "falling through" into whatever code
498 // happens to be laid out next in memory.
499 Options.TrapUnreachable = true;
503 Options.ThreadModel = ThreadModel::Single;
506 Options.EmitStackSizeSection = EmitStackSizeSection;
508 TargetMachine *TM = TheTarget->createTargetMachine(
509 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
513 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
517 extern "C" void LLVMRustConfigurePassManagerBuilder(
518 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
519 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
520 const char* PGOGenPath, const char* PGOUsePath) {
521 unwrap(PMBR)->MergeFunctions = MergeFunctions;
522 unwrap(PMBR)->SLPVectorize = SLPVectorize;
523 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
524 unwrap(PMBR)->LoopVectorize = LoopVectorize;
525 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
529 unwrap(PMBR)->EnablePGOInstrGen = true;
530 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
534 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
538 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
539 // field of a PassManagerBuilder, we expose our own method of doing so.
540 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
542 bool DisableSimplifyLibCalls) {
543 Triple TargetTriple(unwrap(M)->getTargetTriple());
544 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
545 if (DisableSimplifyLibCalls)
546 TLI->disableAllFunctions();
547 unwrap(PMBR)->LibraryInfo = TLI;
550 // Unfortunately, the LLVM C API doesn't provide a way to create the
551 // TargetLibraryInfo pass, so we use this method to do so.
552 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
553 bool DisableSimplifyLibCalls) {
554 Triple TargetTriple(unwrap(M)->getTargetTriple());
555 TargetLibraryInfoImpl TLII(TargetTriple);
556 if (DisableSimplifyLibCalls)
557 TLII.disableAllFunctions();
558 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
561 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
562 // all the functions in a module, so we do that manually here. You'll find
563 // similar code in clang's BackendUtil.cpp file.
564 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
566 llvm::legacy::FunctionPassManager *P =
567 unwrap<llvm::legacy::FunctionPassManager>(PMR);
568 P->doInitialization();
570 // Upgrade all calls to old intrinsics first.
571 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
572 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
574 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
576 if (!I->isDeclaration())
582 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
583 // Initializing the command-line options more than once is not allowed. So,
584 // check if they've already been initialized. (This could happen if we're
585 // being called from rustpkg, for example). If the arguments change, then
586 // that's just kinda unfortunate.
587 static bool Initialized = false;
591 cl::ParseCommandLineOptions(Argc, Argv);
594 enum class LLVMRustFileType {
599 #if LLVM_VERSION_GE(10, 0)
600 static CodeGenFileType fromRust(LLVMRustFileType Type) {
602 case LLVMRustFileType::AssemblyFile:
603 return CGFT_AssemblyFile;
604 case LLVMRustFileType::ObjectFile:
605 return CGFT_ObjectFile;
607 report_fatal_error("Bad FileType.");
611 static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
613 case LLVMRustFileType::AssemblyFile:
614 return TargetMachine::CGFT_AssemblyFile;
615 case LLVMRustFileType::ObjectFile:
616 return TargetMachine::CGFT_ObjectFile;
618 report_fatal_error("Bad FileType.");
623 extern "C" LLVMRustResult
624 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
625 LLVMModuleRef M, const char *Path, const char *DwoPath,
626 LLVMRustFileType RustFileType) {
627 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
628 auto FileType = fromRust(RustFileType);
630 std::string ErrorInfo;
632 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
634 ErrorInfo = EC.message();
635 if (ErrorInfo != "") {
636 LLVMRustSetLastError(ErrorInfo.c_str());
637 return LLVMRustResult::Failure;
640 buffer_ostream BOS(OS);
642 raw_fd_ostream DOS(DwoPath, EC, sys::fs::F_None);
645 ErrorInfo = EC.message();
646 if (ErrorInfo != "") {
647 LLVMRustSetLastError(ErrorInfo.c_str());
648 return LLVMRustResult::Failure;
650 buffer_ostream DBOS(DOS);
651 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
654 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
658 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
659 // stream (OS), so the only real safe place to delete this is here? Don't we
660 // wish this was written in Rust?
661 LLVMDisposePassManager(PMR);
662 return LLVMRustResult::Success;
665 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
666 const char*, // pass name
667 const char*); // IR name
668 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
670 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
671 if (any_isa<const Module *>(WrappedIr))
672 return any_cast<const Module *>(WrappedIr)->getName().str();
673 if (any_isa<const Function *>(WrappedIr))
674 return any_cast<const Function *>(WrappedIr)->getName().str();
675 if (any_isa<const Loop *>(WrappedIr))
676 return any_cast<const Loop *>(WrappedIr)->getName().str();
677 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
678 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
683 void LLVMSelfProfileInitializeCallbacks(
684 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
685 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
686 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
687 #if LLVM_VERSION_GE(12, 0)
688 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
689 StringRef Pass, llvm::Any Ir) {
690 std::string PassName = Pass.str();
691 std::string IrName = LLVMRustwrappedIrGetName(Ir);
692 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
695 PIC.registerAfterPassCallback(
696 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
697 const PreservedAnalyses &Preserved) {
698 AfterPassCallback(LlvmSelfProfiler);
701 PIC.registerAfterPassInvalidatedCallback(
702 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
703 AfterPassCallback(LlvmSelfProfiler);
706 PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback](
707 StringRef Pass, llvm::Any Ir) {
708 std::string PassName = Pass.str();
709 std::string IrName = LLVMRustwrappedIrGetName(Ir);
710 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
714 PIC.registerAfterPassCallback(
715 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
716 AfterPassCallback(LlvmSelfProfiler);
719 PIC.registerAfterPassInvalidatedCallback(
720 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) {
721 AfterPassCallback(LlvmSelfProfiler);
725 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
726 StringRef Pass, llvm::Any Ir) {
727 std::string PassName = Pass.str();
728 std::string IrName = LLVMRustwrappedIrGetName(Ir);
729 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
732 PIC.registerAfterAnalysisCallback(
733 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
734 AfterPassCallback(LlvmSelfProfiler);
738 enum class LLVMRustOptStage {
746 struct LLVMRustSanitizerOptions {
747 bool SanitizeAddress;
748 bool SanitizeAddressRecover;
750 bool SanitizeMemoryRecover;
751 int SanitizeMemoryTrackOrigins;
753 bool SanitizeHWAddress;
754 bool SanitizeHWAddressRecover;
758 LLVMRustOptimizeWithNewPassManager(
759 LLVMModuleRef ModuleRef,
760 LLVMTargetMachineRef TMRef,
761 LLVMRustPassBuilderOptLevel OptLevelRust,
762 LLVMRustOptStage OptStage,
763 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
764 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
765 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
766 LLVMRustSanitizerOptions *SanitizerOptions,
767 const char *PGOGenPath, const char *PGOUsePath,
768 void* LlvmSelfProfiler,
769 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
770 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
771 Module *TheModule = unwrap(ModuleRef);
772 TargetMachine *TM = unwrap(TMRef);
773 PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust);
776 PipelineTuningOptions PTO;
777 PTO.LoopUnrolling = UnrollLoops;
778 PTO.LoopInterleaving = UnrollLoops;
779 PTO.LoopVectorization = LoopVectorize;
780 PTO.SLPVectorization = SLPVectorize;
781 #if LLVM_VERSION_GE(12, 0)
782 PTO.MergeFunctions = MergeFunctions;
784 // MergeFunctions is not supported by NewPM in older LLVM versions.
785 (void) MergeFunctions;
788 // FIXME: We may want to expose this as an option.
789 bool DebugPassManager = false;
791 PassInstrumentationCallbacks PIC;
792 #if LLVM_VERSION_GE(12, 0)
793 StandardInstrumentations SI(DebugPassManager);
795 StandardInstrumentations SI;
797 SI.registerCallbacks(PIC);
799 if (LlvmSelfProfiler){
800 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
803 Optional<PGOOptions> PGOOpt;
806 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr);
807 } else if (PGOUsePath) {
809 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse);
812 #if LLVM_VERSION_GE(12, 0)
813 PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC);
815 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
818 LoopAnalysisManager LAM(DebugPassManager);
819 FunctionAnalysisManager FAM(DebugPassManager);
820 CGSCCAnalysisManager CGAM(DebugPassManager);
821 ModuleAnalysisManager MAM(DebugPassManager);
823 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
825 Triple TargetTriple(TheModule->getTargetTriple());
826 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
827 if (DisableSimplifyLibCalls)
828 TLII->disableAllFunctions();
829 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
831 PB.registerModuleAnalyses(MAM);
832 PB.registerCGSCCAnalyses(CGAM);
833 PB.registerFunctionAnalyses(FAM);
834 PB.registerLoopAnalyses(LAM);
835 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
837 // We manually collect pipeline callbacks so we can apply them at O0, where the
838 // PassBuilder does not create a pipeline.
839 std::vector<std::function<void(ModulePassManager &, PassBuilder::OptimizationLevel)>>
840 PipelineStartEPCallbacks;
841 #if LLVM_VERSION_GE(11, 0)
842 std::vector<std::function<void(ModulePassManager &, PassBuilder::OptimizationLevel)>>
843 OptimizerLastEPCallbacks;
845 std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>>
846 OptimizerLastEPCallbacks;
850 PipelineStartEPCallbacks.push_back(
851 [VerifyIR](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
852 MPM.addPass(VerifierPass());
857 if (SanitizerOptions) {
858 if (SanitizerOptions->SanitizeMemory) {
859 MemorySanitizerOptions Options(
860 SanitizerOptions->SanitizeMemoryTrackOrigins,
861 SanitizerOptions->SanitizeMemoryRecover,
862 /*CompileKernel=*/false);
863 #if LLVM_VERSION_GE(11, 0)
864 OptimizerLastEPCallbacks.push_back(
865 [Options](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
866 MPM.addPass(MemorySanitizerPass(Options));
867 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
871 #if LLVM_VERSION_GE(10, 0)
872 PipelineStartEPCallbacks.push_back(
873 [Options](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
874 MPM.addPass(MemorySanitizerPass(Options));
878 OptimizerLastEPCallbacks.push_back(
879 [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
880 FPM.addPass(MemorySanitizerPass(Options));
886 if (SanitizerOptions->SanitizeThread) {
887 #if LLVM_VERSION_GE(11, 0)
888 OptimizerLastEPCallbacks.push_back(
889 [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
890 MPM.addPass(ThreadSanitizerPass());
891 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
895 #if LLVM_VERSION_GE(10, 0)
896 PipelineStartEPCallbacks.push_back(
897 [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
898 MPM.addPass(ThreadSanitizerPass());
902 OptimizerLastEPCallbacks.push_back(
903 [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
904 FPM.addPass(ThreadSanitizerPass());
910 if (SanitizerOptions->SanitizeAddress) {
911 #if LLVM_VERSION_GE(11, 0)
912 OptimizerLastEPCallbacks.push_back(
913 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
914 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
915 MPM.addPass(ModuleAddressSanitizerPass(
916 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
917 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
918 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
919 /*UseAfterScope=*/true)));
923 PipelineStartEPCallbacks.push_back(
924 [&](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
925 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
928 OptimizerLastEPCallbacks.push_back(
929 [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
930 FPM.addPass(AddressSanitizerPass(
931 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
932 /*UseAfterScope=*/true));
935 PipelineStartEPCallbacks.push_back(
936 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
937 MPM.addPass(ModuleAddressSanitizerPass(
938 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
943 if (SanitizerOptions->SanitizeHWAddress) {
944 #if LLVM_VERSION_GE(11, 0)
945 OptimizerLastEPCallbacks.push_back(
946 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
947 MPM.addPass(HWAddressSanitizerPass(
948 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
952 PipelineStartEPCallbacks.push_back(
953 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
954 MPM.addPass(HWAddressSanitizerPass(
955 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
962 ModulePassManager MPM(DebugPassManager);
963 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
964 if (!NoPrepopulatePasses) {
965 if (OptLevel == PassBuilder::OptimizationLevel::O0) {
966 #if LLVM_VERSION_GE(12, 0)
967 for (const auto &C : PipelineStartEPCallbacks)
968 PB.registerPipelineStartEPCallback(C);
969 for (const auto &C : OptimizerLastEPCallbacks)
970 PB.registerOptimizerLastEPCallback(C);
972 // Pass false as we manually schedule ThinLTOBufferPasses below.
973 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
975 for (const auto &C : PipelineStartEPCallbacks)
978 # if LLVM_VERSION_GE(11, 0)
979 for (const auto &C : OptimizerLastEPCallbacks)
982 if (!OptimizerLastEPCallbacks.empty()) {
983 FunctionPassManager FPM(DebugPassManager);
984 for (const auto &C : OptimizerLastEPCallbacks)
986 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
990 MPM.addPass(AlwaysInlinerPass(EmitLifetimeMarkers));
992 # if LLVM_VERSION_GE(10, 0)
994 PB.addPGOInstrPassesForO0(
995 MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr,
996 /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
1001 #if LLVM_VERSION_GE(12, 0)
1002 for (const auto &C : PipelineStartEPCallbacks)
1003 PB.registerPipelineStartEPCallback(C);
1005 for (const auto &C : PipelineStartEPCallbacks)
1006 PB.registerPipelineStartEPCallback([C, OptLevel](ModulePassManager &MPM) {
1010 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
1011 for (const auto &C : OptimizerLastEPCallbacks)
1012 PB.registerOptimizerLastEPCallback(C);
1016 case LLVMRustOptStage::PreLinkNoLTO:
1017 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
1019 case LLVMRustOptStage::PreLinkThinLTO:
1020 #if LLVM_VERSION_GE(12, 0)
1021 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
1022 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
1023 // passes may still run afterwards. This means we need to run the buffer passes again.
1024 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
1025 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
1026 if (OptimizerLastEPCallbacks.empty())
1027 NeedThinLTOBufferPasses = false;
1029 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
1031 #if LLVM_VERSION_GE(11, 0)
1032 for (const auto &C : OptimizerLastEPCallbacks)
1035 if (!OptimizerLastEPCallbacks.empty()) {
1036 FunctionPassManager FPM(DebugPassManager);
1037 for (const auto &C : OptimizerLastEPCallbacks)
1039 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1043 case LLVMRustOptStage::PreLinkFatLTO:
1044 #if LLVM_VERSION_GE(12, 0)
1045 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
1046 NeedThinLTOBufferPasses = false;
1048 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
1051 case LLVMRustOptStage::ThinLTO:
1052 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
1053 // It only seems to be needed for C++ specific optimizations.
1054 #if LLVM_VERSION_GE(12, 0)
1055 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
1057 MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
1060 case LLVMRustOptStage::FatLTO:
1061 #if LLVM_VERSION_GE(12, 0)
1062 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
1064 MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
1071 if (NeedThinLTOBufferPasses) {
1072 MPM.addPass(CanonicalizeAliasesPass());
1073 MPM.addPass(NameAnonGlobalPass());
1076 // Upgrade all calls to old intrinsics first.
1077 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
1078 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
1080 MPM.run(*TheModule, MAM);
1083 // Callback to demangle function name
1085 // * name to be demangled
1088 // * output buffer len
1089 // Returns len of demangled string, or 0 if demangle failed.
1090 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1095 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1096 DemangleFn Demangle;
1097 std::vector<char> Buf;
1100 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1102 // Return empty string if demangle failed
1103 // or if name does not need to be demangled
1104 StringRef CallDemangle(StringRef name) {
1109 if (Buf.size() < name.size() * 2) {
1110 // Semangled name usually shorter than mangled,
1111 // but allocate twice as much memory just in case
1112 Buf.resize(name.size() * 2);
1115 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1121 auto Demangled = StringRef(Buf.data(), R);
1122 if (Demangled == name) {
1123 // Do not print anything if demangled name is equal to mangled.
1130 void emitFunctionAnnot(const Function *F,
1131 formatted_raw_ostream &OS) override {
1132 StringRef Demangled = CallDemangle(F->getName());
1133 if (Demangled.empty()) {
1137 OS << "; " << Demangled << "\n";
1140 void emitInstructionAnnot(const Instruction *I,
1141 formatted_raw_ostream &OS) override {
1144 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1146 Value = CI->getCalledOperand();
1147 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1149 Value = II->getCalledOperand();
1151 // Could demangle more operations, e. g.
1152 // `store %place, @function`.
1156 if (!Value->hasName()) {
1160 StringRef Demangled = CallDemangle(Value->getName());
1161 if (Demangled.empty()) {
1165 OS << "; " << Name << " " << Demangled << "\n";
1171 extern "C" LLVMRustResult
1172 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1173 std::string ErrorInfo;
1175 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
1177 ErrorInfo = EC.message();
1178 if (ErrorInfo != "") {
1179 LLVMRustSetLastError(ErrorInfo.c_str());
1180 return LLVMRustResult::Failure;
1183 RustAssemblyAnnotationWriter AAW(Demangle);
1184 formatted_raw_ostream FOS(OS);
1185 unwrap(M)->print(FOS, &AAW);
1187 return LLVMRustResult::Success;
1190 extern "C" void LLVMRustPrintPasses() {
1191 LLVMInitializePasses();
1192 struct MyListener : PassRegistrationListener {
1193 void passEnumerate(const PassInfo *Info) {
1194 StringRef PassArg = Info->getPassArgument();
1195 StringRef PassName = Info->getPassName();
1196 if (!PassArg.empty()) {
1197 // These unsigned->signed casts could theoretically overflow, but
1198 // realistically never will (and even if, the result is implementation
1199 // defined rather plain UB).
1200 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1201 (int)PassName.size(), PassName.data());
1206 PassRegistry *PR = PassRegistry::getPassRegistry();
1207 PR->enumerateWith(&Listener);
1210 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1211 bool AddLifetimes) {
1212 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1215 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1217 llvm::legacy::PassManager passes;
1219 auto PreserveFunctions = [=](const GlobalValue &GV) {
1220 for (size_t I = 0; I < Len; I++) {
1221 if (GV.getName() == Symbols[I]) {
1228 passes.add(llvm::createInternalizePass(PreserveFunctions));
1230 passes.run(*unwrap(M));
1233 extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
1234 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
1236 GV->setDoesNotThrow();
1237 Function *F = dyn_cast<Function>(GV);
1241 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
1242 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
1243 if (isa<InvokeInst>(I)) {
1244 InvokeInst *CI = cast<InvokeInst>(I);
1245 CI->setDoesNotThrow();
1253 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1254 LLVMTargetMachineRef TMR) {
1255 TargetMachine *Target = unwrap(TMR);
1256 unwrap(Module)->setDataLayout(Target->createDataLayout());
1259 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1260 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1263 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1264 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1267 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1268 LLVMRustCodeModel Model) {
1269 auto CM = fromRust(Model);
1272 unwrap(M)->setCodeModel(*CM);
1275 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1276 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1277 // LLVM, and otherwise it's just blanket rejected from other compilers.
1279 // Most of this implementation is straight copied from LLVM. At the time of
1280 // this writing it wasn't *quite* suitable to reuse more code from upstream
1281 // for our purposes, but we should strive to upstream this support once it's
1282 // ready to go! I figure we may want a bit of testing locally first before
1283 // sending this upstream to LLVM. I hear though they're quite eager to receive
1284 // feedback like this!
1286 // If you're reading this code and wondering "what in the world" or you're
1287 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1288 // then fear not! (ok maybe fear a little). All code here is mostly based
1289 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1291 // You'll find that the general layout here roughly corresponds to the `run`
1292 // method in that file as well as `ProcessThinLTOModule`. Functions are
1293 // specifically commented below as well, but if you're updating this code
1294 // or otherwise trying to understand it, the LLVM source will be useful in
1295 // interpreting the mysteries within.
1297 // Otherwise I'll apologize in advance, it probably requires a relatively
1298 // significant investment on your part to "truly understand" what's going on
1299 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1300 // and various online resources about ThinLTO to make heads or tails of all
1303 // This is a shared data structure which *must* be threadsafe to share
1304 // read-only amongst threads. This also corresponds basically to the arguments
1305 // of the `ProcessThinLTOModule` function in the LLVM source.
1306 struct LLVMRustThinLTOData {
1307 // The combined index that is the global analysis over all modules we're
1308 // performing ThinLTO for. This is mostly managed by LLVM.
1309 ModuleSummaryIndex Index;
1311 // All modules we may look at, stored as in-memory serialized versions. This
1312 // is later used when inlining to ensure we can extract any module to inline
1314 StringMap<MemoryBufferRef> ModuleMap;
1316 // A set that we manage of everything we *don't* want internalized. Note that
1317 // this includes all transitive references right now as well, but it may not
1319 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1321 // Not 100% sure what these are, but they impact what's internalized and
1322 // what's inlined across modules, I believe.
1323 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1324 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1325 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1326 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1328 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1331 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1332 struct LLVMRustThinLTOModule {
1333 const char *identifier;
1338 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1340 static const GlobalValueSummary *
1341 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1342 auto StrongDefForLinker = llvm::find_if(
1343 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1344 auto Linkage = Summary->linkage();
1345 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1346 !GlobalValue::isWeakForLinker(Linkage);
1348 if (StrongDefForLinker != GVSummaryList.end())
1349 return StrongDefForLinker->get();
1351 auto FirstDefForLinker = llvm::find_if(
1352 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1353 auto Linkage = Summary->linkage();
1354 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1356 if (FirstDefForLinker == GVSummaryList.end())
1358 return FirstDefForLinker->get();
1361 // The main entry point for creating the global ThinLTO analysis. The structure
1362 // here is basically the same as before threads are spawned in the `run`
1363 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1364 extern "C" LLVMRustThinLTOData*
1365 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1367 const char **preserved_symbols,
1369 #if LLVM_VERSION_GE(10, 0)
1370 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1372 auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
1375 // Load each module's summary and merge it into one combined index
1376 for (int i = 0; i < num_modules; i++) {
1377 auto module = &modules[i];
1378 StringRef buffer(module->data, module->len);
1379 MemoryBufferRef mem_buffer(buffer, module->identifier);
1381 Ret->ModuleMap[module->identifier] = mem_buffer;
1383 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1384 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1389 // Collect for each module the list of function it defines (GUID -> Summary)
1390 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1392 // Convert the preserved symbols set from string to GUID, this is then needed
1393 // for internalization.
1394 for (int i = 0; i < num_symbols; i++) {
1395 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1396 Ret->GUIDPreservedSymbols.insert(GUID);
1399 // Collect the import/export lists for all modules from the call-graph in the
1402 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1403 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1404 return PrevailingType::Unknown;
1406 // We don't have a complete picture in our use of ThinLTO, just our immediate
1407 // crate, so we need `ImportEnabled = false` to limit internalization.
1408 // Otherwise, we sometimes lose `static` values -- see #60184.
1409 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1410 deadIsPrevailing, /* ImportEnabled = */ false);
1411 ComputeCrossModuleImport(
1413 Ret->ModuleToDefinedGVSummaries,
1418 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1419 // impacts the caching.
1421 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1422 // being lifted from `lib/LTO/LTO.cpp` as well
1423 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1424 for (auto &I : Ret->Index) {
1425 if (I.second.SummaryList.size() > 1)
1426 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1428 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1429 const auto &Prevailing = PrevailingCopy.find(GUID);
1430 if (Prevailing == PrevailingCopy.end())
1432 return Prevailing->second == S;
1434 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1435 GlobalValue::GUID GUID,
1436 GlobalValue::LinkageTypes NewLinkage) {
1437 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1440 #if LLVM_VERSION_GE(12,0)
1442 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1443 Ret->GUIDPreservedSymbols);
1445 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
1446 Ret->GUIDPreservedSymbols);
1448 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1449 // callback below. This callback below will dictate the linkage for all
1450 // summaries in the index, and we basically just only want to ensure that dead
1451 // symbols are internalized. Otherwise everything that's already external
1452 // linkage will stay as external, and internal will stay as internal.
1453 std::set<GlobalValue::GUID> ExportedGUIDs;
1454 for (auto &List : Ret->Index) {
1455 for (auto &GVS: List.second.SummaryList) {
1456 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1458 auto GUID = GVS->getOriginalName();
1459 if (GVS->flags().Live)
1460 ExportedGUIDs.insert(GUID);
1463 #if LLVM_VERSION_GE(10, 0)
1464 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1465 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1466 return (ExportList != Ret->ExportLists.end() &&
1467 ExportList->second.count(VI)) ||
1468 ExportedGUIDs.count(VI.getGUID());
1470 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1472 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
1473 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1474 return (ExportList != Ret->ExportLists.end() &&
1475 ExportList->second.count(GUID)) ||
1476 ExportedGUIDs.count(GUID);
1478 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
1481 return Ret.release();
1485 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1489 // Below are the various passes that happen *per module* when doing ThinLTO.
1491 // In other words, these are the functions that are all run concurrently
1492 // with one another, one per module. The passes here correspond to the analysis
1493 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1494 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1495 // so rustc can save off the intermediate bytecode between each step.
1497 #if LLVM_VERSION_GE(11, 0)
1499 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1500 // When linking an ELF shared object, dso_local should be dropped. We
1501 // conservatively do this for -fpic.
1502 bool ClearDSOLocalOnDeclarations =
1503 TM.getTargetTriple().isOSBinFormatELF() &&
1504 TM.getRelocationModel() != Reloc::Static &&
1505 Mod.getPIELevel() == PIELevel::Default;
1506 return ClearDSOLocalOnDeclarations;
1511 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1512 LLVMTargetMachineRef TM) {
1513 Module &Mod = *unwrap(M);
1514 TargetMachine &Target = *unwrap(TM);
1516 #if LLVM_VERSION_GE(11, 0)
1517 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1518 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1520 bool error = renameModuleForThinLTO(Mod, Data->Index);
1524 LLVMRustSetLastError("renameModuleForThinLTO failed");
1531 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1532 Module &Mod = *unwrap(M);
1533 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1534 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1539 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1540 Module &Mod = *unwrap(M);
1541 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1542 thinLTOInternalizeModule(Mod, DefinedGlobals);
1547 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1548 LLVMTargetMachineRef TM) {
1549 Module &Mod = *unwrap(M);
1550 TargetMachine &Target = *unwrap(TM);
1552 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1553 auto Loader = [&](StringRef Identifier) {
1554 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1555 auto &Context = Mod.getContext();
1556 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1561 // The rest of this closure is a workaround for
1562 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1563 // we accidentally import wasm custom sections into different modules,
1564 // duplicating them by in the final output artifact.
1566 // The issue is worked around here by manually removing the
1567 // `wasm.custom_sections` named metadata node from any imported module. This
1568 // we know isn't used by any optimization pass so there's no need for it to
1571 // Note that the metadata is currently lazily loaded, so we materialize it
1572 // here before looking up if there's metadata inside. The `FunctionImporter`
1573 // will immediately materialize metadata anyway after an import, so this
1574 // shouldn't be a perf hit.
1575 if (Error Err = (*MOrErr)->materializeMetadata()) {
1576 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1580 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1581 if (WasmCustomSections)
1582 WasmCustomSections->eraseFromParent();
1586 #if LLVM_VERSION_GE(11, 0)
1587 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1588 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1590 FunctionImporter Importer(Data->Index, Loader);
1592 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1594 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1600 extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1601 const char*, // importing module name
1602 const char*); // imported module name
1604 // Calls `module_name_callback` for each module import done by ThinLTO.
1605 // The callback is provided with regular null-terminated C strings.
1607 LLVMRustGetThinLTOModules(const LLVMRustThinLTOData *data,
1608 LLVMRustModuleNameCallback module_name_callback,
1609 void* callback_payload) {
1610 for (const auto& importing_module : data->ImportLists) {
1611 const std::string importing_module_id = importing_module.getKey().str();
1612 const auto& imports = importing_module.getValue();
1613 for (const auto& imported_module : imports) {
1614 const std::string imported_module_id = imported_module.getKey().str();
1615 module_name_callback(callback_payload,
1616 importing_module_id.c_str(),
1617 imported_module_id.c_str());
1622 // This struct and various functions are sort of a hack right now, but the
1623 // problem is that we've got in-memory LLVM modules after we generate and
1624 // optimize all codegen-units for one compilation in rustc. To be compatible
1625 // with the LTO support above we need to serialize the modules plus their
1626 // ThinLTO summary into memory.
1628 // This structure is basically an owned version of a serialize module, with
1629 // a ThinLTO summary attached.
1630 struct LLVMRustThinLTOBuffer {
1634 extern "C" LLVMRustThinLTOBuffer*
1635 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1636 #if LLVM_VERSION_GE(10, 0)
1637 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1639 auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
1642 raw_string_ostream OS(Ret->data);
1644 legacy::PassManager PM;
1645 PM.add(createWriteThinLTOBitcodePass(OS));
1649 return Ret.release();
1653 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1657 extern "C" const void*
1658 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1659 return Buffer->data.data();
1663 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1664 return Buffer->data.length();
1667 // This is what we used to parse upstream bitcode for actual ThinLTO
1668 // processing. We'll call this once per module optimized through ThinLTO, and
1669 // it'll be called concurrently on many threads.
1670 extern "C" LLVMModuleRef
1671 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1674 const char *identifier) {
1675 StringRef Data(data, len);
1676 MemoryBufferRef Buffer(Data, identifier);
1677 unwrap(Context)->enableDebugTypeODRUniquing();
1678 Expected<std::unique_ptr<Module>> SrcOrError =
1679 parseBitcodeFile(Buffer, *unwrap(Context));
1681 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1684 return wrap(std::move(*SrcOrError).release());
1687 // Find the bitcode section in the object file data and return it as a slice.
1688 // Fail if the bitcode section is present but empty.
1690 // On success, the return value is the pointer to the start of the slice and
1691 // `out_len` is filled with the (non-zero) length. On failure, the return value
1692 // is `nullptr` and `out_len` is set to zero.
1693 extern "C" const char*
1694 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1699 StringRef Data(data, len);
1700 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1702 Expected<MemoryBufferRef> BitcodeOrError =
1703 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1704 if (!BitcodeOrError) {
1705 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1709 *out_len = BitcodeOrError->getBufferSize();
1710 return BitcodeOrError->getBufferStart();
1713 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1714 // the comment in `back/lto.rs` for why this exists.
1716 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1718 DICompileUnit **B) {
1719 Module *M = unwrap(Mod);
1720 DICompileUnit **Cur = A;
1721 DICompileUnit **Next = B;
1722 for (DICompileUnit *CU : M->debug_compile_units()) {
1731 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1732 // the comment in `back/lto.rs` for why this exists.
1734 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1735 Module *M = unwrap(Mod);
1737 // If the original source module didn't have a `DICompileUnit` then try to
1738 // merge all the existing compile units. If there aren't actually any though
1739 // then there's not much for us to do so return.
1740 if (Unit == nullptr) {
1741 for (DICompileUnit *CU : M->debug_compile_units()) {
1745 if (Unit == nullptr)
1749 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1750 // process it recursively. Note that we used to specifically iterate over
1751 // instructions to ensure we feed everything into it, but `processModule`
1752 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1753 DebugInfoFinder Finder;
1754 Finder.processModule(*M);
1756 // After we've found all our debuginfo, rewrite all subprograms to point to
1757 // the same `DICompileUnit`.
1758 for (auto &F : Finder.subprograms()) {
1759 F->replaceUnit(Unit);
1762 // Erase any other references to other `DICompileUnit` instances, the verifier
1763 // will later ensure that we don't actually have any other stale references to
1765 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1766 MD->clearOperands();
1767 MD->addOperand(Unit);
1770 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1771 // storing the result in 'KeyOut'.
1772 // Currently, this cache key is a SHA-1 hash of anything that could affect
1773 // the result of optimizing this module (e.g. module imports, exports, liveness
1774 // of access globals, etc).
1775 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1776 // used during the normal linker-plugin incremental thin-LTO process.
1778 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1779 SmallString<40> Key;
1780 llvm::lto::Config conf;
1781 const auto &ImportList = Data->ImportLists.lookup(ModId);
1782 const auto &ExportList = Data->ExportLists.lookup(ModId);
1783 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1784 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1785 std::set<GlobalValue::GUID> CfiFunctionDefs;
1786 std::set<GlobalValue::GUID> CfiFunctionDecls;
1788 // Based on the 'InProcessThinBackend' constructor in LLVM
1789 for (auto &Name : Data->Index.cfiFunctionDefs())
1790 CfiFunctionDefs.insert(
1791 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1792 for (auto &Name : Data->Index.cfiFunctionDecls())
1793 CfiFunctionDecls.insert(
1794 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1796 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1797 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1800 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());