6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/StandardInstrumentations.h"
21 #include "llvm/Support/CBindingWrapping.h"
22 #include "llvm/Support/FileSystem.h"
23 #include "llvm/Support/Host.h"
24 #if LLVM_VERSION_LT(14, 0)
25 #include "llvm/Support/TargetRegistry.h"
27 #include "llvm/MC/TargetRegistry.h"
29 #include "llvm/Target/TargetMachine.h"
30 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
31 #include "llvm/Transforms/IPO/AlwaysInliner.h"
32 #include "llvm/Transforms/IPO/FunctionImport.h"
33 #include "llvm/Transforms/Utils/AddDiscriminators.h"
34 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
35 #include "llvm/LTO/LTO.h"
36 #include "llvm-c/Transforms/PassManagerBuilder.h"
38 #include "llvm/Transforms/Instrumentation.h"
39 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
40 #include "llvm/Support/TimeProfiler.h"
41 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
42 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
43 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
44 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
45 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
46 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
47 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
48 #include "llvm/Transforms/Utils.h"
52 typedef struct LLVMOpaquePass *LLVMPassRef;
53 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
55 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
56 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
58 extern "C" void LLVMInitializePasses() {
59 PassRegistry &Registry = *PassRegistry::getPassRegistry();
60 initializeCore(Registry);
61 initializeCodeGen(Registry);
62 initializeScalarOpts(Registry);
63 initializeVectorization(Registry);
64 initializeIPO(Registry);
65 initializeAnalysis(Registry);
66 initializeTransformUtils(Registry);
67 initializeInstCombine(Registry);
68 initializeInstrumentation(Registry);
69 initializeTarget(Registry);
72 extern "C" void LLVMTimeTraceProfilerInitialize() {
73 timeTraceProfilerInitialize(
74 /* TimeTraceGranularity */ 0,
75 /* ProcName */ "rustc");
78 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
79 StringRef FN(FileName);
81 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
83 timeTraceProfilerWrite(OS);
84 timeTraceProfilerCleanup();
87 enum class LLVMRustPassKind {
93 static LLVMRustPassKind toRust(PassKind Kind) {
96 return LLVMRustPassKind::Function;
98 return LLVMRustPassKind::Module;
100 return LLVMRustPassKind::Other;
104 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
105 StringRef SR(PassName);
106 PassRegistry *PR = PassRegistry::getPassRegistry();
108 const PassInfo *PI = PR->getPassInfo(SR);
110 return wrap(PI->createPass());
115 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
116 const bool CompileKernel = false;
117 const bool UseAfterScope = true;
119 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
122 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
123 const bool CompileKernel = false;
125 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
128 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
129 const bool CompileKernel = false;
131 return wrap(createMemorySanitizerLegacyPassPass(
132 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
135 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
136 return wrap(createThreadSanitizerLegacyPassPass());
139 extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) {
140 const bool CompileKernel = false;
142 return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover));
145 extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
147 Pass *Pass = unwrap(RustPass);
148 return toRust(Pass->getPassKind());
151 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
153 Pass *Pass = unwrap(RustPass);
154 PassManagerBase *PMB = unwrap(PMR);
159 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
160 LLVMPassManagerBuilderRef PMBR,
161 LLVMPassManagerRef PMR
163 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
167 void LLVMRustAddLastExtensionPasses(
168 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
169 auto AddExtensionPasses = [Passes, NumPasses](
170 const PassManagerBuilder &Builder, PassManagerBase &PM) {
171 for (size_t I = 0; I < NumPasses; I++) {
172 PM.add(unwrap(Passes[I]));
175 // Add the passes to both of the pre-finalization extension points,
176 // so they are run for optimized and non-optimized builds.
177 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
179 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
183 #ifdef LLVM_COMPONENT_X86
184 #define SUBTARGET_X86 SUBTARGET(X86)
186 #define SUBTARGET_X86
189 #ifdef LLVM_COMPONENT_ARM
190 #define SUBTARGET_ARM SUBTARGET(ARM)
192 #define SUBTARGET_ARM
195 #ifdef LLVM_COMPONENT_AARCH64
196 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
198 #define SUBTARGET_AARCH64
201 #ifdef LLVM_COMPONENT_AVR
202 #define SUBTARGET_AVR SUBTARGET(AVR)
204 #define SUBTARGET_AVR
207 #ifdef LLVM_COMPONENT_M68k
208 #define SUBTARGET_M68K SUBTARGET(M68k)
210 #define SUBTARGET_M68K
213 #ifdef LLVM_COMPONENT_MIPS
214 #define SUBTARGET_MIPS SUBTARGET(Mips)
216 #define SUBTARGET_MIPS
219 #ifdef LLVM_COMPONENT_POWERPC
220 #define SUBTARGET_PPC SUBTARGET(PPC)
222 #define SUBTARGET_PPC
225 #ifdef LLVM_COMPONENT_SYSTEMZ
226 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
228 #define SUBTARGET_SYSTEMZ
231 #ifdef LLVM_COMPONENT_MSP430
232 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
234 #define SUBTARGET_MSP430
237 #ifdef LLVM_COMPONENT_RISCV
238 #define SUBTARGET_RISCV SUBTARGET(RISCV)
240 #define SUBTARGET_RISCV
243 #ifdef LLVM_COMPONENT_SPARC
244 #define SUBTARGET_SPARC SUBTARGET(Sparc)
246 #define SUBTARGET_SPARC
249 #ifdef LLVM_COMPONENT_HEXAGON
250 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
252 #define SUBTARGET_HEXAGON
255 #define GEN_SUBTARGETS \
269 #define SUBTARGET(x) \
271 extern const SubtargetFeatureKV x##FeatureKV[]; \
272 extern const SubtargetFeatureKV x##SubTypeKV[]; \
278 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
279 const char *Feature) {
280 TargetMachine *Target = unwrap(TM);
281 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
282 return MCInfo->checkFeatures(std::string("+") + Feature);
285 enum class LLVMRustCodeModel {
294 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
296 case LLVMRustCodeModel::Tiny:
297 return CodeModel::Tiny;
298 case LLVMRustCodeModel::Small:
299 return CodeModel::Small;
300 case LLVMRustCodeModel::Kernel:
301 return CodeModel::Kernel;
302 case LLVMRustCodeModel::Medium:
303 return CodeModel::Medium;
304 case LLVMRustCodeModel::Large:
305 return CodeModel::Large;
306 case LLVMRustCodeModel::None:
309 report_fatal_error("Bad CodeModel.");
313 enum class LLVMRustCodeGenOptLevel {
320 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
322 case LLVMRustCodeGenOptLevel::None:
323 return CodeGenOpt::None;
324 case LLVMRustCodeGenOptLevel::Less:
325 return CodeGenOpt::Less;
326 case LLVMRustCodeGenOptLevel::Default:
327 return CodeGenOpt::Default;
328 case LLVMRustCodeGenOptLevel::Aggressive:
329 return CodeGenOpt::Aggressive;
331 report_fatal_error("Bad CodeGenOptLevel.");
335 enum class LLVMRustPassBuilderOptLevel {
344 #if LLVM_VERSION_LT(14,0)
345 using OptimizationLevel = PassBuilder::OptimizationLevel;
348 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
350 case LLVMRustPassBuilderOptLevel::O0:
351 return OptimizationLevel::O0;
352 case LLVMRustPassBuilderOptLevel::O1:
353 return OptimizationLevel::O1;
354 case LLVMRustPassBuilderOptLevel::O2:
355 return OptimizationLevel::O2;
356 case LLVMRustPassBuilderOptLevel::O3:
357 return OptimizationLevel::O3;
358 case LLVMRustPassBuilderOptLevel::Os:
359 return OptimizationLevel::Os;
360 case LLVMRustPassBuilderOptLevel::Oz:
361 return OptimizationLevel::Oz;
363 report_fatal_error("Bad PassBuilderOptLevel.");
367 enum class LLVMRustRelocModel {
376 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
378 case LLVMRustRelocModel::Static:
379 return Reloc::Static;
380 case LLVMRustRelocModel::PIC:
382 case LLVMRustRelocModel::DynamicNoPic:
383 return Reloc::DynamicNoPIC;
384 case LLVMRustRelocModel::ROPI:
386 case LLVMRustRelocModel::RWPI:
388 case LLVMRustRelocModel::ROPIRWPI:
389 return Reloc::ROPI_RWPI;
391 report_fatal_error("Bad RelocModel.");
395 /// getLongestEntryLength - Return the length of the longest entry in the table.
396 template<typename KV>
397 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
399 for (auto &I : Table)
400 MaxLen = std::max(MaxLen, std::strlen(I.Key));
404 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
405 const TargetMachine *Target = unwrap(TM);
406 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
407 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
408 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
409 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
410 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
412 printf("Available CPUs for this target:\n");
413 if (HostArch == TargetArch) {
414 const StringRef HostCPU = sys::getHostCPUName();
415 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
416 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
418 for (auto &CPU : CPUTable)
419 printf(" %-*s\n", MaxCPULen, CPU.Key);
423 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
424 const TargetMachine *Target = unwrap(TM);
425 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
426 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
427 return FeatTable.size();
430 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
431 const char** Feature, const char** Desc) {
432 const TargetMachine *Target = unwrap(TM);
433 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
434 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
435 const SubtargetFeatureKV Feat = FeatTable[Index];
442 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
443 printf("Target CPU help is not supported by this LLVM version.\n\n");
446 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
450 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
453 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
454 StringRef Name = sys::getHostCPUName();
459 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
460 const char *TripleStr, const char *CPU, const char *Feature,
461 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
462 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
463 bool FunctionSections,
465 bool UniqueSectionNames,
466 bool TrapUnreachable,
469 bool EmitStackSizeSection,
470 bool RelaxELFRelocations,
472 const char *SplitDwarfFile) {
474 auto OptLevel = fromRust(RustOptLevel);
475 auto RM = fromRust(RustReloc);
476 auto CM = fromRust(RustCM);
479 Triple Trip(Triple::normalize(TripleStr));
480 const llvm::Target *TheTarget =
481 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
482 if (TheTarget == nullptr) {
483 LLVMRustSetLastError(Error.c_str());
487 TargetOptions Options;
489 Options.FloatABIType = FloatABI::Default;
491 Options.FloatABIType = FloatABI::Soft;
493 Options.DataSections = DataSections;
494 Options.FunctionSections = FunctionSections;
495 Options.UniqueSectionNames = UniqueSectionNames;
496 Options.MCOptions.AsmVerbose = AsmComments;
497 Options.MCOptions.PreserveAsmComments = AsmComments;
498 Options.MCOptions.ABIName = ABIStr;
499 if (SplitDwarfFile) {
500 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
502 Options.RelaxELFRelocations = RelaxELFRelocations;
503 Options.UseInitArray = UseInitArray;
505 if (TrapUnreachable) {
506 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
507 // This limits the extent of possible undefined behavior in some cases, as
508 // it prevents control flow from "falling through" into whatever code
509 // happens to be laid out next in memory.
510 Options.TrapUnreachable = true;
514 Options.ThreadModel = ThreadModel::Single;
517 Options.EmitStackSizeSection = EmitStackSizeSection;
519 TargetMachine *TM = TheTarget->createTargetMachine(
520 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
524 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
528 extern "C" void LLVMRustConfigurePassManagerBuilder(
529 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
530 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
531 const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath) {
532 unwrap(PMBR)->MergeFunctions = MergeFunctions;
533 unwrap(PMBR)->SLPVectorize = SLPVectorize;
534 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
535 unwrap(PMBR)->LoopVectorize = LoopVectorize;
536 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
539 assert(!PGOUsePath && !PGOSampleUsePath);
540 unwrap(PMBR)->EnablePGOInstrGen = true;
541 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
542 } else if (PGOUsePath) {
543 assert(!PGOSampleUsePath);
544 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
545 } else if (PGOSampleUsePath) {
546 unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath;
550 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
551 // field of a PassManagerBuilder, we expose our own method of doing so.
552 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
554 bool DisableSimplifyLibCalls) {
555 Triple TargetTriple(unwrap(M)->getTargetTriple());
556 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
557 if (DisableSimplifyLibCalls)
558 TLI->disableAllFunctions();
559 unwrap(PMBR)->LibraryInfo = TLI;
562 // Unfortunately, the LLVM C API doesn't provide a way to create the
563 // TargetLibraryInfo pass, so we use this method to do so.
564 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
565 bool DisableSimplifyLibCalls) {
566 Triple TargetTriple(unwrap(M)->getTargetTriple());
567 TargetLibraryInfoImpl TLII(TargetTriple);
568 if (DisableSimplifyLibCalls)
569 TLII.disableAllFunctions();
570 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
573 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
574 // all the functions in a module, so we do that manually here. You'll find
575 // similar code in clang's BackendUtil.cpp file.
576 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
578 llvm::legacy::FunctionPassManager *P =
579 unwrap<llvm::legacy::FunctionPassManager>(PMR);
580 P->doInitialization();
582 // Upgrade all calls to old intrinsics first.
583 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
584 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
586 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
588 if (!I->isDeclaration())
594 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
595 // Initializing the command-line options more than once is not allowed. So,
596 // check if they've already been initialized. (This could happen if we're
597 // being called from rustpkg, for example). If the arguments change, then
598 // that's just kinda unfortunate.
599 static bool Initialized = false;
603 cl::ParseCommandLineOptions(Argc, Argv);
606 enum class LLVMRustFileType {
611 static CodeGenFileType fromRust(LLVMRustFileType Type) {
613 case LLVMRustFileType::AssemblyFile:
614 return CGFT_AssemblyFile;
615 case LLVMRustFileType::ObjectFile:
616 return CGFT_ObjectFile;
618 report_fatal_error("Bad FileType.");
622 extern "C" LLVMRustResult
623 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
624 LLVMModuleRef M, const char *Path, const char *DwoPath,
625 LLVMRustFileType RustFileType) {
626 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
627 auto FileType = fromRust(RustFileType);
629 std::string ErrorInfo;
631 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
633 ErrorInfo = EC.message();
634 if (ErrorInfo != "") {
635 LLVMRustSetLastError(ErrorInfo.c_str());
636 return LLVMRustResult::Failure;
639 buffer_ostream BOS(OS);
641 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
644 ErrorInfo = EC.message();
645 if (ErrorInfo != "") {
646 LLVMRustSetLastError(ErrorInfo.c_str());
647 return LLVMRustResult::Failure;
649 buffer_ostream DBOS(DOS);
650 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
653 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
657 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
658 // stream (OS), so the only real safe place to delete this is here? Don't we
659 // wish this was written in Rust?
660 LLVMDisposePassManager(PMR);
661 return LLVMRustResult::Success;
664 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
665 const char*, // pass name
666 const char*); // IR name
667 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
669 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
670 if (any_isa<const Module *>(WrappedIr))
671 return any_cast<const Module *>(WrappedIr)->getName().str();
672 if (any_isa<const Function *>(WrappedIr))
673 return any_cast<const Function *>(WrappedIr)->getName().str();
674 if (any_isa<const Loop *>(WrappedIr))
675 return any_cast<const Loop *>(WrappedIr)->getName().str();
676 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
677 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
682 void LLVMSelfProfileInitializeCallbacks(
683 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
684 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
685 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
686 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
687 StringRef Pass, llvm::Any Ir) {
688 std::string PassName = Pass.str();
689 std::string IrName = LLVMRustwrappedIrGetName(Ir);
690 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
693 PIC.registerAfterPassCallback(
694 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
695 const PreservedAnalyses &Preserved) {
696 AfterPassCallback(LlvmSelfProfiler);
699 PIC.registerAfterPassInvalidatedCallback(
700 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
701 AfterPassCallback(LlvmSelfProfiler);
704 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
705 StringRef Pass, llvm::Any Ir) {
706 std::string PassName = Pass.str();
707 std::string IrName = LLVMRustwrappedIrGetName(Ir);
708 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
711 PIC.registerAfterAnalysisCallback(
712 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
713 AfterPassCallback(LlvmSelfProfiler);
717 enum class LLVMRustOptStage {
725 struct LLVMRustSanitizerOptions {
726 bool SanitizeAddress;
727 bool SanitizeAddressRecover;
729 bool SanitizeMemoryRecover;
730 int SanitizeMemoryTrackOrigins;
732 bool SanitizeHWAddress;
733 bool SanitizeHWAddressRecover;
736 extern "C" LLVMRustResult
737 LLVMRustOptimizeWithNewPassManager(
738 LLVMModuleRef ModuleRef,
739 LLVMTargetMachineRef TMRef,
740 LLVMRustPassBuilderOptLevel OptLevelRust,
741 LLVMRustOptStage OptStage,
742 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
743 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
744 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
745 LLVMRustSanitizerOptions *SanitizerOptions,
746 const char *PGOGenPath, const char *PGOUsePath,
747 bool InstrumentCoverage, bool InstrumentGCOV,
748 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
749 void* LlvmSelfProfiler,
750 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
751 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
752 const char *ExtraPasses, size_t ExtraPassesLen) {
753 Module *TheModule = unwrap(ModuleRef);
754 TargetMachine *TM = unwrap(TMRef);
755 OptimizationLevel OptLevel = fromRust(OptLevelRust);
758 PipelineTuningOptions PTO;
759 PTO.LoopUnrolling = UnrollLoops;
760 PTO.LoopInterleaving = UnrollLoops;
761 PTO.LoopVectorization = LoopVectorize;
762 PTO.SLPVectorization = SLPVectorize;
763 PTO.MergeFunctions = MergeFunctions;
765 // FIXME: We may want to expose this as an option.
766 bool DebugPassManager = false;
768 PassInstrumentationCallbacks PIC;
769 StandardInstrumentations SI(DebugPassManager);
770 SI.registerCallbacks(PIC);
772 if (LlvmSelfProfiler){
773 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
776 Optional<PGOOptions> PGOOpt;
778 assert(!PGOUsePath && !PGOSampleUsePath);
779 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
780 PGOOptions::NoCSAction, DebugInfoForProfiling);
781 } else if (PGOUsePath) {
782 assert(!PGOSampleUsePath);
783 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
784 PGOOptions::NoCSAction, DebugInfoForProfiling);
785 } else if (PGOSampleUsePath) {
786 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
787 PGOOptions::NoCSAction, DebugInfoForProfiling);
788 } else if (DebugInfoForProfiling) {
789 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
790 PGOOptions::NoCSAction, DebugInfoForProfiling);
793 #if LLVM_VERSION_GE(13, 0)
794 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
795 LoopAnalysisManager LAM;
796 FunctionAnalysisManager FAM;
797 CGSCCAnalysisManager CGAM;
798 ModuleAnalysisManager MAM;
800 PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC);
801 LoopAnalysisManager LAM(DebugPassManager);
802 FunctionAnalysisManager FAM(DebugPassManager);
803 CGSCCAnalysisManager CGAM(DebugPassManager);
804 ModuleAnalysisManager MAM(DebugPassManager);
807 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
809 Triple TargetTriple(TheModule->getTargetTriple());
810 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
811 if (DisableSimplifyLibCalls)
812 TLII->disableAllFunctions();
813 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
815 PB.registerModuleAnalyses(MAM);
816 PB.registerCGSCCAnalyses(CGAM);
817 PB.registerFunctionAnalyses(FAM);
818 PB.registerLoopAnalyses(LAM);
819 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
821 // We manually collect pipeline callbacks so we can apply them at O0, where the
822 // PassBuilder does not create a pipeline.
823 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
824 PipelineStartEPCallbacks;
825 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
826 OptimizerLastEPCallbacks;
829 PipelineStartEPCallbacks.push_back(
830 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
831 MPM.addPass(VerifierPass());
836 if (InstrumentGCOV) {
837 PipelineStartEPCallbacks.push_back(
838 [](ModulePassManager &MPM, OptimizationLevel Level) {
839 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
844 if (InstrumentCoverage) {
845 PipelineStartEPCallbacks.push_back(
846 [](ModulePassManager &MPM, OptimizationLevel Level) {
847 InstrProfOptions Options;
848 MPM.addPass(InstrProfiling(Options, false));
853 if (SanitizerOptions) {
854 if (SanitizerOptions->SanitizeMemory) {
855 MemorySanitizerOptions Options(
856 SanitizerOptions->SanitizeMemoryTrackOrigins,
857 SanitizerOptions->SanitizeMemoryRecover,
858 /*CompileKernel=*/false);
859 OptimizerLastEPCallbacks.push_back(
860 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
861 #if LLVM_VERSION_GE(14, 0)
862 MPM.addPass(ModuleMemorySanitizerPass(Options));
864 MPM.addPass(MemorySanitizerPass(Options));
866 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
871 if (SanitizerOptions->SanitizeThread) {
872 OptimizerLastEPCallbacks.push_back(
873 [](ModulePassManager &MPM, OptimizationLevel Level) {
874 #if LLVM_VERSION_GE(14, 0)
875 MPM.addPass(ModuleThreadSanitizerPass());
877 MPM.addPass(ThreadSanitizerPass());
879 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
884 if (SanitizerOptions->SanitizeAddress) {
885 OptimizerLastEPCallbacks.push_back(
886 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
887 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
888 MPM.addPass(ModuleAddressSanitizerPass(
889 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
890 #if LLVM_VERSION_GE(14, 0)
891 AddressSanitizerOptions opts(/*CompileKernel=*/false,
892 SanitizerOptions->SanitizeAddressRecover,
893 /*UseAfterScope=*/true,
894 AsanDetectStackUseAfterReturnMode::Runtime);
895 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(opts)));
897 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
898 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
899 /*UseAfterScope=*/true)));
904 if (SanitizerOptions->SanitizeHWAddress) {
905 OptimizerLastEPCallbacks.push_back(
906 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
907 #if LLVM_VERSION_GE(14, 0)
908 HWAddressSanitizerOptions opts(
909 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
910 /*DisableOptimization=*/false);
911 MPM.addPass(HWAddressSanitizerPass(opts));
913 MPM.addPass(HWAddressSanitizerPass(
914 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
921 #if LLVM_VERSION_GE(13, 0)
922 ModulePassManager MPM;
924 ModulePassManager MPM(DebugPassManager);
926 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
927 if (!NoPrepopulatePasses) {
928 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
929 // At the same time, the LTO pipelines do support O0 and using them is required.
930 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
931 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
932 for (const auto &C : PipelineStartEPCallbacks)
933 PB.registerPipelineStartEPCallback(C);
934 for (const auto &C : OptimizerLastEPCallbacks)
935 PB.registerOptimizerLastEPCallback(C);
937 // Pass false as we manually schedule ThinLTOBufferPasses below.
938 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
940 for (const auto &C : PipelineStartEPCallbacks)
941 PB.registerPipelineStartEPCallback(C);
942 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
943 for (const auto &C : OptimizerLastEPCallbacks)
944 PB.registerOptimizerLastEPCallback(C);
948 case LLVMRustOptStage::PreLinkNoLTO:
949 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
951 case LLVMRustOptStage::PreLinkThinLTO:
952 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
953 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
954 // passes may still run afterwards. This means we need to run the buffer passes again.
955 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
956 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
957 if (OptimizerLastEPCallbacks.empty())
958 NeedThinLTOBufferPasses = false;
959 for (const auto &C : OptimizerLastEPCallbacks)
962 case LLVMRustOptStage::PreLinkFatLTO:
963 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
964 NeedThinLTOBufferPasses = false;
966 case LLVMRustOptStage::ThinLTO:
967 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
968 // It only seems to be needed for C++ specific optimizations.
969 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
971 case LLVMRustOptStage::FatLTO:
972 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
978 if (ExtraPassesLen) {
979 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
980 std::string ErrMsg = toString(std::move(Err));
981 LLVMRustSetLastError(ErrMsg.c_str());
982 return LLVMRustResult::Failure;
986 if (NeedThinLTOBufferPasses) {
987 MPM.addPass(CanonicalizeAliasesPass());
988 MPM.addPass(NameAnonGlobalPass());
991 // Upgrade all calls to old intrinsics first.
992 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
993 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
995 MPM.run(*TheModule, MAM);
996 return LLVMRustResult::Success;
999 // Callback to demangle function name
1001 // * name to be demangled
1004 // * output buffer len
1005 // Returns len of demangled string, or 0 if demangle failed.
1006 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1011 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1012 DemangleFn Demangle;
1013 std::vector<char> Buf;
1016 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1018 // Return empty string if demangle failed
1019 // or if name does not need to be demangled
1020 StringRef CallDemangle(StringRef name) {
1025 if (Buf.size() < name.size() * 2) {
1026 // Semangled name usually shorter than mangled,
1027 // but allocate twice as much memory just in case
1028 Buf.resize(name.size() * 2);
1031 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1037 auto Demangled = StringRef(Buf.data(), R);
1038 if (Demangled == name) {
1039 // Do not print anything if demangled name is equal to mangled.
1046 void emitFunctionAnnot(const Function *F,
1047 formatted_raw_ostream &OS) override {
1048 StringRef Demangled = CallDemangle(F->getName());
1049 if (Demangled.empty()) {
1053 OS << "; " << Demangled << "\n";
1056 void emitInstructionAnnot(const Instruction *I,
1057 formatted_raw_ostream &OS) override {
1060 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1062 Value = CI->getCalledOperand();
1063 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1065 Value = II->getCalledOperand();
1067 // Could demangle more operations, e. g.
1068 // `store %place, @function`.
1072 if (!Value->hasName()) {
1076 StringRef Demangled = CallDemangle(Value->getName());
1077 if (Demangled.empty()) {
1081 OS << "; " << Name << " " << Demangled << "\n";
1087 extern "C" LLVMRustResult
1088 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1089 std::string ErrorInfo;
1091 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
1093 ErrorInfo = EC.message();
1094 if (ErrorInfo != "") {
1095 LLVMRustSetLastError(ErrorInfo.c_str());
1096 return LLVMRustResult::Failure;
1099 RustAssemblyAnnotationWriter AAW(Demangle);
1100 formatted_raw_ostream FOS(OS);
1101 unwrap(M)->print(FOS, &AAW);
1103 return LLVMRustResult::Success;
1106 extern "C" void LLVMRustPrintPasses() {
1107 LLVMInitializePasses();
1108 struct MyListener : PassRegistrationListener {
1109 void passEnumerate(const PassInfo *Info) {
1110 StringRef PassArg = Info->getPassArgument();
1111 StringRef PassName = Info->getPassName();
1112 if (!PassArg.empty()) {
1113 // These unsigned->signed casts could theoretically overflow, but
1114 // realistically never will (and even if, the result is implementation
1115 // defined rather plain UB).
1116 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1117 (int)PassName.size(), PassName.data());
1122 PassRegistry *PR = PassRegistry::getPassRegistry();
1123 PR->enumerateWith(&Listener);
1126 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1127 bool AddLifetimes) {
1128 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1131 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1133 llvm::legacy::PassManager passes;
1135 auto PreserveFunctions = [=](const GlobalValue &GV) {
1136 for (size_t I = 0; I < Len; I++) {
1137 if (GV.getName() == Symbols[I]) {
1144 passes.add(llvm::createInternalizePass(PreserveFunctions));
1146 passes.run(*unwrap(M));
1149 extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
1150 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
1152 GV->setDoesNotThrow();
1153 Function *F = dyn_cast<Function>(GV);
1157 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
1158 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
1159 if (isa<InvokeInst>(I)) {
1160 InvokeInst *CI = cast<InvokeInst>(I);
1161 CI->setDoesNotThrow();
1169 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1170 LLVMTargetMachineRef TMR) {
1171 TargetMachine *Target = unwrap(TMR);
1172 unwrap(Module)->setDataLayout(Target->createDataLayout());
1175 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1176 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1179 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1180 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1183 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1184 LLVMRustCodeModel Model) {
1185 auto CM = fromRust(Model);
1188 unwrap(M)->setCodeModel(*CM);
1191 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1192 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1193 // LLVM, and otherwise it's just blanket rejected from other compilers.
1195 // Most of this implementation is straight copied from LLVM. At the time of
1196 // this writing it wasn't *quite* suitable to reuse more code from upstream
1197 // for our purposes, but we should strive to upstream this support once it's
1198 // ready to go! I figure we may want a bit of testing locally first before
1199 // sending this upstream to LLVM. I hear though they're quite eager to receive
1200 // feedback like this!
1202 // If you're reading this code and wondering "what in the world" or you're
1203 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1204 // then fear not! (ok maybe fear a little). All code here is mostly based
1205 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1207 // You'll find that the general layout here roughly corresponds to the `run`
1208 // method in that file as well as `ProcessThinLTOModule`. Functions are
1209 // specifically commented below as well, but if you're updating this code
1210 // or otherwise trying to understand it, the LLVM source will be useful in
1211 // interpreting the mysteries within.
1213 // Otherwise I'll apologize in advance, it probably requires a relatively
1214 // significant investment on your part to "truly understand" what's going on
1215 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1216 // and various online resources about ThinLTO to make heads or tails of all
1219 // This is a shared data structure which *must* be threadsafe to share
1220 // read-only amongst threads. This also corresponds basically to the arguments
1221 // of the `ProcessThinLTOModule` function in the LLVM source.
1222 struct LLVMRustThinLTOData {
1223 // The combined index that is the global analysis over all modules we're
1224 // performing ThinLTO for. This is mostly managed by LLVM.
1225 ModuleSummaryIndex Index;
1227 // All modules we may look at, stored as in-memory serialized versions. This
1228 // is later used when inlining to ensure we can extract any module to inline
1230 StringMap<MemoryBufferRef> ModuleMap;
1232 // A set that we manage of everything we *don't* want internalized. Note that
1233 // this includes all transitive references right now as well, but it may not
1235 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1237 // Not 100% sure what these are, but they impact what's internalized and
1238 // what's inlined across modules, I believe.
1239 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1240 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1241 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1242 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1244 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1247 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1248 struct LLVMRustThinLTOModule {
1249 const char *identifier;
1254 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1256 static const GlobalValueSummary *
1257 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1258 auto StrongDefForLinker = llvm::find_if(
1259 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1260 auto Linkage = Summary->linkage();
1261 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1262 !GlobalValue::isWeakForLinker(Linkage);
1264 if (StrongDefForLinker != GVSummaryList.end())
1265 return StrongDefForLinker->get();
1267 auto FirstDefForLinker = llvm::find_if(
1268 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1269 auto Linkage = Summary->linkage();
1270 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1272 if (FirstDefForLinker == GVSummaryList.end())
1274 return FirstDefForLinker->get();
1277 // The main entry point for creating the global ThinLTO analysis. The structure
1278 // here is basically the same as before threads are spawned in the `run`
1279 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1280 extern "C" LLVMRustThinLTOData*
1281 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1283 const char **preserved_symbols,
1285 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1287 // Load each module's summary and merge it into one combined index
1288 for (int i = 0; i < num_modules; i++) {
1289 auto module = &modules[i];
1290 StringRef buffer(module->data, module->len);
1291 MemoryBufferRef mem_buffer(buffer, module->identifier);
1293 Ret->ModuleMap[module->identifier] = mem_buffer;
1295 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1296 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1301 // Collect for each module the list of function it defines (GUID -> Summary)
1302 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1304 // Convert the preserved symbols set from string to GUID, this is then needed
1305 // for internalization.
1306 for (int i = 0; i < num_symbols; i++) {
1307 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1308 Ret->GUIDPreservedSymbols.insert(GUID);
1311 // Collect the import/export lists for all modules from the call-graph in the
1314 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1315 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1316 return PrevailingType::Unknown;
1318 // We don't have a complete picture in our use of ThinLTO, just our immediate
1319 // crate, so we need `ImportEnabled = false` to limit internalization.
1320 // Otherwise, we sometimes lose `static` values -- see #60184.
1321 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1322 deadIsPrevailing, /* ImportEnabled = */ false);
1323 ComputeCrossModuleImport(
1325 Ret->ModuleToDefinedGVSummaries,
1330 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1331 // impacts the caching.
1333 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1334 // being lifted from `lib/LTO/LTO.cpp` as well
1335 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1336 for (auto &I : Ret->Index) {
1337 if (I.second.SummaryList.size() > 1)
1338 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1340 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1341 const auto &Prevailing = PrevailingCopy.find(GUID);
1342 if (Prevailing == PrevailingCopy.end())
1344 return Prevailing->second == S;
1346 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1347 GlobalValue::GUID GUID,
1348 GlobalValue::LinkageTypes NewLinkage) {
1349 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1352 #if LLVM_VERSION_GE(13,0)
1353 // Uses FromPrevailing visibility scheme which works for many binary
1354 // formats. We probably could and should use ELF visibility scheme for many of
1355 // our targets, however.
1357 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1358 Ret->GUIDPreservedSymbols);
1360 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
1361 Ret->GUIDPreservedSymbols);
1363 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1364 // callback below. This callback below will dictate the linkage for all
1365 // summaries in the index, and we basically just only want to ensure that dead
1366 // symbols are internalized. Otherwise everything that's already external
1367 // linkage will stay as external, and internal will stay as internal.
1368 std::set<GlobalValue::GUID> ExportedGUIDs;
1369 for (auto &List : Ret->Index) {
1370 for (auto &GVS: List.second.SummaryList) {
1371 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1373 auto GUID = GVS->getOriginalName();
1374 if (GVS->flags().Live)
1375 ExportedGUIDs.insert(GUID);
1378 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1379 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1380 return (ExportList != Ret->ExportLists.end() &&
1381 ExportList->second.count(VI)) ||
1382 ExportedGUIDs.count(VI.getGUID());
1384 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1386 return Ret.release();
1390 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1394 // Below are the various passes that happen *per module* when doing ThinLTO.
1396 // In other words, these are the functions that are all run concurrently
1397 // with one another, one per module. The passes here correspond to the analysis
1398 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1399 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1400 // so rustc can save off the intermediate bytecode between each step.
1403 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1404 // When linking an ELF shared object, dso_local should be dropped. We
1405 // conservatively do this for -fpic.
1406 bool ClearDSOLocalOnDeclarations =
1407 TM.getTargetTriple().isOSBinFormatELF() &&
1408 TM.getRelocationModel() != Reloc::Static &&
1409 Mod.getPIELevel() == PIELevel::Default;
1410 return ClearDSOLocalOnDeclarations;
1414 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1415 LLVMTargetMachineRef TM) {
1416 Module &Mod = *unwrap(M);
1417 TargetMachine &Target = *unwrap(TM);
1419 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1420 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1423 LLVMRustSetLastError("renameModuleForThinLTO failed");
1430 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1431 Module &Mod = *unwrap(M);
1432 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1433 #if LLVM_VERSION_GE(14, 0)
1434 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1436 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1442 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1443 Module &Mod = *unwrap(M);
1444 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1445 thinLTOInternalizeModule(Mod, DefinedGlobals);
1450 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1451 LLVMTargetMachineRef TM) {
1452 Module &Mod = *unwrap(M);
1453 TargetMachine &Target = *unwrap(TM);
1455 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1456 auto Loader = [&](StringRef Identifier) {
1457 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1458 auto &Context = Mod.getContext();
1459 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1464 // The rest of this closure is a workaround for
1465 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1466 // we accidentally import wasm custom sections into different modules,
1467 // duplicating them by in the final output artifact.
1469 // The issue is worked around here by manually removing the
1470 // `wasm.custom_sections` named metadata node from any imported module. This
1471 // we know isn't used by any optimization pass so there's no need for it to
1474 // Note that the metadata is currently lazily loaded, so we materialize it
1475 // here before looking up if there's metadata inside. The `FunctionImporter`
1476 // will immediately materialize metadata anyway after an import, so this
1477 // shouldn't be a perf hit.
1478 if (Error Err = (*MOrErr)->materializeMetadata()) {
1479 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1483 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1484 if (WasmCustomSections)
1485 WasmCustomSections->eraseFromParent();
1489 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1490 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1491 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1493 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1499 extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1500 const char*, // importing module name
1501 const char*); // imported module name
1503 // Calls `module_name_callback` for each module import done by ThinLTO.
1504 // The callback is provided with regular null-terminated C strings.
1506 LLVMRustGetThinLTOModules(const LLVMRustThinLTOData *data,
1507 LLVMRustModuleNameCallback module_name_callback,
1508 void* callback_payload) {
1509 for (const auto& importing_module : data->ImportLists) {
1510 const std::string importing_module_id = importing_module.getKey().str();
1511 const auto& imports = importing_module.getValue();
1512 for (const auto& imported_module : imports) {
1513 const std::string imported_module_id = imported_module.getKey().str();
1514 module_name_callback(callback_payload,
1515 importing_module_id.c_str(),
1516 imported_module_id.c_str());
1521 // This struct and various functions are sort of a hack right now, but the
1522 // problem is that we've got in-memory LLVM modules after we generate and
1523 // optimize all codegen-units for one compilation in rustc. To be compatible
1524 // with the LTO support above we need to serialize the modules plus their
1525 // ThinLTO summary into memory.
1527 // This structure is basically an owned version of a serialize module, with
1528 // a ThinLTO summary attached.
1529 struct LLVMRustThinLTOBuffer {
1533 extern "C" LLVMRustThinLTOBuffer*
1534 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1535 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1537 raw_string_ostream OS(Ret->data);
1539 legacy::PassManager PM;
1540 PM.add(createWriteThinLTOBitcodePass(OS));
1544 return Ret.release();
1548 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1552 extern "C" const void*
1553 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1554 return Buffer->data.data();
1558 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1559 return Buffer->data.length();
1562 // This is what we used to parse upstream bitcode for actual ThinLTO
1563 // processing. We'll call this once per module optimized through ThinLTO, and
1564 // it'll be called concurrently on many threads.
1565 extern "C" LLVMModuleRef
1566 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1569 const char *identifier) {
1570 StringRef Data(data, len);
1571 MemoryBufferRef Buffer(Data, identifier);
1572 unwrap(Context)->enableDebugTypeODRUniquing();
1573 Expected<std::unique_ptr<Module>> SrcOrError =
1574 parseBitcodeFile(Buffer, *unwrap(Context));
1576 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1579 return wrap(std::move(*SrcOrError).release());
1582 // Find the bitcode section in the object file data and return it as a slice.
1583 // Fail if the bitcode section is present but empty.
1585 // On success, the return value is the pointer to the start of the slice and
1586 // `out_len` is filled with the (non-zero) length. On failure, the return value
1587 // is `nullptr` and `out_len` is set to zero.
1588 extern "C" const char*
1589 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1594 StringRef Data(data, len);
1595 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1597 Expected<MemoryBufferRef> BitcodeOrError =
1598 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1599 if (!BitcodeOrError) {
1600 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1604 *out_len = BitcodeOrError->getBufferSize();
1605 return BitcodeOrError->getBufferStart();
1608 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1609 // the comment in `back/lto.rs` for why this exists.
1611 LLVMRustLTOGetDICompileUnit(LLVMModuleRef Mod,
1613 DICompileUnit **B) {
1614 Module *M = unwrap(Mod);
1615 DICompileUnit **Cur = A;
1616 DICompileUnit **Next = B;
1617 for (DICompileUnit *CU : M->debug_compile_units()) {
1626 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1627 // the comment in `back/lto.rs` for why this exists.
1629 LLVMRustLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1630 Module *M = unwrap(Mod);
1632 // If the original source module didn't have a `DICompileUnit` then try to
1633 // merge all the existing compile units. If there aren't actually any though
1634 // then there's not much for us to do so return.
1635 if (Unit == nullptr) {
1636 for (DICompileUnit *CU : M->debug_compile_units()) {
1640 if (Unit == nullptr)
1644 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1645 // process it recursively. Note that we used to specifically iterate over
1646 // instructions to ensure we feed everything into it, but `processModule`
1647 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1648 DebugInfoFinder Finder;
1649 Finder.processModule(*M);
1651 // After we've found all our debuginfo, rewrite all subprograms to point to
1652 // the same `DICompileUnit`.
1653 for (auto &F : Finder.subprograms()) {
1654 F->replaceUnit(Unit);
1657 // Erase any other references to other `DICompileUnit` instances, the verifier
1658 // will later ensure that we don't actually have any other stale references to
1660 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1661 MD->clearOperands();
1662 MD->addOperand(Unit);
1665 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1666 // storing the result in 'KeyOut'.
1667 // Currently, this cache key is a SHA-1 hash of anything that could affect
1668 // the result of optimizing this module (e.g. module imports, exports, liveness
1669 // of access globals, etc).
1670 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1671 // used during the normal linker-plugin incremental thin-LTO process.
1673 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1674 SmallString<40> Key;
1675 llvm::lto::Config conf;
1676 const auto &ImportList = Data->ImportLists.lookup(ModId);
1677 const auto &ExportList = Data->ExportLists.lookup(ModId);
1678 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1679 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1680 std::set<GlobalValue::GUID> CfiFunctionDefs;
1681 std::set<GlobalValue::GUID> CfiFunctionDecls;
1683 // Based on the 'InProcessThinBackend' constructor in LLVM
1684 for (auto &Name : Data->Index.cfiFunctionDefs())
1685 CfiFunctionDefs.insert(
1686 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1687 for (auto &Name : Data->Index.cfiFunctionDecls())
1688 CfiFunctionDecls.insert(
1689 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1691 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1692 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1695 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());