6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/StandardInstrumentations.h"
21 #include "llvm/Support/CBindingWrapping.h"
22 #include "llvm/Support/FileSystem.h"
23 #include "llvm/Support/Host.h"
24 #if LLVM_VERSION_LT(14, 0)
25 #include "llvm/Support/TargetRegistry.h"
27 #include "llvm/MC/TargetRegistry.h"
29 #include "llvm/Target/TargetMachine.h"
30 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
31 #include "llvm/Transforms/IPO/AlwaysInliner.h"
32 #include "llvm/Transforms/IPO/FunctionImport.h"
33 #include "llvm/Transforms/Utils/AddDiscriminators.h"
34 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
35 #include "llvm/LTO/LTO.h"
36 #include "llvm-c/Transforms/PassManagerBuilder.h"
38 #include "llvm/Transforms/Instrumentation.h"
39 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
40 #include "llvm/Support/TimeProfiler.h"
41 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
42 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
43 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
44 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
45 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
46 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
47 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
48 #include "llvm/Transforms/Utils.h"
52 typedef struct LLVMOpaquePass *LLVMPassRef;
53 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
55 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
56 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
58 extern "C" void LLVMInitializePasses() {
59 PassRegistry &Registry = *PassRegistry::getPassRegistry();
60 initializeCore(Registry);
61 initializeCodeGen(Registry);
62 initializeScalarOpts(Registry);
63 initializeVectorization(Registry);
64 initializeIPO(Registry);
65 initializeAnalysis(Registry);
66 initializeTransformUtils(Registry);
67 initializeInstCombine(Registry);
68 initializeInstrumentation(Registry);
69 initializeTarget(Registry);
72 extern "C" void LLVMTimeTraceProfilerInitialize() {
73 timeTraceProfilerInitialize(
74 /* TimeTraceGranularity */ 0,
75 /* ProcName */ "rustc");
78 extern "C" void LLVMTimeTraceProfilerFinishThread() {
79 timeTraceProfilerFinishThread();
82 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
83 StringRef FN(FileName);
85 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
87 timeTraceProfilerWrite(OS);
88 timeTraceProfilerCleanup();
91 enum class LLVMRustPassKind {
97 static LLVMRustPassKind toRust(PassKind Kind) {
100 return LLVMRustPassKind::Function;
102 return LLVMRustPassKind::Module;
104 return LLVMRustPassKind::Other;
108 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
109 StringRef SR(PassName);
110 PassRegistry *PR = PassRegistry::getPassRegistry();
112 const PassInfo *PI = PR->getPassInfo(SR);
114 return wrap(PI->createPass());
119 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
120 const bool CompileKernel = false;
121 const bool UseAfterScope = true;
123 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
126 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
127 const bool CompileKernel = false;
129 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
132 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
133 const bool CompileKernel = false;
135 return wrap(createMemorySanitizerLegacyPassPass(
136 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
139 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
140 return wrap(createThreadSanitizerLegacyPassPass());
143 extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) {
144 const bool CompileKernel = false;
146 return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover));
149 extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
151 Pass *Pass = unwrap(RustPass);
152 return toRust(Pass->getPassKind());
155 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
157 Pass *Pass = unwrap(RustPass);
158 PassManagerBase *PMB = unwrap(PMR);
163 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
164 LLVMPassManagerBuilderRef PMBR,
165 LLVMPassManagerRef PMR
167 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
171 void LLVMRustAddLastExtensionPasses(
172 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
173 auto AddExtensionPasses = [Passes, NumPasses](
174 const PassManagerBuilder &Builder, PassManagerBase &PM) {
175 for (size_t I = 0; I < NumPasses; I++) {
176 PM.add(unwrap(Passes[I]));
179 // Add the passes to both of the pre-finalization extension points,
180 // so they are run for optimized and non-optimized builds.
181 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
183 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
187 #ifdef LLVM_COMPONENT_X86
188 #define SUBTARGET_X86 SUBTARGET(X86)
190 #define SUBTARGET_X86
193 #ifdef LLVM_COMPONENT_ARM
194 #define SUBTARGET_ARM SUBTARGET(ARM)
196 #define SUBTARGET_ARM
199 #ifdef LLVM_COMPONENT_AARCH64
200 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
202 #define SUBTARGET_AARCH64
205 #ifdef LLVM_COMPONENT_AVR
206 #define SUBTARGET_AVR SUBTARGET(AVR)
208 #define SUBTARGET_AVR
211 #ifdef LLVM_COMPONENT_M68k
212 #define SUBTARGET_M68K SUBTARGET(M68k)
214 #define SUBTARGET_M68K
217 #ifdef LLVM_COMPONENT_MIPS
218 #define SUBTARGET_MIPS SUBTARGET(Mips)
220 #define SUBTARGET_MIPS
223 #ifdef LLVM_COMPONENT_POWERPC
224 #define SUBTARGET_PPC SUBTARGET(PPC)
226 #define SUBTARGET_PPC
229 #ifdef LLVM_COMPONENT_SYSTEMZ
230 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
232 #define SUBTARGET_SYSTEMZ
235 #ifdef LLVM_COMPONENT_MSP430
236 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
238 #define SUBTARGET_MSP430
241 #ifdef LLVM_COMPONENT_RISCV
242 #define SUBTARGET_RISCV SUBTARGET(RISCV)
244 #define SUBTARGET_RISCV
247 #ifdef LLVM_COMPONENT_SPARC
248 #define SUBTARGET_SPARC SUBTARGET(Sparc)
250 #define SUBTARGET_SPARC
253 #ifdef LLVM_COMPONENT_HEXAGON
254 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
256 #define SUBTARGET_HEXAGON
259 #define GEN_SUBTARGETS \
273 #define SUBTARGET(x) \
275 extern const SubtargetFeatureKV x##FeatureKV[]; \
276 extern const SubtargetFeatureKV x##SubTypeKV[]; \
282 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
283 const char *Feature) {
284 TargetMachine *Target = unwrap(TM);
285 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
286 return MCInfo->checkFeatures(std::string("+") + Feature);
289 enum class LLVMRustCodeModel {
298 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
300 case LLVMRustCodeModel::Tiny:
301 return CodeModel::Tiny;
302 case LLVMRustCodeModel::Small:
303 return CodeModel::Small;
304 case LLVMRustCodeModel::Kernel:
305 return CodeModel::Kernel;
306 case LLVMRustCodeModel::Medium:
307 return CodeModel::Medium;
308 case LLVMRustCodeModel::Large:
309 return CodeModel::Large;
310 case LLVMRustCodeModel::None:
313 report_fatal_error("Bad CodeModel.");
317 enum class LLVMRustCodeGenOptLevel {
324 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
326 case LLVMRustCodeGenOptLevel::None:
327 return CodeGenOpt::None;
328 case LLVMRustCodeGenOptLevel::Less:
329 return CodeGenOpt::Less;
330 case LLVMRustCodeGenOptLevel::Default:
331 return CodeGenOpt::Default;
332 case LLVMRustCodeGenOptLevel::Aggressive:
333 return CodeGenOpt::Aggressive;
335 report_fatal_error("Bad CodeGenOptLevel.");
339 enum class LLVMRustPassBuilderOptLevel {
348 #if LLVM_VERSION_LT(14,0)
349 using OptimizationLevel = PassBuilder::OptimizationLevel;
352 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
354 case LLVMRustPassBuilderOptLevel::O0:
355 return OptimizationLevel::O0;
356 case LLVMRustPassBuilderOptLevel::O1:
357 return OptimizationLevel::O1;
358 case LLVMRustPassBuilderOptLevel::O2:
359 return OptimizationLevel::O2;
360 case LLVMRustPassBuilderOptLevel::O3:
361 return OptimizationLevel::O3;
362 case LLVMRustPassBuilderOptLevel::Os:
363 return OptimizationLevel::Os;
364 case LLVMRustPassBuilderOptLevel::Oz:
365 return OptimizationLevel::Oz;
367 report_fatal_error("Bad PassBuilderOptLevel.");
371 enum class LLVMRustRelocModel {
380 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
382 case LLVMRustRelocModel::Static:
383 return Reloc::Static;
384 case LLVMRustRelocModel::PIC:
386 case LLVMRustRelocModel::DynamicNoPic:
387 return Reloc::DynamicNoPIC;
388 case LLVMRustRelocModel::ROPI:
390 case LLVMRustRelocModel::RWPI:
392 case LLVMRustRelocModel::ROPIRWPI:
393 return Reloc::ROPI_RWPI;
395 report_fatal_error("Bad RelocModel.");
399 /// getLongestEntryLength - Return the length of the longest entry in the table.
400 template<typename KV>
401 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
403 for (auto &I : Table)
404 MaxLen = std::max(MaxLen, std::strlen(I.Key));
408 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
409 const TargetMachine *Target = unwrap(TM);
410 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
411 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
412 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
413 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
414 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
416 printf("Available CPUs for this target:\n");
417 if (HostArch == TargetArch) {
418 const StringRef HostCPU = sys::getHostCPUName();
419 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
420 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
422 for (auto &CPU : CPUTable)
423 printf(" %-*s\n", MaxCPULen, CPU.Key);
427 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
428 const TargetMachine *Target = unwrap(TM);
429 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
430 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
431 return FeatTable.size();
434 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
435 const char** Feature, const char** Desc) {
436 const TargetMachine *Target = unwrap(TM);
437 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
438 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
439 const SubtargetFeatureKV Feat = FeatTable[Index];
446 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
447 printf("Target CPU help is not supported by this LLVM version.\n\n");
450 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
454 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
457 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
458 StringRef Name = sys::getHostCPUName();
463 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
464 const char *TripleStr, const char *CPU, const char *Feature,
465 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
466 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
467 bool FunctionSections,
469 bool UniqueSectionNames,
470 bool TrapUnreachable,
473 bool EmitStackSizeSection,
474 bool RelaxELFRelocations,
476 const char *SplitDwarfFile) {
478 auto OptLevel = fromRust(RustOptLevel);
479 auto RM = fromRust(RustReloc);
480 auto CM = fromRust(RustCM);
483 Triple Trip(Triple::normalize(TripleStr));
484 const llvm::Target *TheTarget =
485 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
486 if (TheTarget == nullptr) {
487 LLVMRustSetLastError(Error.c_str());
491 TargetOptions Options;
493 Options.FloatABIType = FloatABI::Default;
495 Options.FloatABIType = FloatABI::Soft;
497 Options.DataSections = DataSections;
498 Options.FunctionSections = FunctionSections;
499 Options.UniqueSectionNames = UniqueSectionNames;
500 Options.MCOptions.AsmVerbose = AsmComments;
501 Options.MCOptions.PreserveAsmComments = AsmComments;
502 Options.MCOptions.ABIName = ABIStr;
503 if (SplitDwarfFile) {
504 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
506 Options.RelaxELFRelocations = RelaxELFRelocations;
507 Options.UseInitArray = UseInitArray;
509 if (TrapUnreachable) {
510 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
511 // This limits the extent of possible undefined behavior in some cases, as
512 // it prevents control flow from "falling through" into whatever code
513 // happens to be laid out next in memory.
514 Options.TrapUnreachable = true;
518 Options.ThreadModel = ThreadModel::Single;
521 Options.EmitStackSizeSection = EmitStackSizeSection;
523 TargetMachine *TM = TheTarget->createTargetMachine(
524 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
528 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
532 extern "C" void LLVMRustConfigurePassManagerBuilder(
533 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
534 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
535 const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath) {
536 unwrap(PMBR)->MergeFunctions = MergeFunctions;
537 unwrap(PMBR)->SLPVectorize = SLPVectorize;
538 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
539 unwrap(PMBR)->LoopVectorize = LoopVectorize;
540 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
543 assert(!PGOUsePath && !PGOSampleUsePath);
544 unwrap(PMBR)->EnablePGOInstrGen = true;
545 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
546 } else if (PGOUsePath) {
547 assert(!PGOSampleUsePath);
548 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
549 } else if (PGOSampleUsePath) {
550 unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath;
554 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
555 // field of a PassManagerBuilder, we expose our own method of doing so.
556 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
558 bool DisableSimplifyLibCalls) {
559 Triple TargetTriple(unwrap(M)->getTargetTriple());
560 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
561 if (DisableSimplifyLibCalls)
562 TLI->disableAllFunctions();
563 unwrap(PMBR)->LibraryInfo = TLI;
566 // Unfortunately, the LLVM C API doesn't provide a way to create the
567 // TargetLibraryInfo pass, so we use this method to do so.
568 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
569 bool DisableSimplifyLibCalls) {
570 Triple TargetTriple(unwrap(M)->getTargetTriple());
571 TargetLibraryInfoImpl TLII(TargetTriple);
572 if (DisableSimplifyLibCalls)
573 TLII.disableAllFunctions();
574 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
577 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
578 // all the functions in a module, so we do that manually here. You'll find
579 // similar code in clang's BackendUtil.cpp file.
580 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
582 llvm::legacy::FunctionPassManager *P =
583 unwrap<llvm::legacy::FunctionPassManager>(PMR);
584 P->doInitialization();
586 // Upgrade all calls to old intrinsics first.
587 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
588 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
590 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
592 if (!I->isDeclaration())
598 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
599 // Initializing the command-line options more than once is not allowed. So,
600 // check if they've already been initialized. (This could happen if we're
601 // being called from rustpkg, for example). If the arguments change, then
602 // that's just kinda unfortunate.
603 static bool Initialized = false;
607 cl::ParseCommandLineOptions(Argc, Argv);
610 enum class LLVMRustFileType {
615 static CodeGenFileType fromRust(LLVMRustFileType Type) {
617 case LLVMRustFileType::AssemblyFile:
618 return CGFT_AssemblyFile;
619 case LLVMRustFileType::ObjectFile:
620 return CGFT_ObjectFile;
622 report_fatal_error("Bad FileType.");
626 extern "C" LLVMRustResult
627 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
628 LLVMModuleRef M, const char *Path, const char *DwoPath,
629 LLVMRustFileType RustFileType) {
630 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
631 auto FileType = fromRust(RustFileType);
633 std::string ErrorInfo;
635 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
637 ErrorInfo = EC.message();
638 if (ErrorInfo != "") {
639 LLVMRustSetLastError(ErrorInfo.c_str());
640 return LLVMRustResult::Failure;
643 buffer_ostream BOS(OS);
645 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
648 ErrorInfo = EC.message();
649 if (ErrorInfo != "") {
650 LLVMRustSetLastError(ErrorInfo.c_str());
651 return LLVMRustResult::Failure;
653 buffer_ostream DBOS(DOS);
654 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
657 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
661 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
662 // stream (OS), so the only real safe place to delete this is here? Don't we
663 // wish this was written in Rust?
664 LLVMDisposePassManager(PMR);
665 return LLVMRustResult::Success;
668 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
669 const char*, // pass name
670 const char*); // IR name
671 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
673 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
674 if (any_isa<const Module *>(WrappedIr))
675 return any_cast<const Module *>(WrappedIr)->getName().str();
676 if (any_isa<const Function *>(WrappedIr))
677 return any_cast<const Function *>(WrappedIr)->getName().str();
678 if (any_isa<const Loop *>(WrappedIr))
679 return any_cast<const Loop *>(WrappedIr)->getName().str();
680 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
681 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
686 void LLVMSelfProfileInitializeCallbacks(
687 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
688 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
689 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
690 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
691 StringRef Pass, llvm::Any Ir) {
692 std::string PassName = Pass.str();
693 std::string IrName = LLVMRustwrappedIrGetName(Ir);
694 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
697 PIC.registerAfterPassCallback(
698 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
699 const PreservedAnalyses &Preserved) {
700 AfterPassCallback(LlvmSelfProfiler);
703 PIC.registerAfterPassInvalidatedCallback(
704 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
705 AfterPassCallback(LlvmSelfProfiler);
708 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
709 StringRef Pass, llvm::Any Ir) {
710 std::string PassName = Pass.str();
711 std::string IrName = LLVMRustwrappedIrGetName(Ir);
712 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
715 PIC.registerAfterAnalysisCallback(
716 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
717 AfterPassCallback(LlvmSelfProfiler);
721 enum class LLVMRustOptStage {
729 struct LLVMRustSanitizerOptions {
730 bool SanitizeAddress;
731 bool SanitizeAddressRecover;
733 bool SanitizeMemoryRecover;
734 int SanitizeMemoryTrackOrigins;
736 bool SanitizeHWAddress;
737 bool SanitizeHWAddressRecover;
740 extern "C" LLVMRustResult
741 LLVMRustOptimizeWithNewPassManager(
742 LLVMModuleRef ModuleRef,
743 LLVMTargetMachineRef TMRef,
744 LLVMRustPassBuilderOptLevel OptLevelRust,
745 LLVMRustOptStage OptStage,
746 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
747 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
748 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
749 LLVMRustSanitizerOptions *SanitizerOptions,
750 const char *PGOGenPath, const char *PGOUsePath,
751 bool InstrumentCoverage, bool InstrumentGCOV,
752 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
753 void* LlvmSelfProfiler,
754 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
755 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
756 const char *ExtraPasses, size_t ExtraPassesLen) {
757 Module *TheModule = unwrap(ModuleRef);
758 TargetMachine *TM = unwrap(TMRef);
759 OptimizationLevel OptLevel = fromRust(OptLevelRust);
762 PipelineTuningOptions PTO;
763 PTO.LoopUnrolling = UnrollLoops;
764 PTO.LoopInterleaving = UnrollLoops;
765 PTO.LoopVectorization = LoopVectorize;
766 PTO.SLPVectorization = SLPVectorize;
767 PTO.MergeFunctions = MergeFunctions;
769 // FIXME: We may want to expose this as an option.
770 bool DebugPassManager = false;
772 PassInstrumentationCallbacks PIC;
773 StandardInstrumentations SI(DebugPassManager);
774 SI.registerCallbacks(PIC);
776 if (LlvmSelfProfiler){
777 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
780 Optional<PGOOptions> PGOOpt;
782 assert(!PGOUsePath && !PGOSampleUsePath);
783 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
784 PGOOptions::NoCSAction, DebugInfoForProfiling);
785 } else if (PGOUsePath) {
786 assert(!PGOSampleUsePath);
787 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
788 PGOOptions::NoCSAction, DebugInfoForProfiling);
789 } else if (PGOSampleUsePath) {
790 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
791 PGOOptions::NoCSAction, DebugInfoForProfiling);
792 } else if (DebugInfoForProfiling) {
793 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
794 PGOOptions::NoCSAction, DebugInfoForProfiling);
797 #if LLVM_VERSION_GE(13, 0)
798 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
799 LoopAnalysisManager LAM;
800 FunctionAnalysisManager FAM;
801 CGSCCAnalysisManager CGAM;
802 ModuleAnalysisManager MAM;
804 PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC);
805 LoopAnalysisManager LAM(DebugPassManager);
806 FunctionAnalysisManager FAM(DebugPassManager);
807 CGSCCAnalysisManager CGAM(DebugPassManager);
808 ModuleAnalysisManager MAM(DebugPassManager);
811 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
813 Triple TargetTriple(TheModule->getTargetTriple());
814 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
815 if (DisableSimplifyLibCalls)
816 TLII->disableAllFunctions();
817 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
819 PB.registerModuleAnalyses(MAM);
820 PB.registerCGSCCAnalyses(CGAM);
821 PB.registerFunctionAnalyses(FAM);
822 PB.registerLoopAnalyses(LAM);
823 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
825 // We manually collect pipeline callbacks so we can apply them at O0, where the
826 // PassBuilder does not create a pipeline.
827 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
828 PipelineStartEPCallbacks;
829 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
830 OptimizerLastEPCallbacks;
833 PipelineStartEPCallbacks.push_back(
834 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
835 MPM.addPass(VerifierPass());
840 if (InstrumentGCOV) {
841 PipelineStartEPCallbacks.push_back(
842 [](ModulePassManager &MPM, OptimizationLevel Level) {
843 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
848 if (InstrumentCoverage) {
849 PipelineStartEPCallbacks.push_back(
850 [](ModulePassManager &MPM, OptimizationLevel Level) {
851 InstrProfOptions Options;
852 MPM.addPass(InstrProfiling(Options, false));
857 if (SanitizerOptions) {
858 if (SanitizerOptions->SanitizeMemory) {
859 MemorySanitizerOptions Options(
860 SanitizerOptions->SanitizeMemoryTrackOrigins,
861 SanitizerOptions->SanitizeMemoryRecover,
862 /*CompileKernel=*/false);
863 OptimizerLastEPCallbacks.push_back(
864 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
865 #if LLVM_VERSION_GE(14, 0)
866 MPM.addPass(ModuleMemorySanitizerPass(Options));
868 MPM.addPass(MemorySanitizerPass(Options));
870 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
875 if (SanitizerOptions->SanitizeThread) {
876 OptimizerLastEPCallbacks.push_back(
877 [](ModulePassManager &MPM, OptimizationLevel Level) {
878 #if LLVM_VERSION_GE(14, 0)
879 MPM.addPass(ModuleThreadSanitizerPass());
881 MPM.addPass(ThreadSanitizerPass());
883 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
888 if (SanitizerOptions->SanitizeAddress) {
889 OptimizerLastEPCallbacks.push_back(
890 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
891 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
892 MPM.addPass(ModuleAddressSanitizerPass(
893 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
894 #if LLVM_VERSION_GE(14, 0)
895 AddressSanitizerOptions opts(/*CompileKernel=*/false,
896 SanitizerOptions->SanitizeAddressRecover,
897 /*UseAfterScope=*/true,
898 AsanDetectStackUseAfterReturnMode::Runtime);
899 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(opts)));
901 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
902 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
903 /*UseAfterScope=*/true)));
908 if (SanitizerOptions->SanitizeHWAddress) {
909 OptimizerLastEPCallbacks.push_back(
910 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
911 #if LLVM_VERSION_GE(14, 0)
912 HWAddressSanitizerOptions opts(
913 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
914 /*DisableOptimization=*/false);
915 MPM.addPass(HWAddressSanitizerPass(opts));
917 MPM.addPass(HWAddressSanitizerPass(
918 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
925 #if LLVM_VERSION_GE(13, 0)
926 ModulePassManager MPM;
928 ModulePassManager MPM(DebugPassManager);
930 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
931 if (!NoPrepopulatePasses) {
932 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
933 // At the same time, the LTO pipelines do support O0 and using them is required.
934 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
935 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
936 for (const auto &C : PipelineStartEPCallbacks)
937 PB.registerPipelineStartEPCallback(C);
938 for (const auto &C : OptimizerLastEPCallbacks)
939 PB.registerOptimizerLastEPCallback(C);
941 // Pass false as we manually schedule ThinLTOBufferPasses below.
942 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
944 for (const auto &C : PipelineStartEPCallbacks)
945 PB.registerPipelineStartEPCallback(C);
946 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
947 for (const auto &C : OptimizerLastEPCallbacks)
948 PB.registerOptimizerLastEPCallback(C);
952 case LLVMRustOptStage::PreLinkNoLTO:
953 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
955 case LLVMRustOptStage::PreLinkThinLTO:
956 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
957 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
958 // passes may still run afterwards. This means we need to run the buffer passes again.
959 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
960 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
961 if (OptimizerLastEPCallbacks.empty())
962 NeedThinLTOBufferPasses = false;
963 for (const auto &C : OptimizerLastEPCallbacks)
966 case LLVMRustOptStage::PreLinkFatLTO:
967 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
968 NeedThinLTOBufferPasses = false;
970 case LLVMRustOptStage::ThinLTO:
971 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
972 // It only seems to be needed for C++ specific optimizations.
973 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
975 case LLVMRustOptStage::FatLTO:
976 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
982 if (ExtraPassesLen) {
983 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
984 std::string ErrMsg = toString(std::move(Err));
985 LLVMRustSetLastError(ErrMsg.c_str());
986 return LLVMRustResult::Failure;
990 if (NeedThinLTOBufferPasses) {
991 MPM.addPass(CanonicalizeAliasesPass());
992 MPM.addPass(NameAnonGlobalPass());
995 // Upgrade all calls to old intrinsics first.
996 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
997 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
999 MPM.run(*TheModule, MAM);
1000 return LLVMRustResult::Success;
1003 // Callback to demangle function name
1005 // * name to be demangled
1008 // * output buffer len
1009 // Returns len of demangled string, or 0 if demangle failed.
1010 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1015 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1016 DemangleFn Demangle;
1017 std::vector<char> Buf;
1020 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1022 // Return empty string if demangle failed
1023 // or if name does not need to be demangled
1024 StringRef CallDemangle(StringRef name) {
1029 if (Buf.size() < name.size() * 2) {
1030 // Semangled name usually shorter than mangled,
1031 // but allocate twice as much memory just in case
1032 Buf.resize(name.size() * 2);
1035 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1041 auto Demangled = StringRef(Buf.data(), R);
1042 if (Demangled == name) {
1043 // Do not print anything if demangled name is equal to mangled.
1050 void emitFunctionAnnot(const Function *F,
1051 formatted_raw_ostream &OS) override {
1052 StringRef Demangled = CallDemangle(F->getName());
1053 if (Demangled.empty()) {
1057 OS << "; " << Demangled << "\n";
1060 void emitInstructionAnnot(const Instruction *I,
1061 formatted_raw_ostream &OS) override {
1064 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1066 Value = CI->getCalledOperand();
1067 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1069 Value = II->getCalledOperand();
1071 // Could demangle more operations, e. g.
1072 // `store %place, @function`.
1076 if (!Value->hasName()) {
1080 StringRef Demangled = CallDemangle(Value->getName());
1081 if (Demangled.empty()) {
1085 OS << "; " << Name << " " << Demangled << "\n";
1091 extern "C" LLVMRustResult
1092 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1093 std::string ErrorInfo;
1095 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
1097 ErrorInfo = EC.message();
1098 if (ErrorInfo != "") {
1099 LLVMRustSetLastError(ErrorInfo.c_str());
1100 return LLVMRustResult::Failure;
1103 RustAssemblyAnnotationWriter AAW(Demangle);
1104 formatted_raw_ostream FOS(OS);
1105 unwrap(M)->print(FOS, &AAW);
1107 return LLVMRustResult::Success;
1110 extern "C" void LLVMRustPrintPasses() {
1111 LLVMInitializePasses();
1112 struct MyListener : PassRegistrationListener {
1113 void passEnumerate(const PassInfo *Info) {
1114 StringRef PassArg = Info->getPassArgument();
1115 StringRef PassName = Info->getPassName();
1116 if (!PassArg.empty()) {
1117 // These unsigned->signed casts could theoretically overflow, but
1118 // realistically never will (and even if, the result is implementation
1119 // defined rather plain UB).
1120 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1121 (int)PassName.size(), PassName.data());
1126 PassRegistry *PR = PassRegistry::getPassRegistry();
1127 PR->enumerateWith(&Listener);
1130 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1131 bool AddLifetimes) {
1132 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1135 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1137 llvm::legacy::PassManager passes;
1139 auto PreserveFunctions = [=](const GlobalValue &GV) {
1140 for (size_t I = 0; I < Len; I++) {
1141 if (GV.getName() == Symbols[I]) {
1148 passes.add(llvm::createInternalizePass(PreserveFunctions));
1150 passes.run(*unwrap(M));
1153 extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
1154 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
1156 GV->setDoesNotThrow();
1157 Function *F = dyn_cast<Function>(GV);
1161 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
1162 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
1163 if (isa<InvokeInst>(I)) {
1164 InvokeInst *CI = cast<InvokeInst>(I);
1165 CI->setDoesNotThrow();
1173 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1174 LLVMTargetMachineRef TMR) {
1175 TargetMachine *Target = unwrap(TMR);
1176 unwrap(Module)->setDataLayout(Target->createDataLayout());
1179 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1180 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1183 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1184 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1187 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1188 LLVMRustCodeModel Model) {
1189 auto CM = fromRust(Model);
1192 unwrap(M)->setCodeModel(*CM);
1195 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1196 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1197 // LLVM, and otherwise it's just blanket rejected from other compilers.
1199 // Most of this implementation is straight copied from LLVM. At the time of
1200 // this writing it wasn't *quite* suitable to reuse more code from upstream
1201 // for our purposes, but we should strive to upstream this support once it's
1202 // ready to go! I figure we may want a bit of testing locally first before
1203 // sending this upstream to LLVM. I hear though they're quite eager to receive
1204 // feedback like this!
1206 // If you're reading this code and wondering "what in the world" or you're
1207 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1208 // then fear not! (ok maybe fear a little). All code here is mostly based
1209 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1211 // You'll find that the general layout here roughly corresponds to the `run`
1212 // method in that file as well as `ProcessThinLTOModule`. Functions are
1213 // specifically commented below as well, but if you're updating this code
1214 // or otherwise trying to understand it, the LLVM source will be useful in
1215 // interpreting the mysteries within.
1217 // Otherwise I'll apologize in advance, it probably requires a relatively
1218 // significant investment on your part to "truly understand" what's going on
1219 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1220 // and various online resources about ThinLTO to make heads or tails of all
1223 // This is a shared data structure which *must* be threadsafe to share
1224 // read-only amongst threads. This also corresponds basically to the arguments
1225 // of the `ProcessThinLTOModule` function in the LLVM source.
1226 struct LLVMRustThinLTOData {
1227 // The combined index that is the global analysis over all modules we're
1228 // performing ThinLTO for. This is mostly managed by LLVM.
1229 ModuleSummaryIndex Index;
1231 // All modules we may look at, stored as in-memory serialized versions. This
1232 // is later used when inlining to ensure we can extract any module to inline
1234 StringMap<MemoryBufferRef> ModuleMap;
1236 // A set that we manage of everything we *don't* want internalized. Note that
1237 // this includes all transitive references right now as well, but it may not
1239 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1241 // Not 100% sure what these are, but they impact what's internalized and
1242 // what's inlined across modules, I believe.
1243 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1244 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1245 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1246 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1248 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1251 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1252 struct LLVMRustThinLTOModule {
1253 const char *identifier;
1258 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1260 static const GlobalValueSummary *
1261 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1262 auto StrongDefForLinker = llvm::find_if(
1263 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1264 auto Linkage = Summary->linkage();
1265 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1266 !GlobalValue::isWeakForLinker(Linkage);
1268 if (StrongDefForLinker != GVSummaryList.end())
1269 return StrongDefForLinker->get();
1271 auto FirstDefForLinker = llvm::find_if(
1272 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1273 auto Linkage = Summary->linkage();
1274 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1276 if (FirstDefForLinker == GVSummaryList.end())
1278 return FirstDefForLinker->get();
1281 // The main entry point for creating the global ThinLTO analysis. The structure
1282 // here is basically the same as before threads are spawned in the `run`
1283 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1284 extern "C" LLVMRustThinLTOData*
1285 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1287 const char **preserved_symbols,
1289 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1291 // Load each module's summary and merge it into one combined index
1292 for (int i = 0; i < num_modules; i++) {
1293 auto module = &modules[i];
1294 StringRef buffer(module->data, module->len);
1295 MemoryBufferRef mem_buffer(buffer, module->identifier);
1297 Ret->ModuleMap[module->identifier] = mem_buffer;
1299 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1300 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1305 // Collect for each module the list of function it defines (GUID -> Summary)
1306 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1308 // Convert the preserved symbols set from string to GUID, this is then needed
1309 // for internalization.
1310 for (int i = 0; i < num_symbols; i++) {
1311 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1312 Ret->GUIDPreservedSymbols.insert(GUID);
1315 // Collect the import/export lists for all modules from the call-graph in the
1318 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1319 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1320 return PrevailingType::Unknown;
1322 // We don't have a complete picture in our use of ThinLTO, just our immediate
1323 // crate, so we need `ImportEnabled = false` to limit internalization.
1324 // Otherwise, we sometimes lose `static` values -- see #60184.
1325 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1326 deadIsPrevailing, /* ImportEnabled = */ false);
1327 ComputeCrossModuleImport(
1329 Ret->ModuleToDefinedGVSummaries,
1334 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1335 // impacts the caching.
1337 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1338 // being lifted from `lib/LTO/LTO.cpp` as well
1339 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1340 for (auto &I : Ret->Index) {
1341 if (I.second.SummaryList.size() > 1)
1342 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1344 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1345 const auto &Prevailing = PrevailingCopy.find(GUID);
1346 if (Prevailing == PrevailingCopy.end())
1348 return Prevailing->second == S;
1350 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1351 GlobalValue::GUID GUID,
1352 GlobalValue::LinkageTypes NewLinkage) {
1353 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1356 #if LLVM_VERSION_GE(13,0)
1357 // Uses FromPrevailing visibility scheme which works for many binary
1358 // formats. We probably could and should use ELF visibility scheme for many of
1359 // our targets, however.
1361 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1362 Ret->GUIDPreservedSymbols);
1364 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
1365 Ret->GUIDPreservedSymbols);
1367 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1368 // callback below. This callback below will dictate the linkage for all
1369 // summaries in the index, and we basically just only want to ensure that dead
1370 // symbols are internalized. Otherwise everything that's already external
1371 // linkage will stay as external, and internal will stay as internal.
1372 std::set<GlobalValue::GUID> ExportedGUIDs;
1373 for (auto &List : Ret->Index) {
1374 for (auto &GVS: List.second.SummaryList) {
1375 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1377 auto GUID = GVS->getOriginalName();
1378 if (GVS->flags().Live)
1379 ExportedGUIDs.insert(GUID);
1382 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1383 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1384 return (ExportList != Ret->ExportLists.end() &&
1385 ExportList->second.count(VI)) ||
1386 ExportedGUIDs.count(VI.getGUID());
1388 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1390 return Ret.release();
1394 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1398 // Below are the various passes that happen *per module* when doing ThinLTO.
1400 // In other words, these are the functions that are all run concurrently
1401 // with one another, one per module. The passes here correspond to the analysis
1402 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1403 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1404 // so rustc can save off the intermediate bytecode between each step.
1407 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1408 // When linking an ELF shared object, dso_local should be dropped. We
1409 // conservatively do this for -fpic.
1410 bool ClearDSOLocalOnDeclarations =
1411 TM.getTargetTriple().isOSBinFormatELF() &&
1412 TM.getRelocationModel() != Reloc::Static &&
1413 Mod.getPIELevel() == PIELevel::Default;
1414 return ClearDSOLocalOnDeclarations;
1418 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1419 LLVMTargetMachineRef TM) {
1420 Module &Mod = *unwrap(M);
1421 TargetMachine &Target = *unwrap(TM);
1423 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1424 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1427 LLVMRustSetLastError("renameModuleForThinLTO failed");
1434 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1435 Module &Mod = *unwrap(M);
1436 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1437 #if LLVM_VERSION_GE(14, 0)
1438 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1440 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1446 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1447 Module &Mod = *unwrap(M);
1448 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1449 thinLTOInternalizeModule(Mod, DefinedGlobals);
1454 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1455 LLVMTargetMachineRef TM) {
1456 Module &Mod = *unwrap(M);
1457 TargetMachine &Target = *unwrap(TM);
1459 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1460 auto Loader = [&](StringRef Identifier) {
1461 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1462 auto &Context = Mod.getContext();
1463 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1468 // The rest of this closure is a workaround for
1469 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1470 // we accidentally import wasm custom sections into different modules,
1471 // duplicating them by in the final output artifact.
1473 // The issue is worked around here by manually removing the
1474 // `wasm.custom_sections` named metadata node from any imported module. This
1475 // we know isn't used by any optimization pass so there's no need for it to
1478 // Note that the metadata is currently lazily loaded, so we materialize it
1479 // here before looking up if there's metadata inside. The `FunctionImporter`
1480 // will immediately materialize metadata anyway after an import, so this
1481 // shouldn't be a perf hit.
1482 if (Error Err = (*MOrErr)->materializeMetadata()) {
1483 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1487 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1488 if (WasmCustomSections)
1489 WasmCustomSections->eraseFromParent();
1493 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1494 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1495 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1497 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1503 extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1504 const char*, // importing module name
1505 const char*); // imported module name
1507 // Calls `module_name_callback` for each module import done by ThinLTO.
1508 // The callback is provided with regular null-terminated C strings.
1510 LLVMRustGetThinLTOModules(const LLVMRustThinLTOData *data,
1511 LLVMRustModuleNameCallback module_name_callback,
1512 void* callback_payload) {
1513 for (const auto& importing_module : data->ImportLists) {
1514 const std::string importing_module_id = importing_module.getKey().str();
1515 const auto& imports = importing_module.getValue();
1516 for (const auto& imported_module : imports) {
1517 const std::string imported_module_id = imported_module.getKey().str();
1518 module_name_callback(callback_payload,
1519 importing_module_id.c_str(),
1520 imported_module_id.c_str());
1525 // This struct and various functions are sort of a hack right now, but the
1526 // problem is that we've got in-memory LLVM modules after we generate and
1527 // optimize all codegen-units for one compilation in rustc. To be compatible
1528 // with the LTO support above we need to serialize the modules plus their
1529 // ThinLTO summary into memory.
1531 // This structure is basically an owned version of a serialize module, with
1532 // a ThinLTO summary attached.
1533 struct LLVMRustThinLTOBuffer {
1537 extern "C" LLVMRustThinLTOBuffer*
1538 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1539 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1541 raw_string_ostream OS(Ret->data);
1543 legacy::PassManager PM;
1544 PM.add(createWriteThinLTOBitcodePass(OS));
1548 return Ret.release();
1552 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1556 extern "C" const void*
1557 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1558 return Buffer->data.data();
1562 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1563 return Buffer->data.length();
1566 // This is what we used to parse upstream bitcode for actual ThinLTO
1567 // processing. We'll call this once per module optimized through ThinLTO, and
1568 // it'll be called concurrently on many threads.
1569 extern "C" LLVMModuleRef
1570 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1573 const char *identifier) {
1574 StringRef Data(data, len);
1575 MemoryBufferRef Buffer(Data, identifier);
1576 unwrap(Context)->enableDebugTypeODRUniquing();
1577 Expected<std::unique_ptr<Module>> SrcOrError =
1578 parseBitcodeFile(Buffer, *unwrap(Context));
1580 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1583 return wrap(std::move(*SrcOrError).release());
1586 // Find the bitcode section in the object file data and return it as a slice.
1587 // Fail if the bitcode section is present but empty.
1589 // On success, the return value is the pointer to the start of the slice and
1590 // `out_len` is filled with the (non-zero) length. On failure, the return value
1591 // is `nullptr` and `out_len` is set to zero.
1592 extern "C" const char*
1593 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1598 StringRef Data(data, len);
1599 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1601 Expected<MemoryBufferRef> BitcodeOrError =
1602 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1603 if (!BitcodeOrError) {
1604 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1608 *out_len = BitcodeOrError->getBufferSize();
1609 return BitcodeOrError->getBufferStart();
1612 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1613 // the comment in `back/lto.rs` for why this exists.
1615 LLVMRustLTOGetDICompileUnit(LLVMModuleRef Mod,
1617 DICompileUnit **B) {
1618 Module *M = unwrap(Mod);
1619 DICompileUnit **Cur = A;
1620 DICompileUnit **Next = B;
1621 for (DICompileUnit *CU : M->debug_compile_units()) {
1630 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1631 // the comment in `back/lto.rs` for why this exists.
1633 LLVMRustLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1634 Module *M = unwrap(Mod);
1636 // If the original source module didn't have a `DICompileUnit` then try to
1637 // merge all the existing compile units. If there aren't actually any though
1638 // then there's not much for us to do so return.
1639 if (Unit == nullptr) {
1640 for (DICompileUnit *CU : M->debug_compile_units()) {
1644 if (Unit == nullptr)
1648 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1649 // process it recursively. Note that we used to specifically iterate over
1650 // instructions to ensure we feed everything into it, but `processModule`
1651 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1652 DebugInfoFinder Finder;
1653 Finder.processModule(*M);
1655 // After we've found all our debuginfo, rewrite all subprograms to point to
1656 // the same `DICompileUnit`.
1657 for (auto &F : Finder.subprograms()) {
1658 F->replaceUnit(Unit);
1661 // Erase any other references to other `DICompileUnit` instances, the verifier
1662 // will later ensure that we don't actually have any other stale references to
1664 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1665 MD->clearOperands();
1666 MD->addOperand(Unit);
1669 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1670 // storing the result in 'KeyOut'.
1671 // Currently, this cache key is a SHA-1 hash of anything that could affect
1672 // the result of optimizing this module (e.g. module imports, exports, liveness
1673 // of access globals, etc).
1674 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1675 // used during the normal linker-plugin incremental thin-LTO process.
1677 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1678 SmallString<40> Key;
1679 llvm::lto::Config conf;
1680 const auto &ImportList = Data->ImportLists.lookup(ModId);
1681 const auto &ExportList = Data->ExportLists.lookup(ModId);
1682 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1683 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1684 std::set<GlobalValue::GUID> CfiFunctionDefs;
1685 std::set<GlobalValue::GUID> CfiFunctionDecls;
1687 // Based on the 'InProcessThinBackend' constructor in LLVM
1688 for (auto &Name : Data->Index.cfiFunctionDefs())
1689 CfiFunctionDefs.insert(
1690 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1691 for (auto &Name : Data->Index.cfiFunctionDecls())
1692 CfiFunctionDecls.insert(
1693 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1695 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1696 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1699 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());