6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/PassPlugin.h"
21 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #if LLVM_VERSION_LT(14, 0)
26 #include "llvm/Support/TargetRegistry.h"
28 #include "llvm/MC/TargetRegistry.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
32 #include "llvm/Transforms/IPO/AlwaysInliner.h"
33 #include "llvm/Transforms/IPO/FunctionImport.h"
34 #include "llvm/Transforms/Utils/AddDiscriminators.h"
35 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
36 #include "llvm/LTO/LTO.h"
37 #include "llvm-c/Transforms/PassManagerBuilder.h"
39 #include "llvm/Transforms/Instrumentation.h"
40 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
41 #include "llvm/Support/TimeProfiler.h"
42 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
43 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
44 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
45 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
46 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
47 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
48 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
49 #include "llvm/Transforms/Utils.h"
53 typedef struct LLVMOpaquePass *LLVMPassRef;
54 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
56 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
57 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
59 extern "C" void LLVMInitializePasses() {
60 PassRegistry &Registry = *PassRegistry::getPassRegistry();
61 initializeCore(Registry);
62 initializeCodeGen(Registry);
63 initializeScalarOpts(Registry);
64 initializeVectorization(Registry);
65 initializeIPO(Registry);
66 initializeAnalysis(Registry);
67 initializeTransformUtils(Registry);
68 initializeInstCombine(Registry);
69 initializeInstrumentation(Registry);
70 initializeTarget(Registry);
73 extern "C" void LLVMTimeTraceProfilerInitialize() {
74 timeTraceProfilerInitialize(
75 /* TimeTraceGranularity */ 0,
76 /* ProcName */ "rustc");
79 extern "C" void LLVMTimeTraceProfilerFinishThread() {
80 timeTraceProfilerFinishThread();
83 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
84 StringRef FN(FileName);
86 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
88 timeTraceProfilerWrite(OS);
89 timeTraceProfilerCleanup();
92 enum class LLVMRustPassKind {
98 static LLVMRustPassKind toRust(PassKind Kind) {
101 return LLVMRustPassKind::Function;
103 return LLVMRustPassKind::Module;
105 return LLVMRustPassKind::Other;
109 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
110 StringRef SR(PassName);
111 PassRegistry *PR = PassRegistry::getPassRegistry();
113 const PassInfo *PI = PR->getPassInfo(SR);
115 return wrap(PI->createPass());
120 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
121 const bool CompileKernel = false;
122 const bool UseAfterScope = true;
124 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
127 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
128 const bool CompileKernel = false;
130 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
133 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
134 const bool CompileKernel = false;
136 return wrap(createMemorySanitizerLegacyPassPass(
137 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
140 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
141 return wrap(createThreadSanitizerLegacyPassPass());
144 extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) {
145 const bool CompileKernel = false;
147 return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover));
150 extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
152 Pass *Pass = unwrap(RustPass);
153 return toRust(Pass->getPassKind());
156 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
158 Pass *Pass = unwrap(RustPass);
159 PassManagerBase *PMB = unwrap(PMR);
164 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
165 LLVMPassManagerBuilderRef PMBR,
166 LLVMPassManagerRef PMR
168 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
172 void LLVMRustAddLastExtensionPasses(
173 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
174 auto AddExtensionPasses = [Passes, NumPasses](
175 const PassManagerBuilder &Builder, PassManagerBase &PM) {
176 for (size_t I = 0; I < NumPasses; I++) {
177 PM.add(unwrap(Passes[I]));
180 // Add the passes to both of the pre-finalization extension points,
181 // so they are run for optimized and non-optimized builds.
182 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
184 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
188 #ifdef LLVM_COMPONENT_X86
189 #define SUBTARGET_X86 SUBTARGET(X86)
191 #define SUBTARGET_X86
194 #ifdef LLVM_COMPONENT_ARM
195 #define SUBTARGET_ARM SUBTARGET(ARM)
197 #define SUBTARGET_ARM
200 #ifdef LLVM_COMPONENT_AARCH64
201 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
203 #define SUBTARGET_AARCH64
206 #ifdef LLVM_COMPONENT_AVR
207 #define SUBTARGET_AVR SUBTARGET(AVR)
209 #define SUBTARGET_AVR
212 #ifdef LLVM_COMPONENT_M68k
213 #define SUBTARGET_M68K SUBTARGET(M68k)
215 #define SUBTARGET_M68K
218 #ifdef LLVM_COMPONENT_MIPS
219 #define SUBTARGET_MIPS SUBTARGET(Mips)
221 #define SUBTARGET_MIPS
224 #ifdef LLVM_COMPONENT_POWERPC
225 #define SUBTARGET_PPC SUBTARGET(PPC)
227 #define SUBTARGET_PPC
230 #ifdef LLVM_COMPONENT_SYSTEMZ
231 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
233 #define SUBTARGET_SYSTEMZ
236 #ifdef LLVM_COMPONENT_MSP430
237 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
239 #define SUBTARGET_MSP430
242 #ifdef LLVM_COMPONENT_RISCV
243 #define SUBTARGET_RISCV SUBTARGET(RISCV)
245 #define SUBTARGET_RISCV
248 #ifdef LLVM_COMPONENT_SPARC
249 #define SUBTARGET_SPARC SUBTARGET(Sparc)
251 #define SUBTARGET_SPARC
254 #ifdef LLVM_COMPONENT_HEXAGON
255 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
257 #define SUBTARGET_HEXAGON
260 #define GEN_SUBTARGETS \
274 #define SUBTARGET(x) \
276 extern const SubtargetFeatureKV x##FeatureKV[]; \
277 extern const SubtargetFeatureKV x##SubTypeKV[]; \
283 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
284 const char *Feature) {
285 TargetMachine *Target = unwrap(TM);
286 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
287 return MCInfo->checkFeatures(std::string("+") + Feature);
290 enum class LLVMRustCodeModel {
299 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
301 case LLVMRustCodeModel::Tiny:
302 return CodeModel::Tiny;
303 case LLVMRustCodeModel::Small:
304 return CodeModel::Small;
305 case LLVMRustCodeModel::Kernel:
306 return CodeModel::Kernel;
307 case LLVMRustCodeModel::Medium:
308 return CodeModel::Medium;
309 case LLVMRustCodeModel::Large:
310 return CodeModel::Large;
311 case LLVMRustCodeModel::None:
314 report_fatal_error("Bad CodeModel.");
318 enum class LLVMRustCodeGenOptLevel {
325 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
327 case LLVMRustCodeGenOptLevel::None:
328 return CodeGenOpt::None;
329 case LLVMRustCodeGenOptLevel::Less:
330 return CodeGenOpt::Less;
331 case LLVMRustCodeGenOptLevel::Default:
332 return CodeGenOpt::Default;
333 case LLVMRustCodeGenOptLevel::Aggressive:
334 return CodeGenOpt::Aggressive;
336 report_fatal_error("Bad CodeGenOptLevel.");
340 enum class LLVMRustPassBuilderOptLevel {
349 #if LLVM_VERSION_LT(14,0)
350 using OptimizationLevel = PassBuilder::OptimizationLevel;
353 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
355 case LLVMRustPassBuilderOptLevel::O0:
356 return OptimizationLevel::O0;
357 case LLVMRustPassBuilderOptLevel::O1:
358 return OptimizationLevel::O1;
359 case LLVMRustPassBuilderOptLevel::O2:
360 return OptimizationLevel::O2;
361 case LLVMRustPassBuilderOptLevel::O3:
362 return OptimizationLevel::O3;
363 case LLVMRustPassBuilderOptLevel::Os:
364 return OptimizationLevel::Os;
365 case LLVMRustPassBuilderOptLevel::Oz:
366 return OptimizationLevel::Oz;
368 report_fatal_error("Bad PassBuilderOptLevel.");
372 enum class LLVMRustRelocModel {
381 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
383 case LLVMRustRelocModel::Static:
384 return Reloc::Static;
385 case LLVMRustRelocModel::PIC:
387 case LLVMRustRelocModel::DynamicNoPic:
388 return Reloc::DynamicNoPIC;
389 case LLVMRustRelocModel::ROPI:
391 case LLVMRustRelocModel::RWPI:
393 case LLVMRustRelocModel::ROPIRWPI:
394 return Reloc::ROPI_RWPI;
396 report_fatal_error("Bad RelocModel.");
400 /// getLongestEntryLength - Return the length of the longest entry in the table.
401 template<typename KV>
402 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
404 for (auto &I : Table)
405 MaxLen = std::max(MaxLen, std::strlen(I.Key));
409 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
410 const TargetMachine *Target = unwrap(TM);
411 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
412 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
413 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
414 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
415 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
417 printf("Available CPUs for this target:\n");
418 if (HostArch == TargetArch) {
419 const StringRef HostCPU = sys::getHostCPUName();
420 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
421 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
423 for (auto &CPU : CPUTable)
424 printf(" %-*s\n", MaxCPULen, CPU.Key);
428 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
429 const TargetMachine *Target = unwrap(TM);
430 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
431 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
432 return FeatTable.size();
435 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
436 const char** Feature, const char** Desc) {
437 const TargetMachine *Target = unwrap(TM);
438 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
439 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
440 const SubtargetFeatureKV Feat = FeatTable[Index];
447 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
448 printf("Target CPU help is not supported by this LLVM version.\n\n");
451 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
455 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
458 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
459 StringRef Name = sys::getHostCPUName();
464 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
465 const char *TripleStr, const char *CPU, const char *Feature,
466 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
467 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
468 bool FunctionSections,
470 bool UniqueSectionNames,
471 bool TrapUnreachable,
474 bool EmitStackSizeSection,
475 bool RelaxELFRelocations,
477 const char *SplitDwarfFile) {
479 auto OptLevel = fromRust(RustOptLevel);
480 auto RM = fromRust(RustReloc);
481 auto CM = fromRust(RustCM);
484 Triple Trip(Triple::normalize(TripleStr));
485 const llvm::Target *TheTarget =
486 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
487 if (TheTarget == nullptr) {
488 LLVMRustSetLastError(Error.c_str());
492 TargetOptions Options;
494 Options.FloatABIType = FloatABI::Default;
496 Options.FloatABIType = FloatABI::Soft;
498 Options.DataSections = DataSections;
499 Options.FunctionSections = FunctionSections;
500 Options.UniqueSectionNames = UniqueSectionNames;
501 Options.MCOptions.AsmVerbose = AsmComments;
502 Options.MCOptions.PreserveAsmComments = AsmComments;
503 Options.MCOptions.ABIName = ABIStr;
504 if (SplitDwarfFile) {
505 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
507 Options.RelaxELFRelocations = RelaxELFRelocations;
508 Options.UseInitArray = UseInitArray;
510 if (TrapUnreachable) {
511 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
512 // This limits the extent of possible undefined behavior in some cases, as
513 // it prevents control flow from "falling through" into whatever code
514 // happens to be laid out next in memory.
515 Options.TrapUnreachable = true;
519 Options.ThreadModel = ThreadModel::Single;
522 Options.EmitStackSizeSection = EmitStackSizeSection;
524 TargetMachine *TM = TheTarget->createTargetMachine(
525 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
529 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
533 extern "C" void LLVMRustConfigurePassManagerBuilder(
534 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
535 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
536 const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath) {
537 unwrap(PMBR)->MergeFunctions = MergeFunctions;
538 unwrap(PMBR)->SLPVectorize = SLPVectorize;
539 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
540 unwrap(PMBR)->LoopVectorize = LoopVectorize;
541 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
544 assert(!PGOUsePath && !PGOSampleUsePath);
545 unwrap(PMBR)->EnablePGOInstrGen = true;
546 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
547 } else if (PGOUsePath) {
548 assert(!PGOSampleUsePath);
549 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
550 } else if (PGOSampleUsePath) {
551 unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath;
555 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
556 // field of a PassManagerBuilder, we expose our own method of doing so.
557 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
559 bool DisableSimplifyLibCalls) {
560 Triple TargetTriple(unwrap(M)->getTargetTriple());
561 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
562 if (DisableSimplifyLibCalls)
563 TLI->disableAllFunctions();
564 unwrap(PMBR)->LibraryInfo = TLI;
567 // Unfortunately, the LLVM C API doesn't provide a way to create the
568 // TargetLibraryInfo pass, so we use this method to do so.
569 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
570 bool DisableSimplifyLibCalls) {
571 Triple TargetTriple(unwrap(M)->getTargetTriple());
572 TargetLibraryInfoImpl TLII(TargetTriple);
573 if (DisableSimplifyLibCalls)
574 TLII.disableAllFunctions();
575 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
578 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
579 // all the functions in a module, so we do that manually here. You'll find
580 // similar code in clang's BackendUtil.cpp file.
581 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
583 llvm::legacy::FunctionPassManager *P =
584 unwrap<llvm::legacy::FunctionPassManager>(PMR);
585 P->doInitialization();
587 // Upgrade all calls to old intrinsics first.
588 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
589 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
591 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
593 if (!I->isDeclaration())
599 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
600 // Initializing the command-line options more than once is not allowed. So,
601 // check if they've already been initialized. (This could happen if we're
602 // being called from rustpkg, for example). If the arguments change, then
603 // that's just kinda unfortunate.
604 static bool Initialized = false;
608 cl::ParseCommandLineOptions(Argc, Argv);
611 enum class LLVMRustFileType {
616 static CodeGenFileType fromRust(LLVMRustFileType Type) {
618 case LLVMRustFileType::AssemblyFile:
619 return CGFT_AssemblyFile;
620 case LLVMRustFileType::ObjectFile:
621 return CGFT_ObjectFile;
623 report_fatal_error("Bad FileType.");
627 extern "C" LLVMRustResult
628 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
629 LLVMModuleRef M, const char *Path, const char *DwoPath,
630 LLVMRustFileType RustFileType) {
631 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
632 auto FileType = fromRust(RustFileType);
634 std::string ErrorInfo;
636 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
638 ErrorInfo = EC.message();
639 if (ErrorInfo != "") {
640 LLVMRustSetLastError(ErrorInfo.c_str());
641 return LLVMRustResult::Failure;
644 buffer_ostream BOS(OS);
646 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
649 ErrorInfo = EC.message();
650 if (ErrorInfo != "") {
651 LLVMRustSetLastError(ErrorInfo.c_str());
652 return LLVMRustResult::Failure;
654 buffer_ostream DBOS(DOS);
655 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
658 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
662 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
663 // stream (OS), so the only real safe place to delete this is here? Don't we
664 // wish this was written in Rust?
665 LLVMDisposePassManager(PMR);
666 return LLVMRustResult::Success;
669 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
670 const char*, // pass name
671 const char*); // IR name
672 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
674 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
675 if (any_isa<const Module *>(WrappedIr))
676 return any_cast<const Module *>(WrappedIr)->getName().str();
677 if (any_isa<const Function *>(WrappedIr))
678 return any_cast<const Function *>(WrappedIr)->getName().str();
679 if (any_isa<const Loop *>(WrappedIr))
680 return any_cast<const Loop *>(WrappedIr)->getName().str();
681 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
682 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
687 void LLVMSelfProfileInitializeCallbacks(
688 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
689 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
690 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
691 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
692 StringRef Pass, llvm::Any Ir) {
693 std::string PassName = Pass.str();
694 std::string IrName = LLVMRustwrappedIrGetName(Ir);
695 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
698 PIC.registerAfterPassCallback(
699 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
700 const PreservedAnalyses &Preserved) {
701 AfterPassCallback(LlvmSelfProfiler);
704 PIC.registerAfterPassInvalidatedCallback(
705 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
706 AfterPassCallback(LlvmSelfProfiler);
709 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
710 StringRef Pass, llvm::Any Ir) {
711 std::string PassName = Pass.str();
712 std::string IrName = LLVMRustwrappedIrGetName(Ir);
713 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
716 PIC.registerAfterAnalysisCallback(
717 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
718 AfterPassCallback(LlvmSelfProfiler);
722 enum class LLVMRustOptStage {
730 struct LLVMRustSanitizerOptions {
731 bool SanitizeAddress;
732 bool SanitizeAddressRecover;
734 bool SanitizeMemoryRecover;
735 int SanitizeMemoryTrackOrigins;
737 bool SanitizeHWAddress;
738 bool SanitizeHWAddressRecover;
741 extern "C" LLVMRustResult
742 LLVMRustOptimizeWithNewPassManager(
743 LLVMModuleRef ModuleRef,
744 LLVMTargetMachineRef TMRef,
745 LLVMRustPassBuilderOptLevel OptLevelRust,
746 LLVMRustOptStage OptStage,
747 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
748 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
749 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
750 LLVMRustSanitizerOptions *SanitizerOptions,
751 const char *PGOGenPath, const char *PGOUsePath,
752 bool InstrumentCoverage, bool InstrumentGCOV,
753 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
754 void* LlvmSelfProfiler,
755 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
756 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
757 const char *ExtraPasses, size_t ExtraPassesLen,
758 const char *LLVMPlugins, size_t LLVMPluginsLen) {
759 Module *TheModule = unwrap(ModuleRef);
760 TargetMachine *TM = unwrap(TMRef);
761 OptimizationLevel OptLevel = fromRust(OptLevelRust);
764 PipelineTuningOptions PTO;
765 PTO.LoopUnrolling = UnrollLoops;
766 PTO.LoopInterleaving = UnrollLoops;
767 PTO.LoopVectorization = LoopVectorize;
768 PTO.SLPVectorization = SLPVectorize;
769 PTO.MergeFunctions = MergeFunctions;
771 // FIXME: We may want to expose this as an option.
772 bool DebugPassManager = false;
774 PassInstrumentationCallbacks PIC;
775 StandardInstrumentations SI(DebugPassManager);
776 SI.registerCallbacks(PIC);
778 if (LlvmSelfProfiler){
779 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
782 Optional<PGOOptions> PGOOpt;
784 assert(!PGOUsePath && !PGOSampleUsePath);
785 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
786 PGOOptions::NoCSAction, DebugInfoForProfiling);
787 } else if (PGOUsePath) {
788 assert(!PGOSampleUsePath);
789 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
790 PGOOptions::NoCSAction, DebugInfoForProfiling);
791 } else if (PGOSampleUsePath) {
792 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
793 PGOOptions::NoCSAction, DebugInfoForProfiling);
794 } else if (DebugInfoForProfiling) {
795 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
796 PGOOptions::NoCSAction, DebugInfoForProfiling);
799 #if LLVM_VERSION_GE(13, 0)
800 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
801 LoopAnalysisManager LAM;
802 FunctionAnalysisManager FAM;
803 CGSCCAnalysisManager CGAM;
804 ModuleAnalysisManager MAM;
806 PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC);
807 LoopAnalysisManager LAM(DebugPassManager);
808 FunctionAnalysisManager FAM(DebugPassManager);
809 CGSCCAnalysisManager CGAM(DebugPassManager);
810 ModuleAnalysisManager MAM(DebugPassManager);
813 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
815 Triple TargetTriple(TheModule->getTargetTriple());
816 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
817 if (DisableSimplifyLibCalls)
818 TLII->disableAllFunctions();
819 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
821 PB.registerModuleAnalyses(MAM);
822 PB.registerCGSCCAnalyses(CGAM);
823 PB.registerFunctionAnalyses(FAM);
824 PB.registerLoopAnalyses(LAM);
825 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
827 // We manually collect pipeline callbacks so we can apply them at O0, where the
828 // PassBuilder does not create a pipeline.
829 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
830 PipelineStartEPCallbacks;
831 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
832 OptimizerLastEPCallbacks;
835 PipelineStartEPCallbacks.push_back(
836 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
837 MPM.addPass(VerifierPass());
842 if (InstrumentGCOV) {
843 PipelineStartEPCallbacks.push_back(
844 [](ModulePassManager &MPM, OptimizationLevel Level) {
845 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
850 if (InstrumentCoverage) {
851 PipelineStartEPCallbacks.push_back(
852 [](ModulePassManager &MPM, OptimizationLevel Level) {
853 InstrProfOptions Options;
854 MPM.addPass(InstrProfiling(Options, false));
859 if (SanitizerOptions) {
860 if (SanitizerOptions->SanitizeMemory) {
861 MemorySanitizerOptions Options(
862 SanitizerOptions->SanitizeMemoryTrackOrigins,
863 SanitizerOptions->SanitizeMemoryRecover,
864 /*CompileKernel=*/false);
865 OptimizerLastEPCallbacks.push_back(
866 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
867 #if LLVM_VERSION_GE(14, 0)
868 MPM.addPass(ModuleMemorySanitizerPass(Options));
870 MPM.addPass(MemorySanitizerPass(Options));
872 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
877 if (SanitizerOptions->SanitizeThread) {
878 OptimizerLastEPCallbacks.push_back(
879 [](ModulePassManager &MPM, OptimizationLevel Level) {
880 #if LLVM_VERSION_GE(14, 0)
881 MPM.addPass(ModuleThreadSanitizerPass());
883 MPM.addPass(ThreadSanitizerPass());
885 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
890 if (SanitizerOptions->SanitizeAddress) {
891 OptimizerLastEPCallbacks.push_back(
892 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
893 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
894 #if LLVM_VERSION_GE(14, 0)
895 AddressSanitizerOptions opts = AddressSanitizerOptions{
896 /*CompileKernel=*/false,
897 SanitizerOptions->SanitizeAddressRecover,
898 /*UseAfterScope=*/true,
899 AsanDetectStackUseAfterReturnMode::Runtime,
901 MPM.addPass(ModuleAddressSanitizerPass(opts));
903 MPM.addPass(ModuleAddressSanitizerPass(
904 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
905 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
906 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
907 /*UseAfterScope=*/true)));
912 if (SanitizerOptions->SanitizeHWAddress) {
913 OptimizerLastEPCallbacks.push_back(
914 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
915 #if LLVM_VERSION_GE(14, 0)
916 HWAddressSanitizerOptions opts(
917 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
918 /*DisableOptimization=*/false);
919 MPM.addPass(HWAddressSanitizerPass(opts));
921 MPM.addPass(HWAddressSanitizerPass(
922 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
929 if (LLVMPluginsLen) {
930 auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
931 SmallVector<StringRef> Plugins;
932 PluginsStr.split(Plugins, ',', -1, false);
933 for (auto PluginPath: Plugins) {
934 auto Plugin = PassPlugin::Load(PluginPath.str());
936 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
939 Plugin->registerPassBuilderCallbacks(PB);
943 #if LLVM_VERSION_GE(13, 0)
944 ModulePassManager MPM;
946 ModulePassManager MPM(DebugPassManager);
948 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
949 if (!NoPrepopulatePasses) {
950 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
951 // At the same time, the LTO pipelines do support O0 and using them is required.
952 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
953 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
954 for (const auto &C : PipelineStartEPCallbacks)
955 PB.registerPipelineStartEPCallback(C);
956 for (const auto &C : OptimizerLastEPCallbacks)
957 PB.registerOptimizerLastEPCallback(C);
959 // Pass false as we manually schedule ThinLTOBufferPasses below.
960 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
962 for (const auto &C : PipelineStartEPCallbacks)
963 PB.registerPipelineStartEPCallback(C);
964 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
965 for (const auto &C : OptimizerLastEPCallbacks)
966 PB.registerOptimizerLastEPCallback(C);
970 case LLVMRustOptStage::PreLinkNoLTO:
971 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
973 case LLVMRustOptStage::PreLinkThinLTO:
974 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
975 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
976 // passes may still run afterwards. This means we need to run the buffer passes again.
977 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
978 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
979 if (OptimizerLastEPCallbacks.empty())
980 NeedThinLTOBufferPasses = false;
981 for (const auto &C : OptimizerLastEPCallbacks)
984 case LLVMRustOptStage::PreLinkFatLTO:
985 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
986 NeedThinLTOBufferPasses = false;
988 case LLVMRustOptStage::ThinLTO:
989 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
990 // It only seems to be needed for C++ specific optimizations.
991 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
993 case LLVMRustOptStage::FatLTO:
994 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
1000 if (ExtraPassesLen) {
1001 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
1002 std::string ErrMsg = toString(std::move(Err));
1003 LLVMRustSetLastError(ErrMsg.c_str());
1004 return LLVMRustResult::Failure;
1008 if (NeedThinLTOBufferPasses) {
1009 MPM.addPass(CanonicalizeAliasesPass());
1010 MPM.addPass(NameAnonGlobalPass());
1013 // Upgrade all calls to old intrinsics first.
1014 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
1015 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
1017 MPM.run(*TheModule, MAM);
1018 return LLVMRustResult::Success;
1021 // Callback to demangle function name
1023 // * name to be demangled
1026 // * output buffer len
1027 // Returns len of demangled string, or 0 if demangle failed.
1028 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1033 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1034 DemangleFn Demangle;
1035 std::vector<char> Buf;
1038 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1040 // Return empty string if demangle failed
1041 // or if name does not need to be demangled
1042 StringRef CallDemangle(StringRef name) {
1047 if (Buf.size() < name.size() * 2) {
1048 // Semangled name usually shorter than mangled,
1049 // but allocate twice as much memory just in case
1050 Buf.resize(name.size() * 2);
1053 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1059 auto Demangled = StringRef(Buf.data(), R);
1060 if (Demangled == name) {
1061 // Do not print anything if demangled name is equal to mangled.
1068 void emitFunctionAnnot(const Function *F,
1069 formatted_raw_ostream &OS) override {
1070 StringRef Demangled = CallDemangle(F->getName());
1071 if (Demangled.empty()) {
1075 OS << "; " << Demangled << "\n";
1078 void emitInstructionAnnot(const Instruction *I,
1079 formatted_raw_ostream &OS) override {
1082 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1084 Value = CI->getCalledOperand();
1085 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1087 Value = II->getCalledOperand();
1089 // Could demangle more operations, e. g.
1090 // `store %place, @function`.
1094 if (!Value->hasName()) {
1098 StringRef Demangled = CallDemangle(Value->getName());
1099 if (Demangled.empty()) {
1103 OS << "; " << Name << " " << Demangled << "\n";
1109 extern "C" LLVMRustResult
1110 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1111 std::string ErrorInfo;
1113 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
1115 ErrorInfo = EC.message();
1116 if (ErrorInfo != "") {
1117 LLVMRustSetLastError(ErrorInfo.c_str());
1118 return LLVMRustResult::Failure;
1121 RustAssemblyAnnotationWriter AAW(Demangle);
1122 formatted_raw_ostream FOS(OS);
1123 unwrap(M)->print(FOS, &AAW);
1125 return LLVMRustResult::Success;
1128 extern "C" void LLVMRustPrintPasses() {
1129 LLVMInitializePasses();
1130 struct MyListener : PassRegistrationListener {
1131 void passEnumerate(const PassInfo *Info) {
1132 StringRef PassArg = Info->getPassArgument();
1133 StringRef PassName = Info->getPassName();
1134 if (!PassArg.empty()) {
1135 // These unsigned->signed casts could theoretically overflow, but
1136 // realistically never will (and even if, the result is implementation
1137 // defined rather plain UB).
1138 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1139 (int)PassName.size(), PassName.data());
1144 PassRegistry *PR = PassRegistry::getPassRegistry();
1145 PR->enumerateWith(&Listener);
1148 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1149 bool AddLifetimes) {
1150 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1153 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1155 llvm::legacy::PassManager passes;
1157 auto PreserveFunctions = [=](const GlobalValue &GV) {
1158 for (size_t I = 0; I < Len; I++) {
1159 if (GV.getName() == Symbols[I]) {
1166 passes.add(llvm::createInternalizePass(PreserveFunctions));
1168 passes.run(*unwrap(M));
1172 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1173 LLVMTargetMachineRef TMR) {
1174 TargetMachine *Target = unwrap(TMR);
1175 unwrap(Module)->setDataLayout(Target->createDataLayout());
1178 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1179 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1182 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1183 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1186 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1187 LLVMRustCodeModel Model) {
1188 auto CM = fromRust(Model);
1191 unwrap(M)->setCodeModel(*CM);
1194 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1195 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1196 // LLVM, and otherwise it's just blanket rejected from other compilers.
1198 // Most of this implementation is straight copied from LLVM. At the time of
1199 // this writing it wasn't *quite* suitable to reuse more code from upstream
1200 // for our purposes, but we should strive to upstream this support once it's
1201 // ready to go! I figure we may want a bit of testing locally first before
1202 // sending this upstream to LLVM. I hear though they're quite eager to receive
1203 // feedback like this!
1205 // If you're reading this code and wondering "what in the world" or you're
1206 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1207 // then fear not! (ok maybe fear a little). All code here is mostly based
1208 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1210 // You'll find that the general layout here roughly corresponds to the `run`
1211 // method in that file as well as `ProcessThinLTOModule`. Functions are
1212 // specifically commented below as well, but if you're updating this code
1213 // or otherwise trying to understand it, the LLVM source will be useful in
1214 // interpreting the mysteries within.
1216 // Otherwise I'll apologize in advance, it probably requires a relatively
1217 // significant investment on your part to "truly understand" what's going on
1218 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1219 // and various online resources about ThinLTO to make heads or tails of all
1222 // This is a shared data structure which *must* be threadsafe to share
1223 // read-only amongst threads. This also corresponds basically to the arguments
1224 // of the `ProcessThinLTOModule` function in the LLVM source.
1225 struct LLVMRustThinLTOData {
1226 // The combined index that is the global analysis over all modules we're
1227 // performing ThinLTO for. This is mostly managed by LLVM.
1228 ModuleSummaryIndex Index;
1230 // All modules we may look at, stored as in-memory serialized versions. This
1231 // is later used when inlining to ensure we can extract any module to inline
1233 StringMap<MemoryBufferRef> ModuleMap;
1235 // A set that we manage of everything we *don't* want internalized. Note that
1236 // this includes all transitive references right now as well, but it may not
1238 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1240 // Not 100% sure what these are, but they impact what's internalized and
1241 // what's inlined across modules, I believe.
1242 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1243 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1244 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1245 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1247 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1250 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1251 struct LLVMRustThinLTOModule {
1252 const char *identifier;
1257 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1259 static const GlobalValueSummary *
1260 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1261 auto StrongDefForLinker = llvm::find_if(
1262 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1263 auto Linkage = Summary->linkage();
1264 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1265 !GlobalValue::isWeakForLinker(Linkage);
1267 if (StrongDefForLinker != GVSummaryList.end())
1268 return StrongDefForLinker->get();
1270 auto FirstDefForLinker = llvm::find_if(
1271 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1272 auto Linkage = Summary->linkage();
1273 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1275 if (FirstDefForLinker == GVSummaryList.end())
1277 return FirstDefForLinker->get();
1280 // The main entry point for creating the global ThinLTO analysis. The structure
1281 // here is basically the same as before threads are spawned in the `run`
1282 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1283 extern "C" LLVMRustThinLTOData*
1284 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1286 const char **preserved_symbols,
1288 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1290 // Load each module's summary and merge it into one combined index
1291 for (int i = 0; i < num_modules; i++) {
1292 auto module = &modules[i];
1293 StringRef buffer(module->data, module->len);
1294 MemoryBufferRef mem_buffer(buffer, module->identifier);
1296 Ret->ModuleMap[module->identifier] = mem_buffer;
1298 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1299 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1304 // Collect for each module the list of function it defines (GUID -> Summary)
1305 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1307 // Convert the preserved symbols set from string to GUID, this is then needed
1308 // for internalization.
1309 for (int i = 0; i < num_symbols; i++) {
1310 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1311 Ret->GUIDPreservedSymbols.insert(GUID);
1314 // Collect the import/export lists for all modules from the call-graph in the
1317 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1318 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1319 return PrevailingType::Unknown;
1321 // We don't have a complete picture in our use of ThinLTO, just our immediate
1322 // crate, so we need `ImportEnabled = false` to limit internalization.
1323 // Otherwise, we sometimes lose `static` values -- see #60184.
1324 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1325 deadIsPrevailing, /* ImportEnabled = */ false);
1326 ComputeCrossModuleImport(
1328 Ret->ModuleToDefinedGVSummaries,
1333 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1334 // impacts the caching.
1336 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1337 // being lifted from `lib/LTO/LTO.cpp` as well
1338 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1339 for (auto &I : Ret->Index) {
1340 if (I.second.SummaryList.size() > 1)
1341 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1343 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1344 const auto &Prevailing = PrevailingCopy.find(GUID);
1345 if (Prevailing == PrevailingCopy.end())
1347 return Prevailing->second == S;
1349 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1350 GlobalValue::GUID GUID,
1351 GlobalValue::LinkageTypes NewLinkage) {
1352 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1355 #if LLVM_VERSION_GE(13,0)
1356 // Uses FromPrevailing visibility scheme which works for many binary
1357 // formats. We probably could and should use ELF visibility scheme for many of
1358 // our targets, however.
1360 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1361 Ret->GUIDPreservedSymbols);
1363 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
1364 Ret->GUIDPreservedSymbols);
1366 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1367 // callback below. This callback below will dictate the linkage for all
1368 // summaries in the index, and we basically just only want to ensure that dead
1369 // symbols are internalized. Otherwise everything that's already external
1370 // linkage will stay as external, and internal will stay as internal.
1371 std::set<GlobalValue::GUID> ExportedGUIDs;
1372 for (auto &List : Ret->Index) {
1373 for (auto &GVS: List.second.SummaryList) {
1374 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1376 auto GUID = GVS->getOriginalName();
1377 if (GVS->flags().Live)
1378 ExportedGUIDs.insert(GUID);
1381 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1382 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1383 return (ExportList != Ret->ExportLists.end() &&
1384 ExportList->second.count(VI)) ||
1385 ExportedGUIDs.count(VI.getGUID());
1387 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1389 return Ret.release();
1393 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1397 // Below are the various passes that happen *per module* when doing ThinLTO.
1399 // In other words, these are the functions that are all run concurrently
1400 // with one another, one per module. The passes here correspond to the analysis
1401 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1402 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1403 // so rustc can save off the intermediate bytecode between each step.
1406 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1407 // When linking an ELF shared object, dso_local should be dropped. We
1408 // conservatively do this for -fpic.
1409 bool ClearDSOLocalOnDeclarations =
1410 TM.getTargetTriple().isOSBinFormatELF() &&
1411 TM.getRelocationModel() != Reloc::Static &&
1412 Mod.getPIELevel() == PIELevel::Default;
1413 return ClearDSOLocalOnDeclarations;
1417 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1418 LLVMTargetMachineRef TM) {
1419 Module &Mod = *unwrap(M);
1420 TargetMachine &Target = *unwrap(TM);
1422 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1423 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1426 LLVMRustSetLastError("renameModuleForThinLTO failed");
1433 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1434 Module &Mod = *unwrap(M);
1435 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1436 #if LLVM_VERSION_GE(14, 0)
1437 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1439 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1445 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1446 Module &Mod = *unwrap(M);
1447 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1448 thinLTOInternalizeModule(Mod, DefinedGlobals);
1453 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1454 LLVMTargetMachineRef TM) {
1455 Module &Mod = *unwrap(M);
1456 TargetMachine &Target = *unwrap(TM);
1458 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1459 auto Loader = [&](StringRef Identifier) {
1460 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1461 auto &Context = Mod.getContext();
1462 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1467 // The rest of this closure is a workaround for
1468 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1469 // we accidentally import wasm custom sections into different modules,
1470 // duplicating them by in the final output artifact.
1472 // The issue is worked around here by manually removing the
1473 // `wasm.custom_sections` named metadata node from any imported module. This
1474 // we know isn't used by any optimization pass so there's no need for it to
1477 // Note that the metadata is currently lazily loaded, so we materialize it
1478 // here before looking up if there's metadata inside. The `FunctionImporter`
1479 // will immediately materialize metadata anyway after an import, so this
1480 // shouldn't be a perf hit.
1481 if (Error Err = (*MOrErr)->materializeMetadata()) {
1482 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1486 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1487 if (WasmCustomSections)
1488 WasmCustomSections->eraseFromParent();
1492 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1493 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1494 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1496 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1502 extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1503 const char*, // importing module name
1504 const char*); // imported module name
1506 // Calls `module_name_callback` for each module import done by ThinLTO.
1507 // The callback is provided with regular null-terminated C strings.
1509 LLVMRustGetThinLTOModules(const LLVMRustThinLTOData *data,
1510 LLVMRustModuleNameCallback module_name_callback,
1511 void* callback_payload) {
1512 for (const auto& importing_module : data->ImportLists) {
1513 const std::string importing_module_id = importing_module.getKey().str();
1514 const auto& imports = importing_module.getValue();
1515 for (const auto& imported_module : imports) {
1516 const std::string imported_module_id = imported_module.getKey().str();
1517 module_name_callback(callback_payload,
1518 importing_module_id.c_str(),
1519 imported_module_id.c_str());
1524 // This struct and various functions are sort of a hack right now, but the
1525 // problem is that we've got in-memory LLVM modules after we generate and
1526 // optimize all codegen-units for one compilation in rustc. To be compatible
1527 // with the LTO support above we need to serialize the modules plus their
1528 // ThinLTO summary into memory.
1530 // This structure is basically an owned version of a serialize module, with
1531 // a ThinLTO summary attached.
1532 struct LLVMRustThinLTOBuffer {
1536 extern "C" LLVMRustThinLTOBuffer*
1537 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1538 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1540 raw_string_ostream OS(Ret->data);
1542 legacy::PassManager PM;
1543 PM.add(createWriteThinLTOBitcodePass(OS));
1547 return Ret.release();
1551 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1555 extern "C" const void*
1556 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1557 return Buffer->data.data();
1561 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1562 return Buffer->data.length();
1565 // This is what we used to parse upstream bitcode for actual ThinLTO
1566 // processing. We'll call this once per module optimized through ThinLTO, and
1567 // it'll be called concurrently on many threads.
1568 extern "C" LLVMModuleRef
1569 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1572 const char *identifier) {
1573 StringRef Data(data, len);
1574 MemoryBufferRef Buffer(Data, identifier);
1575 unwrap(Context)->enableDebugTypeODRUniquing();
1576 Expected<std::unique_ptr<Module>> SrcOrError =
1577 parseBitcodeFile(Buffer, *unwrap(Context));
1579 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1582 return wrap(std::move(*SrcOrError).release());
1585 // Find the bitcode section in the object file data and return it as a slice.
1586 // Fail if the bitcode section is present but empty.
1588 // On success, the return value is the pointer to the start of the slice and
1589 // `out_len` is filled with the (non-zero) length. On failure, the return value
1590 // is `nullptr` and `out_len` is set to zero.
1591 extern "C" const char*
1592 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1597 StringRef Data(data, len);
1598 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1600 Expected<MemoryBufferRef> BitcodeOrError =
1601 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1602 if (!BitcodeOrError) {
1603 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1607 *out_len = BitcodeOrError->getBufferSize();
1608 return BitcodeOrError->getBufferStart();
1611 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1612 // the comment in `back/lto.rs` for why this exists.
1614 LLVMRustLTOGetDICompileUnit(LLVMModuleRef Mod,
1616 DICompileUnit **B) {
1617 Module *M = unwrap(Mod);
1618 DICompileUnit **Cur = A;
1619 DICompileUnit **Next = B;
1620 for (DICompileUnit *CU : M->debug_compile_units()) {
1629 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1630 // the comment in `back/lto.rs` for why this exists.
1632 LLVMRustLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1633 Module *M = unwrap(Mod);
1635 // If the original source module didn't have a `DICompileUnit` then try to
1636 // merge all the existing compile units. If there aren't actually any though
1637 // then there's not much for us to do so return.
1638 if (Unit == nullptr) {
1639 for (DICompileUnit *CU : M->debug_compile_units()) {
1643 if (Unit == nullptr)
1647 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1648 // process it recursively. Note that we used to specifically iterate over
1649 // instructions to ensure we feed everything into it, but `processModule`
1650 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1651 DebugInfoFinder Finder;
1652 Finder.processModule(*M);
1654 // After we've found all our debuginfo, rewrite all subprograms to point to
1655 // the same `DICompileUnit`.
1656 for (auto &F : Finder.subprograms()) {
1657 F->replaceUnit(Unit);
1660 // Erase any other references to other `DICompileUnit` instances, the verifier
1661 // will later ensure that we don't actually have any other stale references to
1663 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1664 MD->clearOperands();
1665 MD->addOperand(Unit);
1668 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1669 // storing the result in 'KeyOut'.
1670 // Currently, this cache key is a SHA-1 hash of anything that could affect
1671 // the result of optimizing this module (e.g. module imports, exports, liveness
1672 // of access globals, etc).
1673 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1674 // used during the normal linker-plugin incremental thin-LTO process.
1676 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1677 SmallString<40> Key;
1678 llvm::lto::Config conf;
1679 const auto &ImportList = Data->ImportLists.lookup(ModId);
1680 const auto &ExportList = Data->ExportLists.lookup(ModId);
1681 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1682 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1683 std::set<GlobalValue::GUID> CfiFunctionDefs;
1684 std::set<GlobalValue::GUID> CfiFunctionDecls;
1686 // Based on the 'InProcessThinBackend' constructor in LLVM
1687 for (auto &Name : Data->Index.cfiFunctionDefs())
1688 CfiFunctionDefs.insert(
1689 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1690 for (auto &Name : Data->Index.cfiFunctionDecls())
1691 CfiFunctionDecls.insert(
1692 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1694 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1695 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1698 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());