8 #include "llvm/Analysis/TargetLibraryInfo.h"
9 #include "llvm/Analysis/TargetTransformInfo.h"
10 #include "llvm/CodeGen/TargetSubtargetInfo.h"
11 #include "llvm/InitializePasses.h"
12 #include "llvm/IR/AutoUpgrade.h"
13 #include "llvm/IR/AssemblyAnnotationWriter.h"
14 #include "llvm/IR/IntrinsicInst.h"
15 #include "llvm/IR/Verifier.h"
16 #include "llvm/Object/ObjectFile.h"
17 #include "llvm/Object/IRObjectFile.h"
18 #include "llvm/Passes/PassBuilder.h"
19 #if LLVM_VERSION_GE(9, 0)
20 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #include "llvm/Target/TargetMachine.h"
26 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
27 #include "llvm/Transforms/IPO/AlwaysInliner.h"
28 #include "llvm/Transforms/IPO/FunctionImport.h"
29 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
30 #include "llvm/LTO/LTO.h"
31 #include "llvm-c/Transforms/PassManagerBuilder.h"
33 #include "llvm/Transforms/Instrumentation.h"
34 #if LLVM_VERSION_GE(9, 0)
35 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
36 #include "llvm/Support/TimeProfiler.h"
38 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
39 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
40 #if LLVM_VERSION_GE(9, 0)
41 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
43 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
47 typedef struct LLVMOpaquePass *LLVMPassRef;
48 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
50 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
51 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
52 #if LLVM_VERSION_LT(11, 0)
53 DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
54 LLVMPassManagerBuilderRef)
57 extern "C" void LLVMInitializePasses() {
58 PassRegistry &Registry = *PassRegistry::getPassRegistry();
59 initializeCore(Registry);
60 initializeCodeGen(Registry);
61 initializeScalarOpts(Registry);
62 initializeVectorization(Registry);
63 initializeIPO(Registry);
64 initializeAnalysis(Registry);
65 initializeTransformUtils(Registry);
66 initializeInstCombine(Registry);
67 initializeInstrumentation(Registry);
68 initializeTarget(Registry);
71 extern "C" void LLVMTimeTraceProfilerInitialize() {
72 #if LLVM_VERSION_GE(10, 0)
73 timeTraceProfilerInitialize(
74 /* TimeTraceGranularity */ 0,
75 /* ProcName */ "rustc");
76 #elif LLVM_VERSION_GE(9, 0)
77 timeTraceProfilerInitialize();
81 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
82 #if LLVM_VERSION_GE(9, 0)
83 StringRef FN(FileName);
85 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
87 timeTraceProfilerWrite(OS);
88 timeTraceProfilerCleanup();
92 enum class LLVMRustPassKind {
98 static LLVMRustPassKind toRust(PassKind Kind) {
101 return LLVMRustPassKind::Function;
103 return LLVMRustPassKind::Module;
105 return LLVMRustPassKind::Other;
109 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
110 StringRef SR(PassName);
111 PassRegistry *PR = PassRegistry::getPassRegistry();
113 const PassInfo *PI = PR->getPassInfo(SR);
115 return wrap(PI->createPass());
120 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
121 const bool CompileKernel = false;
122 const bool UseAfterScope = true;
124 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
127 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
128 const bool CompileKernel = false;
130 #if LLVM_VERSION_GE(9, 0)
131 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
133 return wrap(createAddressSanitizerModulePass(CompileKernel, Recover));
137 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
138 #if LLVM_VERSION_GE(9, 0)
139 const bool CompileKernel = false;
141 return wrap(createMemorySanitizerLegacyPassPass(
142 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
144 return wrap(createMemorySanitizerLegacyPassPass(TrackOrigins, Recover));
148 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
149 return wrap(createThreadSanitizerLegacyPassPass());
152 extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
154 Pass *Pass = unwrap(RustPass);
155 return toRust(Pass->getPassKind());
158 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
160 Pass *Pass = unwrap(RustPass);
161 PassManagerBase *PMB = unwrap(PMR);
166 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
167 LLVMPassManagerBuilderRef PMBR,
168 LLVMPassManagerRef PMR
170 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
174 void LLVMRustAddLastExtensionPasses(
175 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
176 auto AddExtensionPasses = [Passes, NumPasses](
177 const PassManagerBuilder &Builder, PassManagerBase &PM) {
178 for (size_t I = 0; I < NumPasses; I++) {
179 PM.add(unwrap(Passes[I]));
182 // Add the passes to both of the pre-finalization extension points,
183 // so they are run for optimized and non-optimized builds.
184 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
186 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
190 #ifdef LLVM_COMPONENT_X86
191 #define SUBTARGET_X86 SUBTARGET(X86)
193 #define SUBTARGET_X86
196 #ifdef LLVM_COMPONENT_ARM
197 #define SUBTARGET_ARM SUBTARGET(ARM)
199 #define SUBTARGET_ARM
202 #ifdef LLVM_COMPONENT_AARCH64
203 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
205 #define SUBTARGET_AARCH64
208 #ifdef LLVM_COMPONENT_AVR
209 #define SUBTARGET_AVR SUBTARGET(AVR)
211 #define SUBTARGET_AVR
214 #ifdef LLVM_COMPONENT_MIPS
215 #define SUBTARGET_MIPS SUBTARGET(Mips)
217 #define SUBTARGET_MIPS
220 #ifdef LLVM_COMPONENT_POWERPC
221 #define SUBTARGET_PPC SUBTARGET(PPC)
223 #define SUBTARGET_PPC
226 #ifdef LLVM_COMPONENT_SYSTEMZ
227 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
229 #define SUBTARGET_SYSTEMZ
232 #ifdef LLVM_COMPONENT_MSP430
233 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
235 #define SUBTARGET_MSP430
238 #ifdef LLVM_COMPONENT_RISCV
239 #define SUBTARGET_RISCV SUBTARGET(RISCV)
241 #define SUBTARGET_RISCV
244 #ifdef LLVM_COMPONENT_SPARC
245 #define SUBTARGET_SPARC SUBTARGET(Sparc)
247 #define SUBTARGET_SPARC
250 #ifdef LLVM_COMPONENT_HEXAGON
251 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
253 #define SUBTARGET_HEXAGON
256 #define GEN_SUBTARGETS \
269 #define SUBTARGET(x) \
271 extern const SubtargetFeatureKV x##FeatureKV[]; \
272 extern const SubtargetFeatureKV x##SubTypeKV[]; \
278 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
279 const char *Feature) {
280 TargetMachine *Target = unwrap(TM);
281 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
282 return MCInfo->checkFeatures(std::string("+") + Feature);
285 enum class LLVMRustCodeModel {
294 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
296 case LLVMRustCodeModel::Tiny:
297 return CodeModel::Tiny;
298 case LLVMRustCodeModel::Small:
299 return CodeModel::Small;
300 case LLVMRustCodeModel::Kernel:
301 return CodeModel::Kernel;
302 case LLVMRustCodeModel::Medium:
303 return CodeModel::Medium;
304 case LLVMRustCodeModel::Large:
305 return CodeModel::Large;
306 case LLVMRustCodeModel::None:
309 report_fatal_error("Bad CodeModel.");
313 enum class LLVMRustCodeGenOptLevel {
321 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
323 case LLVMRustCodeGenOptLevel::None:
324 return CodeGenOpt::None;
325 case LLVMRustCodeGenOptLevel::Less:
326 return CodeGenOpt::Less;
327 case LLVMRustCodeGenOptLevel::Default:
328 return CodeGenOpt::Default;
329 case LLVMRustCodeGenOptLevel::Aggressive:
330 return CodeGenOpt::Aggressive;
332 report_fatal_error("Bad CodeGenOptLevel.");
336 enum class LLVMRustPassBuilderOptLevel {
345 static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
347 case LLVMRustPassBuilderOptLevel::O0:
348 return PassBuilder::OptimizationLevel::O0;
349 case LLVMRustPassBuilderOptLevel::O1:
350 return PassBuilder::OptimizationLevel::O1;
351 case LLVMRustPassBuilderOptLevel::O2:
352 return PassBuilder::OptimizationLevel::O2;
353 case LLVMRustPassBuilderOptLevel::O3:
354 return PassBuilder::OptimizationLevel::O3;
355 case LLVMRustPassBuilderOptLevel::Os:
356 return PassBuilder::OptimizationLevel::Os;
357 case LLVMRustPassBuilderOptLevel::Oz:
358 return PassBuilder::OptimizationLevel::Oz;
360 report_fatal_error("Bad PassBuilderOptLevel.");
364 enum class LLVMRustRelocModel {
373 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
375 case LLVMRustRelocModel::Static:
376 return Reloc::Static;
377 case LLVMRustRelocModel::PIC:
379 case LLVMRustRelocModel::DynamicNoPic:
380 return Reloc::DynamicNoPIC;
381 case LLVMRustRelocModel::ROPI:
383 case LLVMRustRelocModel::RWPI:
385 case LLVMRustRelocModel::ROPIRWPI:
386 return Reloc::ROPI_RWPI;
388 report_fatal_error("Bad RelocModel.");
392 /// getLongestEntryLength - Return the length of the longest entry in the table.
393 template<typename KV>
394 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
396 for (auto &I : Table)
397 MaxLen = std::max(MaxLen, std::strlen(I.Key));
401 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
402 const TargetMachine *Target = unwrap(TM);
403 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
404 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
405 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
406 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
407 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
409 printf("Available CPUs for this target:\n");
410 if (HostArch == TargetArch) {
411 const StringRef HostCPU = sys::getHostCPUName();
412 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
413 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
415 for (auto &CPU : CPUTable)
416 printf(" %-*s\n", MaxCPULen, CPU.Key);
420 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
421 const TargetMachine *Target = unwrap(TM);
422 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
423 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
424 unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
426 printf("Available features for this target:\n");
427 for (auto &Feature : FeatTable)
428 printf(" %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
429 printf("\nRust-specific features:\n");
430 printf(" %-*s - %s.\n",
433 "Enables libraries with C Run-time Libraries(CRT) to be statically linked"
437 printf("Use +feature to enable a feature, or -feature to disable it.\n"
438 "For example, rustc -C -target-cpu=mycpu -C "
439 "target-feature=+feature1,-feature2\n\n");
444 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
445 printf("Target CPU help is not supported by this LLVM version.\n\n");
448 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
449 printf("Target features help is not supported by this LLVM version.\n\n");
453 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
454 StringRef Name = sys::getHostCPUName();
459 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
460 const char *TripleStr, const char *CPU, const char *Feature,
461 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
462 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
463 bool FunctionSections,
465 bool TrapUnreachable,
468 bool EmitStackSizeSection,
469 bool RelaxELFRelocations,
472 auto OptLevel = fromRust(RustOptLevel);
473 auto RM = fromRust(RustReloc);
474 auto CM = fromRust(RustCM);
477 Triple Trip(Triple::normalize(TripleStr));
478 const llvm::Target *TheTarget =
479 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
480 if (TheTarget == nullptr) {
481 LLVMRustSetLastError(Error.c_str());
485 TargetOptions Options;
487 Options.FloatABIType = FloatABI::Default;
489 Options.FloatABIType = FloatABI::Soft;
491 Options.DataSections = DataSections;
492 Options.FunctionSections = FunctionSections;
493 Options.MCOptions.AsmVerbose = AsmComments;
494 Options.MCOptions.PreserveAsmComments = AsmComments;
495 Options.MCOptions.ABIName = ABIStr;
496 Options.RelaxELFRelocations = RelaxELFRelocations;
497 Options.UseInitArray = UseInitArray;
499 if (TrapUnreachable) {
500 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
501 // This limits the extent of possible undefined behavior in some cases, as
502 // it prevents control flow from "falling through" into whatever code
503 // happens to be laid out next in memory.
504 Options.TrapUnreachable = true;
508 Options.ThreadModel = ThreadModel::Single;
511 Options.EmitStackSizeSection = EmitStackSizeSection;
513 TargetMachine *TM = TheTarget->createTargetMachine(
514 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
518 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
522 extern "C" void LLVMRustConfigurePassManagerBuilder(
523 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
524 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
525 const char* PGOGenPath, const char* PGOUsePath) {
526 unwrap(PMBR)->MergeFunctions = MergeFunctions;
527 unwrap(PMBR)->SLPVectorize = SLPVectorize;
528 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
529 unwrap(PMBR)->LoopVectorize = LoopVectorize;
530 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
534 unwrap(PMBR)->EnablePGOInstrGen = true;
535 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
539 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
543 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
544 // field of a PassManagerBuilder, we expose our own method of doing so.
545 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
547 bool DisableSimplifyLibCalls) {
548 Triple TargetTriple(unwrap(M)->getTargetTriple());
549 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
550 if (DisableSimplifyLibCalls)
551 TLI->disableAllFunctions();
552 unwrap(PMBR)->LibraryInfo = TLI;
555 // Unfortunately, the LLVM C API doesn't provide a way to create the
556 // TargetLibraryInfo pass, so we use this method to do so.
557 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
558 bool DisableSimplifyLibCalls) {
559 Triple TargetTriple(unwrap(M)->getTargetTriple());
560 TargetLibraryInfoImpl TLII(TargetTriple);
561 if (DisableSimplifyLibCalls)
562 TLII.disableAllFunctions();
563 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
566 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
567 // all the functions in a module, so we do that manually here. You'll find
568 // similar code in clang's BackendUtil.cpp file.
569 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
571 llvm::legacy::FunctionPassManager *P =
572 unwrap<llvm::legacy::FunctionPassManager>(PMR);
573 P->doInitialization();
575 // Upgrade all calls to old intrinsics first.
576 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
577 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
579 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
581 if (!I->isDeclaration())
587 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
588 // Initializing the command-line options more than once is not allowed. So,
589 // check if they've already been initialized. (This could happen if we're
590 // being called from rustpkg, for example). If the arguments change, then
591 // that's just kinda unfortunate.
592 static bool Initialized = false;
596 cl::ParseCommandLineOptions(Argc, Argv);
599 enum class LLVMRustFileType {
605 #if LLVM_VERSION_GE(10, 0)
606 static CodeGenFileType fromRust(LLVMRustFileType Type) {
608 case LLVMRustFileType::AssemblyFile:
609 return CGFT_AssemblyFile;
610 case LLVMRustFileType::ObjectFile:
611 return CGFT_ObjectFile;
613 report_fatal_error("Bad FileType.");
617 static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
619 case LLVMRustFileType::AssemblyFile:
620 return TargetMachine::CGFT_AssemblyFile;
621 case LLVMRustFileType::ObjectFile:
622 return TargetMachine::CGFT_ObjectFile;
624 report_fatal_error("Bad FileType.");
629 extern "C" LLVMRustResult
630 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
631 LLVMModuleRef M, const char *Path,
632 LLVMRustFileType RustFileType) {
633 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
634 auto FileType = fromRust(RustFileType);
636 std::string ErrorInfo;
638 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
640 ErrorInfo = EC.message();
641 if (ErrorInfo != "") {
642 LLVMRustSetLastError(ErrorInfo.c_str());
643 return LLVMRustResult::Failure;
646 buffer_ostream BOS(OS);
647 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
650 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
651 // stream (OS), so the only real safe place to delete this is here? Don't we
652 // wish this was written in Rust?
653 LLVMDisposePassManager(PMR);
654 return LLVMRustResult::Success;
657 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
658 const char*, // pass name
659 const char*); // IR name
660 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
662 #if LLVM_VERSION_GE(9, 0)
664 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
665 if (any_isa<const Module *>(WrappedIr))
666 return any_cast<const Module *>(WrappedIr)->getName().str();
667 if (any_isa<const Function *>(WrappedIr))
668 return any_cast<const Function *>(WrappedIr)->getName().str();
669 if (any_isa<const Loop *>(WrappedIr))
670 return any_cast<const Loop *>(WrappedIr)->getName().str();
671 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
672 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
677 void LLVMSelfProfileInitializeCallbacks(
678 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
679 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
680 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
681 PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback](
682 StringRef Pass, llvm::Any Ir) {
683 std::string PassName = Pass.str();
684 std::string IrName = LLVMRustwrappedIrGetName(Ir);
685 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
689 PIC.registerAfterPassCallback(
690 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
691 AfterPassCallback(LlvmSelfProfiler);
694 PIC.registerAfterPassInvalidatedCallback(
695 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) {
696 AfterPassCallback(LlvmSelfProfiler);
699 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
700 StringRef Pass, llvm::Any Ir) {
701 std::string PassName = Pass.str();
702 std::string IrName = LLVMRustwrappedIrGetName(Ir);
703 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
706 PIC.registerAfterAnalysisCallback(
707 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
708 AfterPassCallback(LlvmSelfProfiler);
713 enum class LLVMRustOptStage {
721 struct LLVMRustSanitizerOptions {
722 bool SanitizeAddress;
723 bool SanitizeAddressRecover;
725 bool SanitizeMemoryRecover;
726 int SanitizeMemoryTrackOrigins;
731 LLVMRustOptimizeWithNewPassManager(
732 LLVMModuleRef ModuleRef,
733 LLVMTargetMachineRef TMRef,
734 LLVMRustPassBuilderOptLevel OptLevelRust,
735 LLVMRustOptStage OptStage,
736 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
737 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
738 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
739 LLVMRustSanitizerOptions *SanitizerOptions,
740 const char *PGOGenPath, const char *PGOUsePath,
741 void* LlvmSelfProfiler,
742 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
743 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
744 #if LLVM_VERSION_GE(9, 0)
745 Module *TheModule = unwrap(ModuleRef);
746 TargetMachine *TM = unwrap(TMRef);
747 PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust);
749 // FIXME: MergeFunctions is not supported by NewPM yet.
750 (void) MergeFunctions;
752 PipelineTuningOptions PTO;
753 PTO.LoopUnrolling = UnrollLoops;
754 PTO.LoopInterleaving = UnrollLoops;
755 PTO.LoopVectorization = LoopVectorize;
756 PTO.SLPVectorization = SLPVectorize;
758 PassInstrumentationCallbacks PIC;
759 StandardInstrumentations SI;
760 SI.registerCallbacks(PIC);
762 if (LlvmSelfProfiler){
763 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
766 Optional<PGOOptions> PGOOpt;
769 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr);
770 } else if (PGOUsePath) {
772 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse);
775 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
777 // FIXME: We may want to expose this as an option.
778 bool DebugPassManager = false;
779 LoopAnalysisManager LAM(DebugPassManager);
780 FunctionAnalysisManager FAM(DebugPassManager);
781 CGSCCAnalysisManager CGAM(DebugPassManager);
782 ModuleAnalysisManager MAM(DebugPassManager);
784 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
786 Triple TargetTriple(TheModule->getTargetTriple());
787 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
788 if (DisableSimplifyLibCalls)
789 TLII->disableAllFunctions();
790 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
792 PB.registerModuleAnalyses(MAM);
793 PB.registerCGSCCAnalyses(CGAM);
794 PB.registerFunctionAnalyses(FAM);
795 PB.registerLoopAnalyses(LAM);
796 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
798 // We manually collect pipeline callbacks so we can apply them at O0, where the
799 // PassBuilder does not create a pipeline.
800 std::vector<std::function<void(ModulePassManager &)>> PipelineStartEPCallbacks;
801 #if LLVM_VERSION_GE(11, 0)
802 std::vector<std::function<void(ModulePassManager &, PassBuilder::OptimizationLevel)>>
803 OptimizerLastEPCallbacks;
805 std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>>
806 OptimizerLastEPCallbacks;
810 PipelineStartEPCallbacks.push_back([VerifyIR](ModulePassManager &MPM) {
811 MPM.addPass(VerifierPass());
815 if (SanitizerOptions) {
816 if (SanitizerOptions->SanitizeMemory) {
817 MemorySanitizerOptions Options(
818 SanitizerOptions->SanitizeMemoryTrackOrigins,
819 SanitizerOptions->SanitizeMemoryRecover,
820 /*CompileKernel=*/false);
821 #if LLVM_VERSION_GE(11, 0)
822 OptimizerLastEPCallbacks.push_back(
823 [Options](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
824 MPM.addPass(MemorySanitizerPass(Options));
825 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
829 #if LLVM_VERSION_GE(10, 0)
830 PipelineStartEPCallbacks.push_back([Options](ModulePassManager &MPM) {
831 MPM.addPass(MemorySanitizerPass(Options));
834 OptimizerLastEPCallbacks.push_back(
835 [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
836 FPM.addPass(MemorySanitizerPass(Options));
842 if (SanitizerOptions->SanitizeThread) {
843 #if LLVM_VERSION_GE(11, 0)
844 OptimizerLastEPCallbacks.push_back(
845 [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
846 MPM.addPass(ThreadSanitizerPass());
847 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
851 #if LLVM_VERSION_GE(10, 0)
852 PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM) {
853 MPM.addPass(ThreadSanitizerPass());
856 OptimizerLastEPCallbacks.push_back(
857 [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
858 FPM.addPass(ThreadSanitizerPass());
864 if (SanitizerOptions->SanitizeAddress) {
865 #if LLVM_VERSION_GE(11, 0)
866 OptimizerLastEPCallbacks.push_back(
867 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
868 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
869 MPM.addPass(ModuleAddressSanitizerPass(
870 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
871 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
872 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
873 /*UseAfterScope=*/true)));
877 PipelineStartEPCallbacks.push_back([&](ModulePassManager &MPM) {
878 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
880 OptimizerLastEPCallbacks.push_back(
881 [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
882 FPM.addPass(AddressSanitizerPass(
883 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
884 /*UseAfterScope=*/true));
887 PipelineStartEPCallbacks.push_back(
888 [SanitizerOptions](ModulePassManager &MPM) {
889 MPM.addPass(ModuleAddressSanitizerPass(
890 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
897 ModulePassManager MPM(DebugPassManager);
898 if (!NoPrepopulatePasses) {
899 if (OptLevel == PassBuilder::OptimizationLevel::O0) {
900 for (const auto &C : PipelineStartEPCallbacks)
903 #if LLVM_VERSION_GE(11, 0)
904 for (const auto &C : OptimizerLastEPCallbacks)
907 if (!OptimizerLastEPCallbacks.empty()) {
908 FunctionPassManager FPM(DebugPassManager);
909 for (const auto &C : OptimizerLastEPCallbacks)
911 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
915 MPM.addPass(AlwaysInlinerPass(EmitLifetimeMarkers));
917 #if LLVM_VERSION_GE(10, 0)
919 PB.addPGOInstrPassesForO0(
920 MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr,
921 /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
925 for (const auto &C : PipelineStartEPCallbacks)
926 PB.registerPipelineStartEPCallback(C);
927 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
928 for (const auto &C : OptimizerLastEPCallbacks)
929 PB.registerOptimizerLastEPCallback(C);
933 case LLVMRustOptStage::PreLinkNoLTO:
934 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
936 case LLVMRustOptStage::PreLinkThinLTO:
937 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
938 #if LLVM_VERSION_GE(11, 0)
939 for (const auto &C : OptimizerLastEPCallbacks)
942 if (!OptimizerLastEPCallbacks.empty()) {
943 FunctionPassManager FPM(DebugPassManager);
944 for (const auto &C : OptimizerLastEPCallbacks)
946 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
950 case LLVMRustOptStage::PreLinkFatLTO:
951 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
953 case LLVMRustOptStage::ThinLTO:
954 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
955 // It only seems to be needed for C++ specific optimizations.
956 MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
958 case LLVMRustOptStage::FatLTO:
959 MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
965 if (UseThinLTOBuffers) {
966 MPM.addPass(CanonicalizeAliasesPass());
967 MPM.addPass(NameAnonGlobalPass());
970 // Upgrade all calls to old intrinsics first.
971 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
972 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
974 MPM.run(*TheModule, MAM);
976 // The new pass manager has been available for a long time,
977 // but we don't bother supporting it on old LLVM versions.
978 report_fatal_error("New pass manager only supported since LLVM 9");
982 // Callback to demangle function name
984 // * name to be demangled
987 // * output buffer len
988 // Returns len of demangled string, or 0 if demangle failed.
989 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
994 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
996 std::vector<char> Buf;
999 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1001 // Return empty string if demangle failed
1002 // or if name does not need to be demangled
1003 StringRef CallDemangle(StringRef name) {
1008 if (Buf.size() < name.size() * 2) {
1009 // Semangled name usually shorter than mangled,
1010 // but allocate twice as much memory just in case
1011 Buf.resize(name.size() * 2);
1014 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1020 auto Demangled = StringRef(Buf.data(), R);
1021 if (Demangled == name) {
1022 // Do not print anything if demangled name is equal to mangled.
1029 void emitFunctionAnnot(const Function *F,
1030 formatted_raw_ostream &OS) override {
1031 StringRef Demangled = CallDemangle(F->getName());
1032 if (Demangled.empty()) {
1036 OS << "; " << Demangled << "\n";
1039 void emitInstructionAnnot(const Instruction *I,
1040 formatted_raw_ostream &OS) override {
1043 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1045 Value = CI->getCalledOperand();
1046 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1048 Value = II->getCalledOperand();
1050 // Could demangle more operations, e. g.
1051 // `store %place, @function`.
1055 if (!Value->hasName()) {
1059 StringRef Demangled = CallDemangle(Value->getName());
1060 if (Demangled.empty()) {
1064 OS << "; " << Name << " " << Demangled << "\n";
1070 extern "C" LLVMRustResult
1071 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1072 std::string ErrorInfo;
1074 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
1076 ErrorInfo = EC.message();
1077 if (ErrorInfo != "") {
1078 LLVMRustSetLastError(ErrorInfo.c_str());
1079 return LLVMRustResult::Failure;
1082 RustAssemblyAnnotationWriter AAW(Demangle);
1083 formatted_raw_ostream FOS(OS);
1084 unwrap(M)->print(FOS, &AAW);
1086 return LLVMRustResult::Success;
1089 extern "C" void LLVMRustPrintPasses() {
1090 LLVMInitializePasses();
1091 struct MyListener : PassRegistrationListener {
1092 void passEnumerate(const PassInfo *Info) {
1093 StringRef PassArg = Info->getPassArgument();
1094 StringRef PassName = Info->getPassName();
1095 if (!PassArg.empty()) {
1096 // These unsigned->signed casts could theoretically overflow, but
1097 // realistically never will (and even if, the result is implementation
1098 // defined rather plain UB).
1099 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1100 (int)PassName.size(), PassName.data());
1105 PassRegistry *PR = PassRegistry::getPassRegistry();
1106 PR->enumerateWith(&Listener);
1109 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1110 bool AddLifetimes) {
1111 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1114 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1116 llvm::legacy::PassManager passes;
1118 auto PreserveFunctions = [=](const GlobalValue &GV) {
1119 for (size_t I = 0; I < Len; I++) {
1120 if (GV.getName() == Symbols[I]) {
1127 passes.add(llvm::createInternalizePass(PreserveFunctions));
1129 passes.run(*unwrap(M));
1132 extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
1133 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
1135 GV->setDoesNotThrow();
1136 Function *F = dyn_cast<Function>(GV);
1140 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
1141 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
1142 if (isa<InvokeInst>(I)) {
1143 InvokeInst *CI = cast<InvokeInst>(I);
1144 CI->setDoesNotThrow();
1152 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1153 LLVMTargetMachineRef TMR) {
1154 TargetMachine *Target = unwrap(TMR);
1155 unwrap(Module)->setDataLayout(Target->createDataLayout());
1158 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1159 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1162 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1163 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1166 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1167 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1168 // LLVM, and otherwise it's just blanket rejected from other compilers.
1170 // Most of this implementation is straight copied from LLVM. At the time of
1171 // this writing it wasn't *quite* suitable to reuse more code from upstream
1172 // for our purposes, but we should strive to upstream this support once it's
1173 // ready to go! I figure we may want a bit of testing locally first before
1174 // sending this upstream to LLVM. I hear though they're quite eager to receive
1175 // feedback like this!
1177 // If you're reading this code and wondering "what in the world" or you're
1178 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1179 // then fear not! (ok maybe fear a little). All code here is mostly based
1180 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1182 // You'll find that the general layout here roughly corresponds to the `run`
1183 // method in that file as well as `ProcessThinLTOModule`. Functions are
1184 // specifically commented below as well, but if you're updating this code
1185 // or otherwise trying to understand it, the LLVM source will be useful in
1186 // interpreting the mysteries within.
1188 // Otherwise I'll apologize in advance, it probably requires a relatively
1189 // significant investment on your part to "truly understand" what's going on
1190 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1191 // and various online resources about ThinLTO to make heads or tails of all
1194 // This is a shared data structure which *must* be threadsafe to share
1195 // read-only amongst threads. This also corresponds basically to the arguments
1196 // of the `ProcessThinLTOModule` function in the LLVM source.
1197 struct LLVMRustThinLTOData {
1198 // The combined index that is the global analysis over all modules we're
1199 // performing ThinLTO for. This is mostly managed by LLVM.
1200 ModuleSummaryIndex Index;
1202 // All modules we may look at, stored as in-memory serialized versions. This
1203 // is later used when inlining to ensure we can extract any module to inline
1205 StringMap<MemoryBufferRef> ModuleMap;
1207 // A set that we manage of everything we *don't* want internalized. Note that
1208 // this includes all transitive references right now as well, but it may not
1210 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1212 // Not 100% sure what these are, but they impact what's internalized and
1213 // what's inlined across modules, I believe.
1214 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1215 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1216 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1218 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1221 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1222 struct LLVMRustThinLTOModule {
1223 const char *identifier;
1228 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1230 static const GlobalValueSummary *
1231 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1232 auto StrongDefForLinker = llvm::find_if(
1233 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1234 auto Linkage = Summary->linkage();
1235 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1236 !GlobalValue::isWeakForLinker(Linkage);
1238 if (StrongDefForLinker != GVSummaryList.end())
1239 return StrongDefForLinker->get();
1241 auto FirstDefForLinker = llvm::find_if(
1242 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1243 auto Linkage = Summary->linkage();
1244 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1246 if (FirstDefForLinker == GVSummaryList.end())
1248 return FirstDefForLinker->get();
1251 // The main entry point for creating the global ThinLTO analysis. The structure
1252 // here is basically the same as before threads are spawned in the `run`
1253 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1254 extern "C" LLVMRustThinLTOData*
1255 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1257 const char **preserved_symbols,
1259 #if LLVM_VERSION_GE(10, 0)
1260 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1262 auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
1265 // Load each module's summary and merge it into one combined index
1266 for (int i = 0; i < num_modules; i++) {
1267 auto module = &modules[i];
1268 StringRef buffer(module->data, module->len);
1269 MemoryBufferRef mem_buffer(buffer, module->identifier);
1271 Ret->ModuleMap[module->identifier] = mem_buffer;
1273 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1274 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1279 // Collect for each module the list of function it defines (GUID -> Summary)
1280 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1282 // Convert the preserved symbols set from string to GUID, this is then needed
1283 // for internalization.
1284 for (int i = 0; i < num_symbols; i++) {
1285 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1286 Ret->GUIDPreservedSymbols.insert(GUID);
1289 // Collect the import/export lists for all modules from the call-graph in the
1292 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1293 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1294 return PrevailingType::Unknown;
1296 // We don't have a complete picture in our use of ThinLTO, just our immediate
1297 // crate, so we need `ImportEnabled = false` to limit internalization.
1298 // Otherwise, we sometimes lose `static` values -- see #60184.
1299 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1300 deadIsPrevailing, /* ImportEnabled = */ false);
1301 ComputeCrossModuleImport(
1303 Ret->ModuleToDefinedGVSummaries,
1308 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1309 // impacts the caching.
1311 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1312 // being lifted from `lib/LTO/LTO.cpp` as well
1313 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1314 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1315 for (auto &I : Ret->Index) {
1316 if (I.second.SummaryList.size() > 1)
1317 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1319 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1320 const auto &Prevailing = PrevailingCopy.find(GUID);
1321 if (Prevailing == PrevailingCopy.end())
1323 return Prevailing->second == S;
1325 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1326 GlobalValue::GUID GUID,
1327 GlobalValue::LinkageTypes NewLinkage) {
1328 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1330 #if LLVM_VERSION_GE(9, 0)
1331 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
1332 Ret->GUIDPreservedSymbols);
1334 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage);
1337 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1338 // callback below. This callback below will dictate the linkage for all
1339 // summaries in the index, and we basically just only want to ensure that dead
1340 // symbols are internalized. Otherwise everything that's already external
1341 // linkage will stay as external, and internal will stay as internal.
1342 std::set<GlobalValue::GUID> ExportedGUIDs;
1343 for (auto &List : Ret->Index) {
1344 for (auto &GVS: List.second.SummaryList) {
1345 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1347 auto GUID = GVS->getOriginalName();
1348 if (GVS->flags().Live)
1349 ExportedGUIDs.insert(GUID);
1352 #if LLVM_VERSION_GE(10, 0)
1353 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1354 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1355 return (ExportList != Ret->ExportLists.end() &&
1356 ExportList->second.count(VI)) ||
1357 ExportedGUIDs.count(VI.getGUID());
1359 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1361 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
1362 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1363 return (ExportList != Ret->ExportLists.end() &&
1364 ExportList->second.count(GUID)) ||
1365 ExportedGUIDs.count(GUID);
1367 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
1370 return Ret.release();
1374 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1378 // Below are the various passes that happen *per module* when doing ThinLTO.
1380 // In other words, these are the functions that are all run concurrently
1381 // with one another, one per module. The passes here correspond to the analysis
1382 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1383 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1384 // so rustc can save off the intermediate bytecode between each step.
1386 #if LLVM_VERSION_GE(11, 0)
1388 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1389 // When linking an ELF shared object, dso_local should be dropped. We
1390 // conservatively do this for -fpic.
1391 bool ClearDSOLocalOnDeclarations =
1392 TM.getTargetTriple().isOSBinFormatELF() &&
1393 TM.getRelocationModel() != Reloc::Static &&
1394 Mod.getPIELevel() == PIELevel::Default;
1395 return ClearDSOLocalOnDeclarations;
1400 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1401 LLVMTargetMachineRef TM) {
1402 Module &Mod = *unwrap(M);
1403 TargetMachine &Target = *unwrap(TM);
1405 #if LLVM_VERSION_GE(11, 0)
1406 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1407 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1409 bool error = renameModuleForThinLTO(Mod, Data->Index);
1413 LLVMRustSetLastError("renameModuleForThinLTO failed");
1420 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1421 Module &Mod = *unwrap(M);
1422 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1423 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1428 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1429 Module &Mod = *unwrap(M);
1430 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1431 thinLTOInternalizeModule(Mod, DefinedGlobals);
1436 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1437 LLVMTargetMachineRef TM) {
1438 Module &Mod = *unwrap(M);
1439 TargetMachine &Target = *unwrap(TM);
1441 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1442 auto Loader = [&](StringRef Identifier) {
1443 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1444 auto &Context = Mod.getContext();
1445 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1450 // The rest of this closure is a workaround for
1451 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1452 // we accidentally import wasm custom sections into different modules,
1453 // duplicating them by in the final output artifact.
1455 // The issue is worked around here by manually removing the
1456 // `wasm.custom_sections` named metadata node from any imported module. This
1457 // we know isn't used by any optimization pass so there's no need for it to
1460 // Note that the metadata is currently lazily loaded, so we materialize it
1461 // here before looking up if there's metadata inside. The `FunctionImporter`
1462 // will immediately materialize metadata anyway after an import, so this
1463 // shouldn't be a perf hit.
1464 if (Error Err = (*MOrErr)->materializeMetadata()) {
1465 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1469 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1470 if (WasmCustomSections)
1471 WasmCustomSections->eraseFromParent();
1475 #if LLVM_VERSION_GE(11, 0)
1476 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1477 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1479 FunctionImporter Importer(Data->Index, Loader);
1481 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1483 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1489 extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1490 const char*, // importing module name
1491 const char*); // imported module name
1493 // Calls `module_name_callback` for each module import done by ThinLTO.
1494 // The callback is provided with regular null-terminated C strings.
1496 LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data,
1497 LLVMRustModuleNameCallback module_name_callback,
1498 void* callback_payload) {
1499 for (const auto& importing_module : data->ImportLists) {
1500 const std::string importing_module_id = importing_module.getKey().str();
1501 const auto& imports = importing_module.getValue();
1502 for (const auto& imported_module : imports) {
1503 const std::string imported_module_id = imported_module.getKey().str();
1504 module_name_callback(callback_payload,
1505 importing_module_id.c_str(),
1506 imported_module_id.c_str());
1511 // This struct and various functions are sort of a hack right now, but the
1512 // problem is that we've got in-memory LLVM modules after we generate and
1513 // optimize all codegen-units for one compilation in rustc. To be compatible
1514 // with the LTO support above we need to serialize the modules plus their
1515 // ThinLTO summary into memory.
1517 // This structure is basically an owned version of a serialize module, with
1518 // a ThinLTO summary attached.
1519 struct LLVMRustThinLTOBuffer {
1523 extern "C" LLVMRustThinLTOBuffer*
1524 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1525 #if LLVM_VERSION_GE(10, 0)
1526 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1528 auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
1531 raw_string_ostream OS(Ret->data);
1533 legacy::PassManager PM;
1534 PM.add(createWriteThinLTOBitcodePass(OS));
1538 return Ret.release();
1542 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1546 extern "C" const void*
1547 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1548 return Buffer->data.data();
1552 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1553 return Buffer->data.length();
1556 // This is what we used to parse upstream bitcode for actual ThinLTO
1557 // processing. We'll call this once per module optimized through ThinLTO, and
1558 // it'll be called concurrently on many threads.
1559 extern "C" LLVMModuleRef
1560 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1563 const char *identifier) {
1564 StringRef Data(data, len);
1565 MemoryBufferRef Buffer(Data, identifier);
1566 unwrap(Context)->enableDebugTypeODRUniquing();
1567 Expected<std::unique_ptr<Module>> SrcOrError =
1568 parseBitcodeFile(Buffer, *unwrap(Context));
1570 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1573 return wrap(std::move(*SrcOrError).release());
1576 // Find the bitcode section in the object file data and return it as a slice.
1577 // Fail if the bitcode section is present but empty.
1579 // On success, the return value is the pointer to the start of the slice and
1580 // `out_len` is filled with the (non-zero) length. On failure, the return value
1581 // is `nullptr` and `out_len` is set to zero.
1582 extern "C" const char*
1583 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1588 StringRef Data(data, len);
1589 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1591 Expected<MemoryBufferRef> BitcodeOrError =
1592 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1593 if (!BitcodeOrError) {
1594 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1598 *out_len = BitcodeOrError->getBufferSize();
1599 return BitcodeOrError->getBufferStart();
1602 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1603 // the comment in `back/lto.rs` for why this exists.
1605 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1607 DICompileUnit **B) {
1608 Module *M = unwrap(Mod);
1609 DICompileUnit **Cur = A;
1610 DICompileUnit **Next = B;
1611 for (DICompileUnit *CU : M->debug_compile_units()) {
1620 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1621 // the comment in `back/lto.rs` for why this exists.
1623 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1624 Module *M = unwrap(Mod);
1626 // If the original source module didn't have a `DICompileUnit` then try to
1627 // merge all the existing compile units. If there aren't actually any though
1628 // then there's not much for us to do so return.
1629 if (Unit == nullptr) {
1630 for (DICompileUnit *CU : M->debug_compile_units()) {
1634 if (Unit == nullptr)
1638 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1639 // process it recursively. Note that we used to specifically iterate over
1640 // instructions to ensure we feed everything into it, but `processModule`
1641 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1642 DebugInfoFinder Finder;
1643 Finder.processModule(*M);
1645 // After we've found all our debuginfo, rewrite all subprograms to point to
1646 // the same `DICompileUnit`.
1647 for (auto &F : Finder.subprograms()) {
1648 F->replaceUnit(Unit);
1651 // Erase any other references to other `DICompileUnit` instances, the verifier
1652 // will later ensure that we don't actually have any other stale references to
1654 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1655 MD->clearOperands();
1656 MD->addOperand(Unit);