6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/StandardInstrumentations.h"
21 #include "llvm/Support/CBindingWrapping.h"
22 #include "llvm/Support/FileSystem.h"
23 #include "llvm/Support/Host.h"
24 #include "llvm/Target/TargetMachine.h"
25 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
26 #include "llvm/Transforms/IPO/AlwaysInliner.h"
27 #include "llvm/Transforms/IPO/FunctionImport.h"
28 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
29 #include "llvm/LTO/LTO.h"
30 #include "llvm-c/Transforms/PassManagerBuilder.h"
32 #include "llvm/Transforms/Instrumentation.h"
33 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
34 #include "llvm/Support/TimeProfiler.h"
35 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
36 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
37 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
38 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
39 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
43 typedef struct LLVMOpaquePass *LLVMPassRef;
44 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
46 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
47 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
48 #if LLVM_VERSION_LT(11, 0)
49 DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
50 LLVMPassManagerBuilderRef)
53 extern "C" void LLVMInitializePasses() {
54 PassRegistry &Registry = *PassRegistry::getPassRegistry();
55 initializeCore(Registry);
56 initializeCodeGen(Registry);
57 initializeScalarOpts(Registry);
58 initializeVectorization(Registry);
59 initializeIPO(Registry);
60 initializeAnalysis(Registry);
61 initializeTransformUtils(Registry);
62 initializeInstCombine(Registry);
63 initializeInstrumentation(Registry);
64 initializeTarget(Registry);
67 extern "C" void LLVMTimeTraceProfilerInitialize() {
68 timeTraceProfilerInitialize(
69 /* TimeTraceGranularity */ 0,
70 /* ProcName */ "rustc");
73 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
74 StringRef FN(FileName);
76 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
78 timeTraceProfilerWrite(OS);
79 timeTraceProfilerCleanup();
82 enum class LLVMRustPassKind {
88 static LLVMRustPassKind toRust(PassKind Kind) {
91 return LLVMRustPassKind::Function;
93 return LLVMRustPassKind::Module;
95 return LLVMRustPassKind::Other;
99 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
100 StringRef SR(PassName);
101 PassRegistry *PR = PassRegistry::getPassRegistry();
103 const PassInfo *PI = PR->getPassInfo(SR);
105 return wrap(PI->createPass());
110 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
111 const bool CompileKernel = false;
112 const bool UseAfterScope = true;
114 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
117 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
118 const bool CompileKernel = false;
120 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
123 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
124 const bool CompileKernel = false;
126 return wrap(createMemorySanitizerLegacyPassPass(
127 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
130 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
131 return wrap(createThreadSanitizerLegacyPassPass());
134 extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) {
135 const bool CompileKernel = false;
137 return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover));
140 extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
142 Pass *Pass = unwrap(RustPass);
143 return toRust(Pass->getPassKind());
146 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
148 Pass *Pass = unwrap(RustPass);
149 PassManagerBase *PMB = unwrap(PMR);
154 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
155 LLVMPassManagerBuilderRef PMBR,
156 LLVMPassManagerRef PMR
158 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
162 void LLVMRustAddLastExtensionPasses(
163 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
164 auto AddExtensionPasses = [Passes, NumPasses](
165 const PassManagerBuilder &Builder, PassManagerBase &PM) {
166 for (size_t I = 0; I < NumPasses; I++) {
167 PM.add(unwrap(Passes[I]));
170 // Add the passes to both of the pre-finalization extension points,
171 // so they are run for optimized and non-optimized builds.
172 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
174 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
178 #ifdef LLVM_COMPONENT_X86
179 #define SUBTARGET_X86 SUBTARGET(X86)
181 #define SUBTARGET_X86
184 #ifdef LLVM_COMPONENT_ARM
185 #define SUBTARGET_ARM SUBTARGET(ARM)
187 #define SUBTARGET_ARM
190 #ifdef LLVM_COMPONENT_AARCH64
191 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
193 #define SUBTARGET_AARCH64
196 #ifdef LLVM_COMPONENT_AVR
197 #define SUBTARGET_AVR SUBTARGET(AVR)
199 #define SUBTARGET_AVR
202 #ifdef LLVM_COMPONENT_MIPS
203 #define SUBTARGET_MIPS SUBTARGET(Mips)
205 #define SUBTARGET_MIPS
208 #ifdef LLVM_COMPONENT_POWERPC
209 #define SUBTARGET_PPC SUBTARGET(PPC)
211 #define SUBTARGET_PPC
214 #ifdef LLVM_COMPONENT_SYSTEMZ
215 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
217 #define SUBTARGET_SYSTEMZ
220 #ifdef LLVM_COMPONENT_MSP430
221 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
223 #define SUBTARGET_MSP430
226 #ifdef LLVM_COMPONENT_RISCV
227 #define SUBTARGET_RISCV SUBTARGET(RISCV)
229 #define SUBTARGET_RISCV
232 #ifdef LLVM_COMPONENT_SPARC
233 #define SUBTARGET_SPARC SUBTARGET(Sparc)
235 #define SUBTARGET_SPARC
238 #ifdef LLVM_COMPONENT_HEXAGON
239 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
241 #define SUBTARGET_HEXAGON
244 #define GEN_SUBTARGETS \
257 #define SUBTARGET(x) \
259 extern const SubtargetFeatureKV x##FeatureKV[]; \
260 extern const SubtargetFeatureKV x##SubTypeKV[]; \
266 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
267 const char *Feature) {
268 TargetMachine *Target = unwrap(TM);
269 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
270 return MCInfo->checkFeatures(std::string("+") + Feature);
273 enum class LLVMRustCodeModel {
282 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
284 case LLVMRustCodeModel::Tiny:
285 return CodeModel::Tiny;
286 case LLVMRustCodeModel::Small:
287 return CodeModel::Small;
288 case LLVMRustCodeModel::Kernel:
289 return CodeModel::Kernel;
290 case LLVMRustCodeModel::Medium:
291 return CodeModel::Medium;
292 case LLVMRustCodeModel::Large:
293 return CodeModel::Large;
294 case LLVMRustCodeModel::None:
297 report_fatal_error("Bad CodeModel.");
301 enum class LLVMRustCodeGenOptLevel {
308 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
310 case LLVMRustCodeGenOptLevel::None:
311 return CodeGenOpt::None;
312 case LLVMRustCodeGenOptLevel::Less:
313 return CodeGenOpt::Less;
314 case LLVMRustCodeGenOptLevel::Default:
315 return CodeGenOpt::Default;
316 case LLVMRustCodeGenOptLevel::Aggressive:
317 return CodeGenOpt::Aggressive;
319 report_fatal_error("Bad CodeGenOptLevel.");
323 enum class LLVMRustPassBuilderOptLevel {
332 static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
334 case LLVMRustPassBuilderOptLevel::O0:
335 return PassBuilder::OptimizationLevel::O0;
336 case LLVMRustPassBuilderOptLevel::O1:
337 return PassBuilder::OptimizationLevel::O1;
338 case LLVMRustPassBuilderOptLevel::O2:
339 return PassBuilder::OptimizationLevel::O2;
340 case LLVMRustPassBuilderOptLevel::O3:
341 return PassBuilder::OptimizationLevel::O3;
342 case LLVMRustPassBuilderOptLevel::Os:
343 return PassBuilder::OptimizationLevel::Os;
344 case LLVMRustPassBuilderOptLevel::Oz:
345 return PassBuilder::OptimizationLevel::Oz;
347 report_fatal_error("Bad PassBuilderOptLevel.");
351 enum class LLVMRustRelocModel {
360 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
362 case LLVMRustRelocModel::Static:
363 return Reloc::Static;
364 case LLVMRustRelocModel::PIC:
366 case LLVMRustRelocModel::DynamicNoPic:
367 return Reloc::DynamicNoPIC;
368 case LLVMRustRelocModel::ROPI:
370 case LLVMRustRelocModel::RWPI:
372 case LLVMRustRelocModel::ROPIRWPI:
373 return Reloc::ROPI_RWPI;
375 report_fatal_error("Bad RelocModel.");
379 /// getLongestEntryLength - Return the length of the longest entry in the table.
380 template<typename KV>
381 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
383 for (auto &I : Table)
384 MaxLen = std::max(MaxLen, std::strlen(I.Key));
388 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
389 const TargetMachine *Target = unwrap(TM);
390 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
391 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
392 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
393 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
394 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
396 printf("Available CPUs for this target:\n");
397 if (HostArch == TargetArch) {
398 const StringRef HostCPU = sys::getHostCPUName();
399 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
400 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
402 for (auto &CPU : CPUTable)
403 printf(" %-*s\n", MaxCPULen, CPU.Key);
407 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
408 const TargetMachine *Target = unwrap(TM);
409 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
410 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
411 unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
413 printf("Available features for this target:\n");
414 for (auto &Feature : FeatTable)
415 printf(" %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
416 printf("\nRust-specific features:\n");
417 printf(" %-*s - %s.\n",
420 "Enables libraries with C Run-time Libraries(CRT) to be statically linked"
424 printf("Use +feature to enable a feature, or -feature to disable it.\n"
425 "For example, rustc -C -target-cpu=mycpu -C "
426 "target-feature=+feature1,-feature2\n\n");
431 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
432 printf("Target CPU help is not supported by this LLVM version.\n\n");
435 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
436 printf("Target features help is not supported by this LLVM version.\n\n");
440 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
441 StringRef Name = sys::getHostCPUName();
446 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
447 const char *TripleStr, const char *CPU, const char *Feature,
448 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
449 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
450 bool FunctionSections,
452 bool TrapUnreachable,
455 bool EmitStackSizeSection,
456 bool RelaxELFRelocations,
458 const char *SplitDwarfFile) {
460 auto OptLevel = fromRust(RustOptLevel);
461 auto RM = fromRust(RustReloc);
462 auto CM = fromRust(RustCM);
465 Triple Trip(Triple::normalize(TripleStr));
466 const llvm::Target *TheTarget =
467 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
468 if (TheTarget == nullptr) {
469 LLVMRustSetLastError(Error.c_str());
473 TargetOptions Options;
475 Options.FloatABIType = FloatABI::Default;
477 Options.FloatABIType = FloatABI::Soft;
479 Options.DataSections = DataSections;
480 Options.FunctionSections = FunctionSections;
481 Options.MCOptions.AsmVerbose = AsmComments;
482 Options.MCOptions.PreserveAsmComments = AsmComments;
483 Options.MCOptions.ABIName = ABIStr;
484 if (SplitDwarfFile) {
485 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
487 Options.RelaxELFRelocations = RelaxELFRelocations;
488 Options.UseInitArray = UseInitArray;
490 if (TrapUnreachable) {
491 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
492 // This limits the extent of possible undefined behavior in some cases, as
493 // it prevents control flow from "falling through" into whatever code
494 // happens to be laid out next in memory.
495 Options.TrapUnreachable = true;
499 Options.ThreadModel = ThreadModel::Single;
502 Options.EmitStackSizeSection = EmitStackSizeSection;
504 TargetMachine *TM = TheTarget->createTargetMachine(
505 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
509 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
513 extern "C" void LLVMRustConfigurePassManagerBuilder(
514 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
515 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
516 const char* PGOGenPath, const char* PGOUsePath) {
517 unwrap(PMBR)->MergeFunctions = MergeFunctions;
518 unwrap(PMBR)->SLPVectorize = SLPVectorize;
519 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
520 unwrap(PMBR)->LoopVectorize = LoopVectorize;
521 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
525 unwrap(PMBR)->EnablePGOInstrGen = true;
526 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
530 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
534 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
535 // field of a PassManagerBuilder, we expose our own method of doing so.
536 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
538 bool DisableSimplifyLibCalls) {
539 Triple TargetTriple(unwrap(M)->getTargetTriple());
540 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
541 if (DisableSimplifyLibCalls)
542 TLI->disableAllFunctions();
543 unwrap(PMBR)->LibraryInfo = TLI;
546 // Unfortunately, the LLVM C API doesn't provide a way to create the
547 // TargetLibraryInfo pass, so we use this method to do so.
548 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
549 bool DisableSimplifyLibCalls) {
550 Triple TargetTriple(unwrap(M)->getTargetTriple());
551 TargetLibraryInfoImpl TLII(TargetTriple);
552 if (DisableSimplifyLibCalls)
553 TLII.disableAllFunctions();
554 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
557 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
558 // all the functions in a module, so we do that manually here. You'll find
559 // similar code in clang's BackendUtil.cpp file.
560 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
562 llvm::legacy::FunctionPassManager *P =
563 unwrap<llvm::legacy::FunctionPassManager>(PMR);
564 P->doInitialization();
566 // Upgrade all calls to old intrinsics first.
567 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
568 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
570 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
572 if (!I->isDeclaration())
578 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
579 // Initializing the command-line options more than once is not allowed. So,
580 // check if they've already been initialized. (This could happen if we're
581 // being called from rustpkg, for example). If the arguments change, then
582 // that's just kinda unfortunate.
583 static bool Initialized = false;
587 cl::ParseCommandLineOptions(Argc, Argv);
590 enum class LLVMRustFileType {
595 static CodeGenFileType fromRust(LLVMRustFileType Type) {
597 case LLVMRustFileType::AssemblyFile:
598 return CGFT_AssemblyFile;
599 case LLVMRustFileType::ObjectFile:
600 return CGFT_ObjectFile;
602 report_fatal_error("Bad FileType.");
606 extern "C" LLVMRustResult
607 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
608 LLVMModuleRef M, const char *Path, const char *DwoPath,
609 LLVMRustFileType RustFileType) {
610 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
611 auto FileType = fromRust(RustFileType);
613 std::string ErrorInfo;
615 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
617 ErrorInfo = EC.message();
618 if (ErrorInfo != "") {
619 LLVMRustSetLastError(ErrorInfo.c_str());
620 return LLVMRustResult::Failure;
623 buffer_ostream BOS(OS);
625 raw_fd_ostream DOS(DwoPath, EC, sys::fs::F_None);
628 ErrorInfo = EC.message();
629 if (ErrorInfo != "") {
630 LLVMRustSetLastError(ErrorInfo.c_str());
631 return LLVMRustResult::Failure;
633 buffer_ostream DBOS(DOS);
634 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
637 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
641 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
642 // stream (OS), so the only real safe place to delete this is here? Don't we
643 // wish this was written in Rust?
644 LLVMDisposePassManager(PMR);
645 return LLVMRustResult::Success;
648 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
649 const char*, // pass name
650 const char*); // IR name
651 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
653 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
654 if (any_isa<const Module *>(WrappedIr))
655 return any_cast<const Module *>(WrappedIr)->getName().str();
656 if (any_isa<const Function *>(WrappedIr))
657 return any_cast<const Function *>(WrappedIr)->getName().str();
658 if (any_isa<const Loop *>(WrappedIr))
659 return any_cast<const Loop *>(WrappedIr)->getName().str();
660 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
661 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
666 void LLVMSelfProfileInitializeCallbacks(
667 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
668 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
669 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
670 #if LLVM_VERSION_GE(12, 0)
671 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
672 StringRef Pass, llvm::Any Ir) {
673 std::string PassName = Pass.str();
674 std::string IrName = LLVMRustwrappedIrGetName(Ir);
675 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
678 PIC.registerAfterPassCallback(
679 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
680 const PreservedAnalyses &Preserved) {
681 AfterPassCallback(LlvmSelfProfiler);
684 PIC.registerAfterPassInvalidatedCallback(
685 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
686 AfterPassCallback(LlvmSelfProfiler);
689 PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback](
690 StringRef Pass, llvm::Any Ir) {
691 std::string PassName = Pass.str();
692 std::string IrName = LLVMRustwrappedIrGetName(Ir);
693 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
697 PIC.registerAfterPassCallback(
698 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
699 AfterPassCallback(LlvmSelfProfiler);
702 PIC.registerAfterPassInvalidatedCallback(
703 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) {
704 AfterPassCallback(LlvmSelfProfiler);
708 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
709 StringRef Pass, llvm::Any Ir) {
710 std::string PassName = Pass.str();
711 std::string IrName = LLVMRustwrappedIrGetName(Ir);
712 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
715 PIC.registerAfterAnalysisCallback(
716 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
717 AfterPassCallback(LlvmSelfProfiler);
721 enum class LLVMRustOptStage {
729 struct LLVMRustSanitizerOptions {
730 bool SanitizeAddress;
731 bool SanitizeAddressRecover;
733 bool SanitizeMemoryRecover;
734 int SanitizeMemoryTrackOrigins;
736 bool SanitizeHWAddress;
737 bool SanitizeHWAddressRecover;
741 LLVMRustOptimizeWithNewPassManager(
742 LLVMModuleRef ModuleRef,
743 LLVMTargetMachineRef TMRef,
744 LLVMRustPassBuilderOptLevel OptLevelRust,
745 LLVMRustOptStage OptStage,
746 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
747 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
748 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
749 LLVMRustSanitizerOptions *SanitizerOptions,
750 const char *PGOGenPath, const char *PGOUsePath,
751 void* LlvmSelfProfiler,
752 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
753 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
754 Module *TheModule = unwrap(ModuleRef);
755 TargetMachine *TM = unwrap(TMRef);
756 PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust);
759 PipelineTuningOptions PTO;
760 PTO.LoopUnrolling = UnrollLoops;
761 PTO.LoopInterleaving = UnrollLoops;
762 PTO.LoopVectorization = LoopVectorize;
763 PTO.SLPVectorization = SLPVectorize;
764 #if LLVM_VERSION_GE(12, 0)
765 PTO.MergeFunctions = MergeFunctions;
767 // MergeFunctions is not supported by NewPM in older LLVM versions.
768 (void) MergeFunctions;
771 // FIXME: We may want to expose this as an option.
772 bool DebugPassManager = false;
774 PassInstrumentationCallbacks PIC;
775 #if LLVM_VERSION_GE(12, 0)
776 StandardInstrumentations SI(DebugPassManager);
778 StandardInstrumentations SI;
780 SI.registerCallbacks(PIC);
782 if (LlvmSelfProfiler){
783 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
786 Optional<PGOOptions> PGOOpt;
789 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr);
790 } else if (PGOUsePath) {
792 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse);
795 #if LLVM_VERSION_GE(12, 0)
796 PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC);
798 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
801 LoopAnalysisManager LAM(DebugPassManager);
802 FunctionAnalysisManager FAM(DebugPassManager);
803 CGSCCAnalysisManager CGAM(DebugPassManager);
804 ModuleAnalysisManager MAM(DebugPassManager);
806 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
808 Triple TargetTriple(TheModule->getTargetTriple());
809 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
810 if (DisableSimplifyLibCalls)
811 TLII->disableAllFunctions();
812 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
814 PB.registerModuleAnalyses(MAM);
815 PB.registerCGSCCAnalyses(CGAM);
816 PB.registerFunctionAnalyses(FAM);
817 PB.registerLoopAnalyses(LAM);
818 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
820 // We manually collect pipeline callbacks so we can apply them at O0, where the
821 // PassBuilder does not create a pipeline.
822 std::vector<std::function<void(ModulePassManager &, PassBuilder::OptimizationLevel)>>
823 PipelineStartEPCallbacks;
824 #if LLVM_VERSION_GE(11, 0)
825 std::vector<std::function<void(ModulePassManager &, PassBuilder::OptimizationLevel)>>
826 OptimizerLastEPCallbacks;
828 std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>>
829 OptimizerLastEPCallbacks;
833 PipelineStartEPCallbacks.push_back(
834 [VerifyIR](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
835 MPM.addPass(VerifierPass());
840 if (SanitizerOptions) {
841 if (SanitizerOptions->SanitizeMemory) {
842 MemorySanitizerOptions Options(
843 SanitizerOptions->SanitizeMemoryTrackOrigins,
844 SanitizerOptions->SanitizeMemoryRecover,
845 /*CompileKernel=*/false);
846 #if LLVM_VERSION_GE(11, 0)
847 OptimizerLastEPCallbacks.push_back(
848 [Options](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
849 MPM.addPass(MemorySanitizerPass(Options));
850 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
854 PipelineStartEPCallbacks.push_back(
855 [Options](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
856 MPM.addPass(MemorySanitizerPass(Options));
859 OptimizerLastEPCallbacks.push_back(
860 [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
861 FPM.addPass(MemorySanitizerPass(Options));
867 if (SanitizerOptions->SanitizeThread) {
868 #if LLVM_VERSION_GE(11, 0)
869 OptimizerLastEPCallbacks.push_back(
870 [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
871 MPM.addPass(ThreadSanitizerPass());
872 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
876 PipelineStartEPCallbacks.push_back(
877 [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
878 MPM.addPass(ThreadSanitizerPass());
881 OptimizerLastEPCallbacks.push_back(
882 [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
883 FPM.addPass(ThreadSanitizerPass());
889 if (SanitizerOptions->SanitizeAddress) {
890 #if LLVM_VERSION_GE(11, 0)
891 OptimizerLastEPCallbacks.push_back(
892 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
893 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
894 MPM.addPass(ModuleAddressSanitizerPass(
895 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
896 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
897 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
898 /*UseAfterScope=*/true)));
902 PipelineStartEPCallbacks.push_back(
903 [&](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
904 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
907 OptimizerLastEPCallbacks.push_back(
908 [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
909 FPM.addPass(AddressSanitizerPass(
910 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
911 /*UseAfterScope=*/true));
914 PipelineStartEPCallbacks.push_back(
915 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
916 MPM.addPass(ModuleAddressSanitizerPass(
917 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
922 if (SanitizerOptions->SanitizeHWAddress) {
923 #if LLVM_VERSION_GE(11, 0)
924 OptimizerLastEPCallbacks.push_back(
925 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
926 MPM.addPass(HWAddressSanitizerPass(
927 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
931 PipelineStartEPCallbacks.push_back(
932 [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
933 MPM.addPass(HWAddressSanitizerPass(
934 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
941 ModulePassManager MPM(DebugPassManager);
942 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
943 if (!NoPrepopulatePasses) {
944 if (OptLevel == PassBuilder::OptimizationLevel::O0) {
945 #if LLVM_VERSION_GE(12, 0)
946 for (const auto &C : PipelineStartEPCallbacks)
947 PB.registerPipelineStartEPCallback(C);
948 for (const auto &C : OptimizerLastEPCallbacks)
949 PB.registerOptimizerLastEPCallback(C);
951 // Pass false as we manually schedule ThinLTOBufferPasses below.
952 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
954 for (const auto &C : PipelineStartEPCallbacks)
957 # if LLVM_VERSION_GE(11, 0)
958 for (const auto &C : OptimizerLastEPCallbacks)
961 if (!OptimizerLastEPCallbacks.empty()) {
962 FunctionPassManager FPM(DebugPassManager);
963 for (const auto &C : OptimizerLastEPCallbacks)
965 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
969 MPM.addPass(AlwaysInlinerPass(EmitLifetimeMarkers));
972 PB.addPGOInstrPassesForO0(
973 MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr,
974 /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
978 #if LLVM_VERSION_GE(12, 0)
979 for (const auto &C : PipelineStartEPCallbacks)
980 PB.registerPipelineStartEPCallback(C);
982 for (const auto &C : PipelineStartEPCallbacks)
983 PB.registerPipelineStartEPCallback([C, OptLevel](ModulePassManager &MPM) {
987 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
988 for (const auto &C : OptimizerLastEPCallbacks)
989 PB.registerOptimizerLastEPCallback(C);
993 case LLVMRustOptStage::PreLinkNoLTO:
994 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
996 case LLVMRustOptStage::PreLinkThinLTO:
997 #if LLVM_VERSION_GE(12, 0)
998 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
999 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
1000 // passes may still run afterwards. This means we need to run the buffer passes again.
1001 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
1002 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
1003 if (OptimizerLastEPCallbacks.empty())
1004 NeedThinLTOBufferPasses = false;
1006 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
1008 #if LLVM_VERSION_GE(11, 0)
1009 for (const auto &C : OptimizerLastEPCallbacks)
1012 if (!OptimizerLastEPCallbacks.empty()) {
1013 FunctionPassManager FPM(DebugPassManager);
1014 for (const auto &C : OptimizerLastEPCallbacks)
1016 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1020 case LLVMRustOptStage::PreLinkFatLTO:
1021 #if LLVM_VERSION_GE(12, 0)
1022 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
1023 NeedThinLTOBufferPasses = false;
1025 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
1028 case LLVMRustOptStage::ThinLTO:
1029 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
1030 // It only seems to be needed for C++ specific optimizations.
1031 #if LLVM_VERSION_GE(12, 0)
1032 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
1034 MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
1037 case LLVMRustOptStage::FatLTO:
1038 #if LLVM_VERSION_GE(12, 0)
1039 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
1041 MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
1048 if (NeedThinLTOBufferPasses) {
1049 MPM.addPass(CanonicalizeAliasesPass());
1050 MPM.addPass(NameAnonGlobalPass());
1053 // Upgrade all calls to old intrinsics first.
1054 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
1055 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
1057 MPM.run(*TheModule, MAM);
1060 // Callback to demangle function name
1062 // * name to be demangled
1065 // * output buffer len
1066 // Returns len of demangled string, or 0 if demangle failed.
1067 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1072 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1073 DemangleFn Demangle;
1074 std::vector<char> Buf;
1077 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1079 // Return empty string if demangle failed
1080 // or if name does not need to be demangled
1081 StringRef CallDemangle(StringRef name) {
1086 if (Buf.size() < name.size() * 2) {
1087 // Semangled name usually shorter than mangled,
1088 // but allocate twice as much memory just in case
1089 Buf.resize(name.size() * 2);
1092 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1098 auto Demangled = StringRef(Buf.data(), R);
1099 if (Demangled == name) {
1100 // Do not print anything if demangled name is equal to mangled.
1107 void emitFunctionAnnot(const Function *F,
1108 formatted_raw_ostream &OS) override {
1109 StringRef Demangled = CallDemangle(F->getName());
1110 if (Demangled.empty()) {
1114 OS << "; " << Demangled << "\n";
1117 void emitInstructionAnnot(const Instruction *I,
1118 formatted_raw_ostream &OS) override {
1121 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1123 Value = CI->getCalledOperand();
1124 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1126 Value = II->getCalledOperand();
1128 // Could demangle more operations, e. g.
1129 // `store %place, @function`.
1133 if (!Value->hasName()) {
1137 StringRef Demangled = CallDemangle(Value->getName());
1138 if (Demangled.empty()) {
1142 OS << "; " << Name << " " << Demangled << "\n";
1148 extern "C" LLVMRustResult
1149 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1150 std::string ErrorInfo;
1152 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
1154 ErrorInfo = EC.message();
1155 if (ErrorInfo != "") {
1156 LLVMRustSetLastError(ErrorInfo.c_str());
1157 return LLVMRustResult::Failure;
1160 RustAssemblyAnnotationWriter AAW(Demangle);
1161 formatted_raw_ostream FOS(OS);
1162 unwrap(M)->print(FOS, &AAW);
1164 return LLVMRustResult::Success;
1167 extern "C" void LLVMRustPrintPasses() {
1168 LLVMInitializePasses();
1169 struct MyListener : PassRegistrationListener {
1170 void passEnumerate(const PassInfo *Info) {
1171 StringRef PassArg = Info->getPassArgument();
1172 StringRef PassName = Info->getPassName();
1173 if (!PassArg.empty()) {
1174 // These unsigned->signed casts could theoretically overflow, but
1175 // realistically never will (and even if, the result is implementation
1176 // defined rather plain UB).
1177 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1178 (int)PassName.size(), PassName.data());
1183 PassRegistry *PR = PassRegistry::getPassRegistry();
1184 PR->enumerateWith(&Listener);
1187 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1188 bool AddLifetimes) {
1189 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1192 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1194 llvm::legacy::PassManager passes;
1196 auto PreserveFunctions = [=](const GlobalValue &GV) {
1197 for (size_t I = 0; I < Len; I++) {
1198 if (GV.getName() == Symbols[I]) {
1205 passes.add(llvm::createInternalizePass(PreserveFunctions));
1207 passes.run(*unwrap(M));
1210 extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
1211 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
1213 GV->setDoesNotThrow();
1214 Function *F = dyn_cast<Function>(GV);
1218 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
1219 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
1220 if (isa<InvokeInst>(I)) {
1221 InvokeInst *CI = cast<InvokeInst>(I);
1222 CI->setDoesNotThrow();
1230 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1231 LLVMTargetMachineRef TMR) {
1232 TargetMachine *Target = unwrap(TMR);
1233 unwrap(Module)->setDataLayout(Target->createDataLayout());
1236 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1237 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1240 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1241 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1244 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1245 LLVMRustCodeModel Model) {
1246 auto CM = fromRust(Model);
1249 unwrap(M)->setCodeModel(*CM);
1252 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1253 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1254 // LLVM, and otherwise it's just blanket rejected from other compilers.
1256 // Most of this implementation is straight copied from LLVM. At the time of
1257 // this writing it wasn't *quite* suitable to reuse more code from upstream
1258 // for our purposes, but we should strive to upstream this support once it's
1259 // ready to go! I figure we may want a bit of testing locally first before
1260 // sending this upstream to LLVM. I hear though they're quite eager to receive
1261 // feedback like this!
1263 // If you're reading this code and wondering "what in the world" or you're
1264 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1265 // then fear not! (ok maybe fear a little). All code here is mostly based
1266 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1268 // You'll find that the general layout here roughly corresponds to the `run`
1269 // method in that file as well as `ProcessThinLTOModule`. Functions are
1270 // specifically commented below as well, but if you're updating this code
1271 // or otherwise trying to understand it, the LLVM source will be useful in
1272 // interpreting the mysteries within.
1274 // Otherwise I'll apologize in advance, it probably requires a relatively
1275 // significant investment on your part to "truly understand" what's going on
1276 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1277 // and various online resources about ThinLTO to make heads or tails of all
1280 // This is a shared data structure which *must* be threadsafe to share
1281 // read-only amongst threads. This also corresponds basically to the arguments
1282 // of the `ProcessThinLTOModule` function in the LLVM source.
1283 struct LLVMRustThinLTOData {
1284 // The combined index that is the global analysis over all modules we're
1285 // performing ThinLTO for. This is mostly managed by LLVM.
1286 ModuleSummaryIndex Index;
1288 // All modules we may look at, stored as in-memory serialized versions. This
1289 // is later used when inlining to ensure we can extract any module to inline
1291 StringMap<MemoryBufferRef> ModuleMap;
1293 // A set that we manage of everything we *don't* want internalized. Note that
1294 // this includes all transitive references right now as well, but it may not
1296 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1298 // Not 100% sure what these are, but they impact what's internalized and
1299 // what's inlined across modules, I believe.
1300 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1301 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1302 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1303 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1305 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1308 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1309 struct LLVMRustThinLTOModule {
1310 const char *identifier;
1315 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1317 static const GlobalValueSummary *
1318 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1319 auto StrongDefForLinker = llvm::find_if(
1320 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1321 auto Linkage = Summary->linkage();
1322 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1323 !GlobalValue::isWeakForLinker(Linkage);
1325 if (StrongDefForLinker != GVSummaryList.end())
1326 return StrongDefForLinker->get();
1328 auto FirstDefForLinker = llvm::find_if(
1329 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1330 auto Linkage = Summary->linkage();
1331 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1333 if (FirstDefForLinker == GVSummaryList.end())
1335 return FirstDefForLinker->get();
1338 // The main entry point for creating the global ThinLTO analysis. The structure
1339 // here is basically the same as before threads are spawned in the `run`
1340 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1341 extern "C" LLVMRustThinLTOData*
1342 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1344 const char **preserved_symbols,
1346 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1348 // Load each module's summary and merge it into one combined index
1349 for (int i = 0; i < num_modules; i++) {
1350 auto module = &modules[i];
1351 StringRef buffer(module->data, module->len);
1352 MemoryBufferRef mem_buffer(buffer, module->identifier);
1354 Ret->ModuleMap[module->identifier] = mem_buffer;
1356 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1357 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1362 // Collect for each module the list of function it defines (GUID -> Summary)
1363 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1365 // Convert the preserved symbols set from string to GUID, this is then needed
1366 // for internalization.
1367 for (int i = 0; i < num_symbols; i++) {
1368 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1369 Ret->GUIDPreservedSymbols.insert(GUID);
1372 // Collect the import/export lists for all modules from the call-graph in the
1375 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1376 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1377 return PrevailingType::Unknown;
1379 // We don't have a complete picture in our use of ThinLTO, just our immediate
1380 // crate, so we need `ImportEnabled = false` to limit internalization.
1381 // Otherwise, we sometimes lose `static` values -- see #60184.
1382 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1383 deadIsPrevailing, /* ImportEnabled = */ false);
1384 ComputeCrossModuleImport(
1386 Ret->ModuleToDefinedGVSummaries,
1391 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1392 // impacts the caching.
1394 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1395 // being lifted from `lib/LTO/LTO.cpp` as well
1396 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1397 for (auto &I : Ret->Index) {
1398 if (I.second.SummaryList.size() > 1)
1399 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1401 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1402 const auto &Prevailing = PrevailingCopy.find(GUID);
1403 if (Prevailing == PrevailingCopy.end())
1405 return Prevailing->second == S;
1407 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1408 GlobalValue::GUID GUID,
1409 GlobalValue::LinkageTypes NewLinkage) {
1410 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1413 #if LLVM_VERSION_GE(13,0)
1414 // Uses FromPrevailing visibility scheme which works for many binary
1415 // formats. We probably could and should use ELF visibility scheme for many of
1416 // our targets, however.
1418 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1419 Ret->GUIDPreservedSymbols);
1421 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
1422 Ret->GUIDPreservedSymbols);
1424 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1425 // callback below. This callback below will dictate the linkage for all
1426 // summaries in the index, and we basically just only want to ensure that dead
1427 // symbols are internalized. Otherwise everything that's already external
1428 // linkage will stay as external, and internal will stay as internal.
1429 std::set<GlobalValue::GUID> ExportedGUIDs;
1430 for (auto &List : Ret->Index) {
1431 for (auto &GVS: List.second.SummaryList) {
1432 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1434 auto GUID = GVS->getOriginalName();
1435 if (GVS->flags().Live)
1436 ExportedGUIDs.insert(GUID);
1439 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1440 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1441 return (ExportList != Ret->ExportLists.end() &&
1442 ExportList->second.count(VI)) ||
1443 ExportedGUIDs.count(VI.getGUID());
1445 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1447 return Ret.release();
1451 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1455 // Below are the various passes that happen *per module* when doing ThinLTO.
1457 // In other words, these are the functions that are all run concurrently
1458 // with one another, one per module. The passes here correspond to the analysis
1459 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1460 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1461 // so rustc can save off the intermediate bytecode between each step.
1463 #if LLVM_VERSION_GE(11, 0)
1465 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1466 // When linking an ELF shared object, dso_local should be dropped. We
1467 // conservatively do this for -fpic.
1468 bool ClearDSOLocalOnDeclarations =
1469 TM.getTargetTriple().isOSBinFormatELF() &&
1470 TM.getRelocationModel() != Reloc::Static &&
1471 Mod.getPIELevel() == PIELevel::Default;
1472 return ClearDSOLocalOnDeclarations;
1477 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1478 LLVMTargetMachineRef TM) {
1479 Module &Mod = *unwrap(M);
1480 TargetMachine &Target = *unwrap(TM);
1482 #if LLVM_VERSION_GE(11, 0)
1483 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1484 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1486 bool error = renameModuleForThinLTO(Mod, Data->Index);
1490 LLVMRustSetLastError("renameModuleForThinLTO failed");
1497 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1498 Module &Mod = *unwrap(M);
1499 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1500 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1505 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1506 Module &Mod = *unwrap(M);
1507 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1508 thinLTOInternalizeModule(Mod, DefinedGlobals);
1513 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1514 LLVMTargetMachineRef TM) {
1515 Module &Mod = *unwrap(M);
1516 TargetMachine &Target = *unwrap(TM);
1518 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1519 auto Loader = [&](StringRef Identifier) {
1520 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1521 auto &Context = Mod.getContext();
1522 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1527 // The rest of this closure is a workaround for
1528 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1529 // we accidentally import wasm custom sections into different modules,
1530 // duplicating them by in the final output artifact.
1532 // The issue is worked around here by manually removing the
1533 // `wasm.custom_sections` named metadata node from any imported module. This
1534 // we know isn't used by any optimization pass so there's no need for it to
1537 // Note that the metadata is currently lazily loaded, so we materialize it
1538 // here before looking up if there's metadata inside. The `FunctionImporter`
1539 // will immediately materialize metadata anyway after an import, so this
1540 // shouldn't be a perf hit.
1541 if (Error Err = (*MOrErr)->materializeMetadata()) {
1542 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1546 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1547 if (WasmCustomSections)
1548 WasmCustomSections->eraseFromParent();
1552 #if LLVM_VERSION_GE(11, 0)
1553 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1554 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1556 FunctionImporter Importer(Data->Index, Loader);
1558 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1560 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1566 extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1567 const char*, // importing module name
1568 const char*); // imported module name
1570 // Calls `module_name_callback` for each module import done by ThinLTO.
1571 // The callback is provided with regular null-terminated C strings.
1573 LLVMRustGetThinLTOModules(const LLVMRustThinLTOData *data,
1574 LLVMRustModuleNameCallback module_name_callback,
1575 void* callback_payload) {
1576 for (const auto& importing_module : data->ImportLists) {
1577 const std::string importing_module_id = importing_module.getKey().str();
1578 const auto& imports = importing_module.getValue();
1579 for (const auto& imported_module : imports) {
1580 const std::string imported_module_id = imported_module.getKey().str();
1581 module_name_callback(callback_payload,
1582 importing_module_id.c_str(),
1583 imported_module_id.c_str());
1588 // This struct and various functions are sort of a hack right now, but the
1589 // problem is that we've got in-memory LLVM modules after we generate and
1590 // optimize all codegen-units for one compilation in rustc. To be compatible
1591 // with the LTO support above we need to serialize the modules plus their
1592 // ThinLTO summary into memory.
1594 // This structure is basically an owned version of a serialize module, with
1595 // a ThinLTO summary attached.
1596 struct LLVMRustThinLTOBuffer {
1600 extern "C" LLVMRustThinLTOBuffer*
1601 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1602 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1604 raw_string_ostream OS(Ret->data);
1606 legacy::PassManager PM;
1607 PM.add(createWriteThinLTOBitcodePass(OS));
1611 return Ret.release();
1615 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1619 extern "C" const void*
1620 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1621 return Buffer->data.data();
1625 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1626 return Buffer->data.length();
1629 // This is what we used to parse upstream bitcode for actual ThinLTO
1630 // processing. We'll call this once per module optimized through ThinLTO, and
1631 // it'll be called concurrently on many threads.
1632 extern "C" LLVMModuleRef
1633 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1636 const char *identifier) {
1637 StringRef Data(data, len);
1638 MemoryBufferRef Buffer(Data, identifier);
1639 unwrap(Context)->enableDebugTypeODRUniquing();
1640 Expected<std::unique_ptr<Module>> SrcOrError =
1641 parseBitcodeFile(Buffer, *unwrap(Context));
1643 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1646 return wrap(std::move(*SrcOrError).release());
1649 // Find the bitcode section in the object file data and return it as a slice.
1650 // Fail if the bitcode section is present but empty.
1652 // On success, the return value is the pointer to the start of the slice and
1653 // `out_len` is filled with the (non-zero) length. On failure, the return value
1654 // is `nullptr` and `out_len` is set to zero.
1655 extern "C" const char*
1656 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1661 StringRef Data(data, len);
1662 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1664 Expected<MemoryBufferRef> BitcodeOrError =
1665 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1666 if (!BitcodeOrError) {
1667 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1671 *out_len = BitcodeOrError->getBufferSize();
1672 return BitcodeOrError->getBufferStart();
1675 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1676 // the comment in `back/lto.rs` for why this exists.
1678 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1680 DICompileUnit **B) {
1681 Module *M = unwrap(Mod);
1682 DICompileUnit **Cur = A;
1683 DICompileUnit **Next = B;
1684 for (DICompileUnit *CU : M->debug_compile_units()) {
1693 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1694 // the comment in `back/lto.rs` for why this exists.
1696 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1697 Module *M = unwrap(Mod);
1699 // If the original source module didn't have a `DICompileUnit` then try to
1700 // merge all the existing compile units. If there aren't actually any though
1701 // then there's not much for us to do so return.
1702 if (Unit == nullptr) {
1703 for (DICompileUnit *CU : M->debug_compile_units()) {
1707 if (Unit == nullptr)
1711 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1712 // process it recursively. Note that we used to specifically iterate over
1713 // instructions to ensure we feed everything into it, but `processModule`
1714 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1715 DebugInfoFinder Finder;
1716 Finder.processModule(*M);
1718 // After we've found all our debuginfo, rewrite all subprograms to point to
1719 // the same `DICompileUnit`.
1720 for (auto &F : Finder.subprograms()) {
1721 F->replaceUnit(Unit);
1724 // Erase any other references to other `DICompileUnit` instances, the verifier
1725 // will later ensure that we don't actually have any other stale references to
1727 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1728 MD->clearOperands();
1729 MD->addOperand(Unit);
1732 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1733 // storing the result in 'KeyOut'.
1734 // Currently, this cache key is a SHA-1 hash of anything that could affect
1735 // the result of optimizing this module (e.g. module imports, exports, liveness
1736 // of access globals, etc).
1737 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1738 // used during the normal linker-plugin incremental thin-LTO process.
1740 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1741 SmallString<40> Key;
1742 llvm::lto::Config conf;
1743 const auto &ImportList = Data->ImportLists.lookup(ModId);
1744 const auto &ExportList = Data->ExportLists.lookup(ModId);
1745 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1746 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1747 std::set<GlobalValue::GUID> CfiFunctionDefs;
1748 std::set<GlobalValue::GUID> CfiFunctionDecls;
1750 // Based on the 'InProcessThinBackend' constructor in LLVM
1751 for (auto &Name : Data->Index.cfiFunctionDefs())
1752 CfiFunctionDefs.insert(
1753 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1754 for (auto &Name : Data->Index.cfiFunctionDecls())
1755 CfiFunctionDecls.insert(
1756 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1758 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1759 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1762 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());