6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/PassPlugin.h"
21 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #if LLVM_VERSION_LT(14, 0)
26 #include "llvm/Support/TargetRegistry.h"
28 #include "llvm/MC/TargetRegistry.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
32 #include "llvm/Transforms/IPO/AlwaysInliner.h"
33 #include "llvm/Transforms/IPO/FunctionImport.h"
34 #if LLVM_VERSION_GE(15, 0)
35 #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
37 #include "llvm/Transforms/Utils/AddDiscriminators.h"
38 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
39 #include "llvm/LTO/LTO.h"
40 #include "llvm/Bitcode/BitcodeWriterPass.h"
41 #include "llvm-c/Transforms/PassManagerBuilder.h"
43 #include "llvm/Transforms/Instrumentation.h"
44 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
45 #include "llvm/Support/TimeProfiler.h"
46 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
47 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
48 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
49 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
50 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
51 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
52 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
53 #include "llvm/Transforms/Utils.h"
57 typedef struct LLVMOpaquePass *LLVMPassRef;
58 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
60 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
61 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
63 extern "C" void LLVMInitializePasses() {
64 PassRegistry &Registry = *PassRegistry::getPassRegistry();
65 initializeCore(Registry);
66 initializeCodeGen(Registry);
67 initializeScalarOpts(Registry);
68 initializeVectorization(Registry);
69 initializeIPO(Registry);
70 initializeAnalysis(Registry);
71 initializeTransformUtils(Registry);
72 initializeInstCombine(Registry);
73 initializeInstrumentation(Registry);
74 initializeTarget(Registry);
77 extern "C" void LLVMTimeTraceProfilerInitialize() {
78 timeTraceProfilerInitialize(
79 /* TimeTraceGranularity */ 0,
80 /* ProcName */ "rustc");
83 extern "C" void LLVMTimeTraceProfilerFinishThread() {
84 timeTraceProfilerFinishThread();
87 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
88 StringRef FN(FileName);
90 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
92 timeTraceProfilerWrite(OS);
93 timeTraceProfilerCleanup();
96 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
97 #if LLVM_VERSION_LT(15, 0)
98 StringRef SR(PassName);
99 PassRegistry *PR = PassRegistry::getPassRegistry();
101 const PassInfo *PI = PR->getPassInfo(SR);
103 return wrap(PI->createPass());
107 report_fatal_error("Legacy PM not supported with LLVM 15");
111 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
112 #if LLVM_VERSION_LT(15, 0)
113 const bool CompileKernel = false;
114 const bool UseAfterScope = true;
116 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
118 report_fatal_error("Legacy PM not supported with LLVM 15");
122 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
123 #if LLVM_VERSION_LT(15, 0)
124 const bool CompileKernel = false;
126 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
128 report_fatal_error("Legacy PM not supported with LLVM 15");
132 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
133 #if LLVM_VERSION_LT(15, 0)
134 const bool CompileKernel = false;
136 return wrap(createMemorySanitizerLegacyPassPass(
137 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
139 report_fatal_error("Legacy PM not supported with LLVM 15");
143 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
144 #if LLVM_VERSION_LT(15, 0)
145 return wrap(createThreadSanitizerLegacyPassPass());
147 report_fatal_error("Legacy PM not supported with LLVM 15");
151 extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) {
152 #if LLVM_VERSION_LT(15, 0)
153 const bool CompileKernel = false;
155 return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover));
157 report_fatal_error("Legacy PM not supported with LLVM 15");
161 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
162 #if LLVM_VERSION_LT(15, 0)
164 Pass *Pass = unwrap(RustPass);
165 PassManagerBase *PMB = unwrap(PMR);
168 report_fatal_error("Legacy PM not supported with LLVM 15");
172 extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() {
173 #if LLVM_VERSION_LT(15, 0)
174 return LLVMPassManagerBuilderCreate();
176 report_fatal_error("Legacy PM not supported with LLVM 15");
180 extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) {
181 #if LLVM_VERSION_LT(15, 0)
182 LLVMPassManagerBuilderDispose(PMB);
184 report_fatal_error("Legacy PM not supported with LLVM 15");
188 extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager(
189 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) {
190 #if LLVM_VERSION_LT(15, 0)
191 LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM);
193 report_fatal_error("Legacy PM not supported with LLVM 15");
197 extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager(
198 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) {
199 #if LLVM_VERSION_LT(15, 0)
200 LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM);
202 report_fatal_error("Legacy PM not supported with LLVM 15");
206 extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager(
207 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) {
208 #if LLVM_VERSION_LT(15, 0)
209 LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner);
211 report_fatal_error("Legacy PM not supported with LLVM 15");
216 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
217 LLVMPassManagerBuilderRef PMBR,
218 LLVMPassManagerRef PMR
220 #if LLVM_VERSION_LT(15, 0)
221 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
223 report_fatal_error("Legacy PM not supported with LLVM 15");
227 extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold(
228 LLVMPassManagerBuilderRef PMB, unsigned Threshold) {
229 #if LLVM_VERSION_LT(15, 0)
230 LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold);
232 report_fatal_error("Legacy PM not supported with LLVM 15");
237 void LLVMRustAddLastExtensionPasses(
238 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
239 #if LLVM_VERSION_LT(15, 0)
240 auto AddExtensionPasses = [Passes, NumPasses](
241 const PassManagerBuilder &Builder, PassManagerBase &PM) {
242 for (size_t I = 0; I < NumPasses; I++) {
243 PM.add(unwrap(Passes[I]));
246 // Add the passes to both of the pre-finalization extension points,
247 // so they are run for optimized and non-optimized builds.
248 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
250 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
253 report_fatal_error("Legacy PM not supported with LLVM 15");
257 #ifdef LLVM_COMPONENT_X86
258 #define SUBTARGET_X86 SUBTARGET(X86)
260 #define SUBTARGET_X86
263 #ifdef LLVM_COMPONENT_ARM
264 #define SUBTARGET_ARM SUBTARGET(ARM)
266 #define SUBTARGET_ARM
269 #ifdef LLVM_COMPONENT_AARCH64
270 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
272 #define SUBTARGET_AARCH64
275 #ifdef LLVM_COMPONENT_AVR
276 #define SUBTARGET_AVR SUBTARGET(AVR)
278 #define SUBTARGET_AVR
281 #ifdef LLVM_COMPONENT_M68k
282 #define SUBTARGET_M68K SUBTARGET(M68k)
284 #define SUBTARGET_M68K
287 #ifdef LLVM_COMPONENT_MIPS
288 #define SUBTARGET_MIPS SUBTARGET(Mips)
290 #define SUBTARGET_MIPS
293 #ifdef LLVM_COMPONENT_POWERPC
294 #define SUBTARGET_PPC SUBTARGET(PPC)
296 #define SUBTARGET_PPC
299 #ifdef LLVM_COMPONENT_SYSTEMZ
300 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
302 #define SUBTARGET_SYSTEMZ
305 #ifdef LLVM_COMPONENT_MSP430
306 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
308 #define SUBTARGET_MSP430
311 #ifdef LLVM_COMPONENT_RISCV
312 #define SUBTARGET_RISCV SUBTARGET(RISCV)
314 #define SUBTARGET_RISCV
317 #ifdef LLVM_COMPONENT_SPARC
318 #define SUBTARGET_SPARC SUBTARGET(Sparc)
320 #define SUBTARGET_SPARC
323 #ifdef LLVM_COMPONENT_HEXAGON
324 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
326 #define SUBTARGET_HEXAGON
329 #define GEN_SUBTARGETS \
343 #define SUBTARGET(x) \
345 extern const SubtargetFeatureKV x##FeatureKV[]; \
346 extern const SubtargetFeatureKV x##SubTypeKV[]; \
352 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
353 const char *Feature) {
354 TargetMachine *Target = unwrap(TM);
355 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
356 return MCInfo->checkFeatures(std::string("+") + Feature);
359 enum class LLVMRustCodeModel {
368 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
370 case LLVMRustCodeModel::Tiny:
371 return CodeModel::Tiny;
372 case LLVMRustCodeModel::Small:
373 return CodeModel::Small;
374 case LLVMRustCodeModel::Kernel:
375 return CodeModel::Kernel;
376 case LLVMRustCodeModel::Medium:
377 return CodeModel::Medium;
378 case LLVMRustCodeModel::Large:
379 return CodeModel::Large;
380 case LLVMRustCodeModel::None:
383 report_fatal_error("Bad CodeModel.");
387 enum class LLVMRustCodeGenOptLevel {
394 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
396 case LLVMRustCodeGenOptLevel::None:
397 return CodeGenOpt::None;
398 case LLVMRustCodeGenOptLevel::Less:
399 return CodeGenOpt::Less;
400 case LLVMRustCodeGenOptLevel::Default:
401 return CodeGenOpt::Default;
402 case LLVMRustCodeGenOptLevel::Aggressive:
403 return CodeGenOpt::Aggressive;
405 report_fatal_error("Bad CodeGenOptLevel.");
409 enum class LLVMRustPassBuilderOptLevel {
418 #if LLVM_VERSION_LT(14,0)
419 using OptimizationLevel = PassBuilder::OptimizationLevel;
422 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
424 case LLVMRustPassBuilderOptLevel::O0:
425 return OptimizationLevel::O0;
426 case LLVMRustPassBuilderOptLevel::O1:
427 return OptimizationLevel::O1;
428 case LLVMRustPassBuilderOptLevel::O2:
429 return OptimizationLevel::O2;
430 case LLVMRustPassBuilderOptLevel::O3:
431 return OptimizationLevel::O3;
432 case LLVMRustPassBuilderOptLevel::Os:
433 return OptimizationLevel::Os;
434 case LLVMRustPassBuilderOptLevel::Oz:
435 return OptimizationLevel::Oz;
437 report_fatal_error("Bad PassBuilderOptLevel.");
441 enum class LLVMRustRelocModel {
450 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
452 case LLVMRustRelocModel::Static:
453 return Reloc::Static;
454 case LLVMRustRelocModel::PIC:
456 case LLVMRustRelocModel::DynamicNoPic:
457 return Reloc::DynamicNoPIC;
458 case LLVMRustRelocModel::ROPI:
460 case LLVMRustRelocModel::RWPI:
462 case LLVMRustRelocModel::ROPIRWPI:
463 return Reloc::ROPI_RWPI;
465 report_fatal_error("Bad RelocModel.");
469 /// getLongestEntryLength - Return the length of the longest entry in the table.
470 template<typename KV>
471 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
473 for (auto &I : Table)
474 MaxLen = std::max(MaxLen, std::strlen(I.Key));
478 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
479 const TargetMachine *Target = unwrap(TM);
480 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
481 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
482 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
483 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
484 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
486 printf("Available CPUs for this target:\n");
487 if (HostArch == TargetArch) {
488 const StringRef HostCPU = sys::getHostCPUName();
489 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
490 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
492 for (auto &CPU : CPUTable)
493 printf(" %-*s\n", MaxCPULen, CPU.Key);
497 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
498 const TargetMachine *Target = unwrap(TM);
499 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
500 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
501 return FeatTable.size();
504 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
505 const char** Feature, const char** Desc) {
506 const TargetMachine *Target = unwrap(TM);
507 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
508 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
509 const SubtargetFeatureKV Feat = FeatTable[Index];
516 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
517 printf("Target CPU help is not supported by this LLVM version.\n\n");
520 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
524 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
527 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
528 StringRef Name = sys::getHostCPUName();
533 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
534 const char *TripleStr, const char *CPU, const char *Feature,
535 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
536 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
537 bool FunctionSections,
539 bool UniqueSectionNames,
540 bool TrapUnreachable,
543 bool EmitStackSizeSection,
544 bool RelaxELFRelocations,
546 const char *SplitDwarfFile) {
548 auto OptLevel = fromRust(RustOptLevel);
549 auto RM = fromRust(RustReloc);
550 auto CM = fromRust(RustCM);
553 Triple Trip(Triple::normalize(TripleStr));
554 const llvm::Target *TheTarget =
555 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
556 if (TheTarget == nullptr) {
557 LLVMRustSetLastError(Error.c_str());
561 TargetOptions Options;
563 Options.FloatABIType = FloatABI::Default;
565 Options.FloatABIType = FloatABI::Soft;
567 Options.DataSections = DataSections;
568 Options.FunctionSections = FunctionSections;
569 Options.UniqueSectionNames = UniqueSectionNames;
570 Options.MCOptions.AsmVerbose = AsmComments;
571 Options.MCOptions.PreserveAsmComments = AsmComments;
572 Options.MCOptions.ABIName = ABIStr;
573 if (SplitDwarfFile) {
574 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
576 Options.RelaxELFRelocations = RelaxELFRelocations;
577 Options.UseInitArray = UseInitArray;
579 if (TrapUnreachable) {
580 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
581 // This limits the extent of possible undefined behavior in some cases, as
582 // it prevents control flow from "falling through" into whatever code
583 // happens to be laid out next in memory.
584 Options.TrapUnreachable = true;
588 Options.ThreadModel = ThreadModel::Single;
591 Options.EmitStackSizeSection = EmitStackSizeSection;
593 TargetMachine *TM = TheTarget->createTargetMachine(
594 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
598 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
602 extern "C" void LLVMRustConfigurePassManagerBuilder(
603 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
604 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
605 const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath,
607 #if LLVM_VERSION_LT(15, 0)
608 unwrap(PMBR)->MergeFunctions = MergeFunctions;
609 unwrap(PMBR)->SLPVectorize = SLPVectorize;
610 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
611 unwrap(PMBR)->LoopVectorize = LoopVectorize;
612 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
613 unwrap(PMBR)->SizeLevel = SizeLevel;
614 unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0;
617 assert(!PGOUsePath && !PGOSampleUsePath);
618 unwrap(PMBR)->EnablePGOInstrGen = true;
619 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
620 } else if (PGOUsePath) {
621 assert(!PGOSampleUsePath);
622 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
623 } else if (PGOSampleUsePath) {
624 unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath;
627 report_fatal_error("Legacy PM not supported with LLVM 15");
631 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
632 // field of a PassManagerBuilder, we expose our own method of doing so.
633 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
635 bool DisableSimplifyLibCalls) {
636 Triple TargetTriple(unwrap(M)->getTargetTriple());
637 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
638 if (DisableSimplifyLibCalls)
639 TLI->disableAllFunctions();
640 unwrap(PMBR)->LibraryInfo = TLI;
643 // Unfortunately, the LLVM C API doesn't provide a way to create the
644 // TargetLibraryInfo pass, so we use this method to do so.
645 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
646 bool DisableSimplifyLibCalls) {
647 Triple TargetTriple(unwrap(M)->getTargetTriple());
648 TargetLibraryInfoImpl TLII(TargetTriple);
649 if (DisableSimplifyLibCalls)
650 TLII.disableAllFunctions();
651 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
654 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
655 // all the functions in a module, so we do that manually here. You'll find
656 // similar code in clang's BackendUtil.cpp file.
657 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
659 llvm::legacy::FunctionPassManager *P =
660 unwrap<llvm::legacy::FunctionPassManager>(PMR);
661 P->doInitialization();
663 // Upgrade all calls to old intrinsics first.
664 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
665 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
667 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
669 if (!I->isDeclaration())
675 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
676 // Initializing the command-line options more than once is not allowed. So,
677 // check if they've already been initialized. (This could happen if we're
678 // being called from rustpkg, for example). If the arguments change, then
679 // that's just kinda unfortunate.
680 static bool Initialized = false;
684 cl::ParseCommandLineOptions(Argc, Argv);
687 enum class LLVMRustFileType {
692 static CodeGenFileType fromRust(LLVMRustFileType Type) {
694 case LLVMRustFileType::AssemblyFile:
695 return CGFT_AssemblyFile;
696 case LLVMRustFileType::ObjectFile:
697 return CGFT_ObjectFile;
699 report_fatal_error("Bad FileType.");
703 extern "C" LLVMRustResult
704 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
705 LLVMModuleRef M, const char *Path, const char *DwoPath,
706 LLVMRustFileType RustFileType) {
707 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
708 auto FileType = fromRust(RustFileType);
710 std::string ErrorInfo;
712 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
714 ErrorInfo = EC.message();
715 if (ErrorInfo != "") {
716 LLVMRustSetLastError(ErrorInfo.c_str());
717 return LLVMRustResult::Failure;
720 buffer_ostream BOS(OS);
722 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
725 ErrorInfo = EC.message();
726 if (ErrorInfo != "") {
727 LLVMRustSetLastError(ErrorInfo.c_str());
728 return LLVMRustResult::Failure;
730 buffer_ostream DBOS(DOS);
731 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
734 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
738 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
739 // stream (OS), so the only real safe place to delete this is here? Don't we
740 // wish this was written in Rust?
741 LLVMDisposePassManager(PMR);
742 return LLVMRustResult::Success;
745 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
746 const char*, // pass name
747 const char*); // IR name
748 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
750 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
751 if (any_isa<const Module *>(WrappedIr))
752 return any_cast<const Module *>(WrappedIr)->getName().str();
753 if (any_isa<const Function *>(WrappedIr))
754 return any_cast<const Function *>(WrappedIr)->getName().str();
755 if (any_isa<const Loop *>(WrappedIr))
756 return any_cast<const Loop *>(WrappedIr)->getName().str();
757 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
758 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
763 void LLVMSelfProfileInitializeCallbacks(
764 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
765 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
766 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
767 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
768 StringRef Pass, llvm::Any Ir) {
769 std::string PassName = Pass.str();
770 std::string IrName = LLVMRustwrappedIrGetName(Ir);
771 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
774 PIC.registerAfterPassCallback(
775 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
776 const PreservedAnalyses &Preserved) {
777 AfterPassCallback(LlvmSelfProfiler);
780 PIC.registerAfterPassInvalidatedCallback(
781 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
782 AfterPassCallback(LlvmSelfProfiler);
785 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
786 StringRef Pass, llvm::Any Ir) {
787 std::string PassName = Pass.str();
788 std::string IrName = LLVMRustwrappedIrGetName(Ir);
789 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
792 PIC.registerAfterAnalysisCallback(
793 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
794 AfterPassCallback(LlvmSelfProfiler);
798 enum class LLVMRustOptStage {
806 struct LLVMRustSanitizerOptions {
807 bool SanitizeAddress;
808 bool SanitizeAddressRecover;
810 bool SanitizeMemoryRecover;
811 int SanitizeMemoryTrackOrigins;
813 bool SanitizeHWAddress;
814 bool SanitizeHWAddressRecover;
817 extern "C" LLVMRustResult
818 LLVMRustOptimizeWithNewPassManager(
819 LLVMModuleRef ModuleRef,
820 LLVMTargetMachineRef TMRef,
821 LLVMRustPassBuilderOptLevel OptLevelRust,
822 LLVMRustOptStage OptStage,
823 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
824 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
825 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
826 LLVMRustSanitizerOptions *SanitizerOptions,
827 const char *PGOGenPath, const char *PGOUsePath,
828 bool InstrumentCoverage, const char *InstrProfileOutput,
830 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
831 void* LlvmSelfProfiler,
832 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
833 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
834 const char *ExtraPasses, size_t ExtraPassesLen,
835 const char *LLVMPlugins, size_t LLVMPluginsLen) {
836 Module *TheModule = unwrap(ModuleRef);
837 TargetMachine *TM = unwrap(TMRef);
838 OptimizationLevel OptLevel = fromRust(OptLevelRust);
841 PipelineTuningOptions PTO;
842 PTO.LoopUnrolling = UnrollLoops;
843 PTO.LoopInterleaving = UnrollLoops;
844 PTO.LoopVectorization = LoopVectorize;
845 PTO.SLPVectorization = SLPVectorize;
846 PTO.MergeFunctions = MergeFunctions;
848 // FIXME: We may want to expose this as an option.
849 bool DebugPassManager = false;
851 PassInstrumentationCallbacks PIC;
852 StandardInstrumentations SI(DebugPassManager);
853 SI.registerCallbacks(PIC);
855 if (LlvmSelfProfiler){
856 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
859 Optional<PGOOptions> PGOOpt;
861 assert(!PGOUsePath && !PGOSampleUsePath);
862 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
863 PGOOptions::NoCSAction, DebugInfoForProfiling);
864 } else if (PGOUsePath) {
865 assert(!PGOSampleUsePath);
866 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
867 PGOOptions::NoCSAction, DebugInfoForProfiling);
868 } else if (PGOSampleUsePath) {
869 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
870 PGOOptions::NoCSAction, DebugInfoForProfiling);
871 } else if (DebugInfoForProfiling) {
872 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
873 PGOOptions::NoCSAction, DebugInfoForProfiling);
876 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
877 LoopAnalysisManager LAM;
878 FunctionAnalysisManager FAM;
879 CGSCCAnalysisManager CGAM;
880 ModuleAnalysisManager MAM;
882 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
884 Triple TargetTriple(TheModule->getTargetTriple());
885 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
886 if (DisableSimplifyLibCalls)
887 TLII->disableAllFunctions();
888 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
890 PB.registerModuleAnalyses(MAM);
891 PB.registerCGSCCAnalyses(CGAM);
892 PB.registerFunctionAnalyses(FAM);
893 PB.registerLoopAnalyses(LAM);
894 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
896 // We manually collect pipeline callbacks so we can apply them at O0, where the
897 // PassBuilder does not create a pipeline.
898 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
899 PipelineStartEPCallbacks;
900 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
901 OptimizerLastEPCallbacks;
904 PipelineStartEPCallbacks.push_back(
905 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
906 MPM.addPass(VerifierPass());
911 if (InstrumentGCOV) {
912 PipelineStartEPCallbacks.push_back(
913 [](ModulePassManager &MPM, OptimizationLevel Level) {
914 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
919 if (InstrumentCoverage) {
920 PipelineStartEPCallbacks.push_back(
921 [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) {
922 InstrProfOptions Options;
923 if (InstrProfileOutput) {
924 Options.InstrProfileOutput = InstrProfileOutput;
926 MPM.addPass(InstrProfiling(Options, false));
931 if (SanitizerOptions) {
932 if (SanitizerOptions->SanitizeMemory) {
933 MemorySanitizerOptions Options(
934 SanitizerOptions->SanitizeMemoryTrackOrigins,
935 SanitizerOptions->SanitizeMemoryRecover,
936 /*CompileKernel=*/false);
937 OptimizerLastEPCallbacks.push_back(
938 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
939 #if LLVM_VERSION_GE(14, 0)
940 MPM.addPass(ModuleMemorySanitizerPass(Options));
942 MPM.addPass(MemorySanitizerPass(Options));
944 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
949 if (SanitizerOptions->SanitizeThread) {
950 OptimizerLastEPCallbacks.push_back(
951 [](ModulePassManager &MPM, OptimizationLevel Level) {
952 #if LLVM_VERSION_GE(14, 0)
953 MPM.addPass(ModuleThreadSanitizerPass());
955 MPM.addPass(ThreadSanitizerPass());
957 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
962 if (SanitizerOptions->SanitizeAddress) {
963 OptimizerLastEPCallbacks.push_back(
964 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
965 #if LLVM_VERSION_LT(15, 0)
966 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
968 #if LLVM_VERSION_GE(14, 0)
969 AddressSanitizerOptions opts = AddressSanitizerOptions{
970 /*CompileKernel=*/false,
971 SanitizerOptions->SanitizeAddressRecover,
972 /*UseAfterScope=*/true,
973 AsanDetectStackUseAfterReturnMode::Runtime,
975 MPM.addPass(ModuleAddressSanitizerPass(opts));
977 MPM.addPass(ModuleAddressSanitizerPass(
978 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
979 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
980 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
981 /*UseAfterScope=*/true)));
986 if (SanitizerOptions->SanitizeHWAddress) {
987 OptimizerLastEPCallbacks.push_back(
988 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
989 #if LLVM_VERSION_GE(14, 0)
990 HWAddressSanitizerOptions opts(
991 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
992 /*DisableOptimization=*/false);
993 MPM.addPass(HWAddressSanitizerPass(opts));
995 MPM.addPass(HWAddressSanitizerPass(
996 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
1003 if (LLVMPluginsLen) {
1004 auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
1005 SmallVector<StringRef> Plugins;
1006 PluginsStr.split(Plugins, ',', -1, false);
1007 for (auto PluginPath: Plugins) {
1008 auto Plugin = PassPlugin::Load(PluginPath.str());
1010 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
1013 Plugin->registerPassBuilderCallbacks(PB);
1017 ModulePassManager MPM;
1018 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
1019 if (!NoPrepopulatePasses) {
1020 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
1021 // At the same time, the LTO pipelines do support O0 and using them is required.
1022 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
1023 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
1024 for (const auto &C : PipelineStartEPCallbacks)
1025 PB.registerPipelineStartEPCallback(C);
1026 for (const auto &C : OptimizerLastEPCallbacks)
1027 PB.registerOptimizerLastEPCallback(C);
1029 // Pass false as we manually schedule ThinLTOBufferPasses below.
1030 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
1032 for (const auto &C : PipelineStartEPCallbacks)
1033 PB.registerPipelineStartEPCallback(C);
1034 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
1035 for (const auto &C : OptimizerLastEPCallbacks)
1036 PB.registerOptimizerLastEPCallback(C);
1040 case LLVMRustOptStage::PreLinkNoLTO:
1041 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
1043 case LLVMRustOptStage::PreLinkThinLTO:
1044 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
1045 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
1046 // passes may still run afterwards. This means we need to run the buffer passes again.
1047 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
1048 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
1049 if (OptimizerLastEPCallbacks.empty())
1050 NeedThinLTOBufferPasses = false;
1051 for (const auto &C : OptimizerLastEPCallbacks)
1054 case LLVMRustOptStage::PreLinkFatLTO:
1055 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
1056 NeedThinLTOBufferPasses = false;
1058 case LLVMRustOptStage::ThinLTO:
1059 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
1060 // It only seems to be needed for C++ specific optimizations.
1061 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
1063 case LLVMRustOptStage::FatLTO:
1064 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
1069 // We're not building any of the default pipelines but we still want to
1070 // add the verifier, instrumentation, etc passes if they were requested
1071 for (const auto &C : PipelineStartEPCallbacks)
1073 for (const auto &C : OptimizerLastEPCallbacks)
1077 if (ExtraPassesLen) {
1078 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
1079 std::string ErrMsg = toString(std::move(Err));
1080 LLVMRustSetLastError(ErrMsg.c_str());
1081 return LLVMRustResult::Failure;
1085 if (NeedThinLTOBufferPasses) {
1086 MPM.addPass(CanonicalizeAliasesPass());
1087 MPM.addPass(NameAnonGlobalPass());
1090 // Upgrade all calls to old intrinsics first.
1091 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
1092 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
1094 MPM.run(*TheModule, MAM);
1095 return LLVMRustResult::Success;
1098 // Callback to demangle function name
1100 // * name to be demangled
1103 // * output buffer len
1104 // Returns len of demangled string, or 0 if demangle failed.
1105 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1110 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1111 DemangleFn Demangle;
1112 std::vector<char> Buf;
1115 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1117 // Return empty string if demangle failed
1118 // or if name does not need to be demangled
1119 StringRef CallDemangle(StringRef name) {
1124 if (Buf.size() < name.size() * 2) {
1125 // Semangled name usually shorter than mangled,
1126 // but allocate twice as much memory just in case
1127 Buf.resize(name.size() * 2);
1130 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1136 auto Demangled = StringRef(Buf.data(), R);
1137 if (Demangled == name) {
1138 // Do not print anything if demangled name is equal to mangled.
1145 void emitFunctionAnnot(const Function *F,
1146 formatted_raw_ostream &OS) override {
1147 StringRef Demangled = CallDemangle(F->getName());
1148 if (Demangled.empty()) {
1152 OS << "; " << Demangled << "\n";
1155 void emitInstructionAnnot(const Instruction *I,
1156 formatted_raw_ostream &OS) override {
1159 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1161 Value = CI->getCalledOperand();
1162 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1164 Value = II->getCalledOperand();
1166 // Could demangle more operations, e. g.
1167 // `store %place, @function`.
1171 if (!Value->hasName()) {
1175 StringRef Demangled = CallDemangle(Value->getName());
1176 if (Demangled.empty()) {
1180 OS << "; " << Name << " " << Demangled << "\n";
1186 extern "C" LLVMRustResult
1187 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1188 std::string ErrorInfo;
1190 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
1192 ErrorInfo = EC.message();
1193 if (ErrorInfo != "") {
1194 LLVMRustSetLastError(ErrorInfo.c_str());
1195 return LLVMRustResult::Failure;
1198 RustAssemblyAnnotationWriter AAW(Demangle);
1199 formatted_raw_ostream FOS(OS);
1200 unwrap(M)->print(FOS, &AAW);
1202 return LLVMRustResult::Success;
1205 extern "C" void LLVMRustPrintPasses() {
1206 LLVMInitializePasses();
1207 struct MyListener : PassRegistrationListener {
1208 void passEnumerate(const PassInfo *Info) {
1209 StringRef PassArg = Info->getPassArgument();
1210 StringRef PassName = Info->getPassName();
1211 if (!PassArg.empty()) {
1212 // These unsigned->signed casts could theoretically overflow, but
1213 // realistically never will (and even if, the result is implementation
1214 // defined rather plain UB).
1215 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1216 (int)PassName.size(), PassName.data());
1221 PassRegistry *PR = PassRegistry::getPassRegistry();
1222 PR->enumerateWith(&Listener);
1225 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1226 bool AddLifetimes) {
1227 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1230 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1232 llvm::legacy::PassManager passes;
1234 auto PreserveFunctions = [=](const GlobalValue &GV) {
1235 for (size_t I = 0; I < Len; I++) {
1236 if (GV.getName() == Symbols[I]) {
1243 passes.add(llvm::createInternalizePass(PreserveFunctions));
1245 passes.run(*unwrap(M));
1249 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1250 LLVMTargetMachineRef TMR) {
1251 TargetMachine *Target = unwrap(TMR);
1252 unwrap(Module)->setDataLayout(Target->createDataLayout());
1255 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1256 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1259 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1260 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1263 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1264 LLVMRustCodeModel Model) {
1265 auto CM = fromRust(Model);
1268 unwrap(M)->setCodeModel(*CM);
1271 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1272 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1273 // LLVM, and otherwise it's just blanket rejected from other compilers.
1275 // Most of this implementation is straight copied from LLVM. At the time of
1276 // this writing it wasn't *quite* suitable to reuse more code from upstream
1277 // for our purposes, but we should strive to upstream this support once it's
1278 // ready to go! I figure we may want a bit of testing locally first before
1279 // sending this upstream to LLVM. I hear though they're quite eager to receive
1280 // feedback like this!
1282 // If you're reading this code and wondering "what in the world" or you're
1283 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1284 // then fear not! (ok maybe fear a little). All code here is mostly based
1285 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1287 // You'll find that the general layout here roughly corresponds to the `run`
1288 // method in that file as well as `ProcessThinLTOModule`. Functions are
1289 // specifically commented below as well, but if you're updating this code
1290 // or otherwise trying to understand it, the LLVM source will be useful in
1291 // interpreting the mysteries within.
1293 // Otherwise I'll apologize in advance, it probably requires a relatively
1294 // significant investment on your part to "truly understand" what's going on
1295 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1296 // and various online resources about ThinLTO to make heads or tails of all
1299 // This is a shared data structure which *must* be threadsafe to share
1300 // read-only amongst threads. This also corresponds basically to the arguments
1301 // of the `ProcessThinLTOModule` function in the LLVM source.
1302 struct LLVMRustThinLTOData {
1303 // The combined index that is the global analysis over all modules we're
1304 // performing ThinLTO for. This is mostly managed by LLVM.
1305 ModuleSummaryIndex Index;
1307 // All modules we may look at, stored as in-memory serialized versions. This
1308 // is later used when inlining to ensure we can extract any module to inline
1310 StringMap<MemoryBufferRef> ModuleMap;
1312 // A set that we manage of everything we *don't* want internalized. Note that
1313 // this includes all transitive references right now as well, but it may not
1315 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1317 // Not 100% sure what these are, but they impact what's internalized and
1318 // what's inlined across modules, I believe.
1319 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1320 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1321 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1322 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1324 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1327 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1328 struct LLVMRustThinLTOModule {
1329 const char *identifier;
1334 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1336 static const GlobalValueSummary *
1337 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1338 auto StrongDefForLinker = llvm::find_if(
1339 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1340 auto Linkage = Summary->linkage();
1341 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1342 !GlobalValue::isWeakForLinker(Linkage);
1344 if (StrongDefForLinker != GVSummaryList.end())
1345 return StrongDefForLinker->get();
1347 auto FirstDefForLinker = llvm::find_if(
1348 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1349 auto Linkage = Summary->linkage();
1350 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1352 if (FirstDefForLinker == GVSummaryList.end())
1354 return FirstDefForLinker->get();
1357 // The main entry point for creating the global ThinLTO analysis. The structure
1358 // here is basically the same as before threads are spawned in the `run`
1359 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1360 extern "C" LLVMRustThinLTOData*
1361 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1363 const char **preserved_symbols,
1365 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1367 // Load each module's summary and merge it into one combined index
1368 for (int i = 0; i < num_modules; i++) {
1369 auto module = &modules[i];
1370 StringRef buffer(module->data, module->len);
1371 MemoryBufferRef mem_buffer(buffer, module->identifier);
1373 Ret->ModuleMap[module->identifier] = mem_buffer;
1375 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1376 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1381 // Collect for each module the list of function it defines (GUID -> Summary)
1382 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1384 // Convert the preserved symbols set from string to GUID, this is then needed
1385 // for internalization.
1386 for (int i = 0; i < num_symbols; i++) {
1387 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1388 Ret->GUIDPreservedSymbols.insert(GUID);
1391 // Collect the import/export lists for all modules from the call-graph in the
1394 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1395 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1396 return PrevailingType::Unknown;
1398 // We don't have a complete picture in our use of ThinLTO, just our immediate
1399 // crate, so we need `ImportEnabled = false` to limit internalization.
1400 // Otherwise, we sometimes lose `static` values -- see #60184.
1401 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1402 deadIsPrevailing, /* ImportEnabled = */ false);
1403 ComputeCrossModuleImport(
1405 Ret->ModuleToDefinedGVSummaries,
1410 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1411 // impacts the caching.
1413 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1414 // being lifted from `lib/LTO/LTO.cpp` as well
1415 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1416 for (auto &I : Ret->Index) {
1417 if (I.second.SummaryList.size() > 1)
1418 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1420 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1421 const auto &Prevailing = PrevailingCopy.find(GUID);
1422 if (Prevailing == PrevailingCopy.end())
1424 return Prevailing->second == S;
1426 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1427 GlobalValue::GUID GUID,
1428 GlobalValue::LinkageTypes NewLinkage) {
1429 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1432 // Uses FromPrevailing visibility scheme which works for many binary
1433 // formats. We probably could and should use ELF visibility scheme for many of
1434 // our targets, however.
1436 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1437 Ret->GUIDPreservedSymbols);
1439 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1440 // callback below. This callback below will dictate the linkage for all
1441 // summaries in the index, and we basically just only want to ensure that dead
1442 // symbols are internalized. Otherwise everything that's already external
1443 // linkage will stay as external, and internal will stay as internal.
1444 std::set<GlobalValue::GUID> ExportedGUIDs;
1445 for (auto &List : Ret->Index) {
1446 for (auto &GVS: List.second.SummaryList) {
1447 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1449 auto GUID = GVS->getOriginalName();
1450 if (GVS->flags().Live)
1451 ExportedGUIDs.insert(GUID);
1454 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1455 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1456 return (ExportList != Ret->ExportLists.end() &&
1457 ExportList->second.count(VI)) ||
1458 ExportedGUIDs.count(VI.getGUID());
1460 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1462 return Ret.release();
1466 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1470 // Below are the various passes that happen *per module* when doing ThinLTO.
1472 // In other words, these are the functions that are all run concurrently
1473 // with one another, one per module. The passes here correspond to the analysis
1474 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1475 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1476 // so rustc can save off the intermediate bytecode between each step.
1479 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1480 // When linking an ELF shared object, dso_local should be dropped. We
1481 // conservatively do this for -fpic.
1482 bool ClearDSOLocalOnDeclarations =
1483 TM.getTargetTriple().isOSBinFormatELF() &&
1484 TM.getRelocationModel() != Reloc::Static &&
1485 Mod.getPIELevel() == PIELevel::Default;
1486 return ClearDSOLocalOnDeclarations;
1490 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1491 LLVMTargetMachineRef TM) {
1492 Module &Mod = *unwrap(M);
1493 TargetMachine &Target = *unwrap(TM);
1495 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1496 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1499 LLVMRustSetLastError("renameModuleForThinLTO failed");
1506 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1507 Module &Mod = *unwrap(M);
1508 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1509 #if LLVM_VERSION_GE(14, 0)
1510 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1512 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1518 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1519 Module &Mod = *unwrap(M);
1520 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1521 thinLTOInternalizeModule(Mod, DefinedGlobals);
1526 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1527 LLVMTargetMachineRef TM) {
1528 Module &Mod = *unwrap(M);
1529 TargetMachine &Target = *unwrap(TM);
1531 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1532 auto Loader = [&](StringRef Identifier) {
1533 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1534 auto &Context = Mod.getContext();
1535 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1540 // The rest of this closure is a workaround for
1541 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1542 // we accidentally import wasm custom sections into different modules,
1543 // duplicating them by in the final output artifact.
1545 // The issue is worked around here by manually removing the
1546 // `wasm.custom_sections` named metadata node from any imported module. This
1547 // we know isn't used by any optimization pass so there's no need for it to
1550 // Note that the metadata is currently lazily loaded, so we materialize it
1551 // here before looking up if there's metadata inside. The `FunctionImporter`
1552 // will immediately materialize metadata anyway after an import, so this
1553 // shouldn't be a perf hit.
1554 if (Error Err = (*MOrErr)->materializeMetadata()) {
1555 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1559 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1560 if (WasmCustomSections)
1561 WasmCustomSections->eraseFromParent();
1565 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1566 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1567 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1569 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1575 // This struct and various functions are sort of a hack right now, but the
1576 // problem is that we've got in-memory LLVM modules after we generate and
1577 // optimize all codegen-units for one compilation in rustc. To be compatible
1578 // with the LTO support above we need to serialize the modules plus their
1579 // ThinLTO summary into memory.
1581 // This structure is basically an owned version of a serialize module, with
1582 // a ThinLTO summary attached.
1583 struct LLVMRustThinLTOBuffer {
1587 extern "C" LLVMRustThinLTOBuffer*
1588 LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) {
1589 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1591 raw_string_ostream OS(Ret->data);
1594 #if LLVM_VERSION_LT(15, 0)
1595 legacy::PassManager PM;
1596 PM.add(createWriteThinLTOBitcodePass(OS));
1600 LoopAnalysisManager LAM;
1601 FunctionAnalysisManager FAM;
1602 CGSCCAnalysisManager CGAM;
1603 ModuleAnalysisManager MAM;
1604 PB.registerModuleAnalyses(MAM);
1605 PB.registerCGSCCAnalyses(CGAM);
1606 PB.registerFunctionAnalyses(FAM);
1607 PB.registerLoopAnalyses(LAM);
1608 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1609 ModulePassManager MPM;
1610 MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr));
1611 MPM.run(*unwrap(M), MAM);
1614 legacy::PassManager PM;
1615 PM.add(createBitcodeWriterPass(OS));
1620 return Ret.release();
1624 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1628 extern "C" const void*
1629 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1630 return Buffer->data.data();
1634 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1635 return Buffer->data.length();
1638 // This is what we used to parse upstream bitcode for actual ThinLTO
1639 // processing. We'll call this once per module optimized through ThinLTO, and
1640 // it'll be called concurrently on many threads.
1641 extern "C" LLVMModuleRef
1642 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1645 const char *identifier) {
1646 StringRef Data(data, len);
1647 MemoryBufferRef Buffer(Data, identifier);
1648 unwrap(Context)->enableDebugTypeODRUniquing();
1649 Expected<std::unique_ptr<Module>> SrcOrError =
1650 parseBitcodeFile(Buffer, *unwrap(Context));
1652 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1655 return wrap(std::move(*SrcOrError).release());
1658 // Find the bitcode section in the object file data and return it as a slice.
1659 // Fail if the bitcode section is present but empty.
1661 // On success, the return value is the pointer to the start of the slice and
1662 // `out_len` is filled with the (non-zero) length. On failure, the return value
1663 // is `nullptr` and `out_len` is set to zero.
1664 extern "C" const char*
1665 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1670 StringRef Data(data, len);
1671 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1673 Expected<MemoryBufferRef> BitcodeOrError =
1674 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1675 if (!BitcodeOrError) {
1676 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1680 *out_len = BitcodeOrError->getBufferSize();
1681 return BitcodeOrError->getBufferStart();
1684 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1685 // the comment in `back/lto.rs` for why this exists.
1687 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1689 DICompileUnit **B) {
1690 Module *M = unwrap(Mod);
1691 DICompileUnit **Cur = A;
1692 DICompileUnit **Next = B;
1693 for (DICompileUnit *CU : M->debug_compile_units()) {
1702 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1703 // the comment in `back/lto.rs` for why this exists.
1705 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1706 Module *M = unwrap(Mod);
1708 // If the original source module didn't have a `DICompileUnit` then try to
1709 // merge all the existing compile units. If there aren't actually any though
1710 // then there's not much for us to do so return.
1711 if (Unit == nullptr) {
1712 for (DICompileUnit *CU : M->debug_compile_units()) {
1716 if (Unit == nullptr)
1720 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1721 // process it recursively. Note that we used to specifically iterate over
1722 // instructions to ensure we feed everything into it, but `processModule`
1723 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1724 DebugInfoFinder Finder;
1725 Finder.processModule(*M);
1727 // After we've found all our debuginfo, rewrite all subprograms to point to
1728 // the same `DICompileUnit`.
1729 for (auto &F : Finder.subprograms()) {
1730 F->replaceUnit(Unit);
1733 // Erase any other references to other `DICompileUnit` instances, the verifier
1734 // will later ensure that we don't actually have any other stale references to
1736 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1737 MD->clearOperands();
1738 MD->addOperand(Unit);
1741 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1742 // storing the result in 'KeyOut'.
1743 // Currently, this cache key is a SHA-1 hash of anything that could affect
1744 // the result of optimizing this module (e.g. module imports, exports, liveness
1745 // of access globals, etc).
1746 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1747 // used during the normal linker-plugin incremental thin-LTO process.
1749 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1750 SmallString<40> Key;
1751 llvm::lto::Config conf;
1752 const auto &ImportList = Data->ImportLists.lookup(ModId);
1753 const auto &ExportList = Data->ExportLists.lookup(ModId);
1754 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1755 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1756 std::set<GlobalValue::GUID> CfiFunctionDefs;
1757 std::set<GlobalValue::GUID> CfiFunctionDecls;
1759 // Based on the 'InProcessThinBackend' constructor in LLVM
1760 for (auto &Name : Data->Index.cfiFunctionDefs())
1761 CfiFunctionDefs.insert(
1762 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1763 for (auto &Name : Data->Index.cfiFunctionDecls())
1764 CfiFunctionDecls.insert(
1765 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1767 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1768 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1771 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());