6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/PassPlugin.h"
21 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #if LLVM_VERSION_LT(14, 0)
26 #include "llvm/Support/TargetRegistry.h"
28 #include "llvm/MC/TargetRegistry.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
32 #include "llvm/Transforms/IPO/AlwaysInliner.h"
33 #include "llvm/Transforms/IPO/FunctionImport.h"
34 #if LLVM_VERSION_GE(15, 0)
35 #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
37 #include "llvm/Transforms/Utils/AddDiscriminators.h"
38 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
39 #include "llvm/LTO/LTO.h"
40 #include "llvm/Bitcode/BitcodeWriterPass.h"
41 #include "llvm-c/Transforms/PassManagerBuilder.h"
43 #include "llvm/Transforms/Instrumentation.h"
44 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
45 #include "llvm/Support/TimeProfiler.h"
46 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
47 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
48 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
49 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
50 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
51 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
52 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
53 #include "llvm/Transforms/Utils.h"
57 typedef struct LLVMOpaquePass *LLVMPassRef;
58 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
60 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
61 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
63 extern "C" void LLVMInitializePasses() {
64 PassRegistry &Registry = *PassRegistry::getPassRegistry();
65 initializeCore(Registry);
66 initializeCodeGen(Registry);
67 initializeScalarOpts(Registry);
68 initializeVectorization(Registry);
69 initializeIPO(Registry);
70 initializeAnalysis(Registry);
71 initializeTransformUtils(Registry);
72 initializeInstCombine(Registry);
73 initializeInstrumentation(Registry);
74 initializeTarget(Registry);
77 extern "C" void LLVMTimeTraceProfilerInitialize() {
78 timeTraceProfilerInitialize(
79 /* TimeTraceGranularity */ 0,
80 /* ProcName */ "rustc");
83 extern "C" void LLVMTimeTraceProfilerFinishThread() {
84 timeTraceProfilerFinishThread();
87 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
88 StringRef FN(FileName);
90 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
92 timeTraceProfilerWrite(OS);
93 timeTraceProfilerCleanup();
96 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
97 #if LLVM_VERSION_LT(15, 0)
98 StringRef SR(PassName);
99 PassRegistry *PR = PassRegistry::getPassRegistry();
101 const PassInfo *PI = PR->getPassInfo(SR);
103 return wrap(PI->createPass());
107 report_fatal_error("Legacy PM not supported with LLVM 15");
111 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
112 #if LLVM_VERSION_LT(15, 0)
113 const bool CompileKernel = false;
114 const bool UseAfterScope = true;
116 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
118 report_fatal_error("Legacy PM not supported with LLVM 15");
122 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
123 #if LLVM_VERSION_LT(15, 0)
124 const bool CompileKernel = false;
126 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
128 report_fatal_error("Legacy PM not supported with LLVM 15");
132 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
133 #if LLVM_VERSION_LT(15, 0)
134 const bool CompileKernel = false;
136 return wrap(createMemorySanitizerLegacyPassPass(
137 #if LLVM_VERSION_GE(14, 0)
138 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel, /*EagerChecks=*/true}
140 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}
144 report_fatal_error("Legacy PM not supported with LLVM 15");
148 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
149 #if LLVM_VERSION_LT(15, 0)
150 return wrap(createThreadSanitizerLegacyPassPass());
152 report_fatal_error("Legacy PM not supported with LLVM 15");
156 extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) {
157 #if LLVM_VERSION_LT(15, 0)
158 const bool CompileKernel = false;
160 return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover));
162 report_fatal_error("Legacy PM not supported with LLVM 15");
166 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
167 #if LLVM_VERSION_LT(15, 0)
169 Pass *Pass = unwrap(RustPass);
170 PassManagerBase *PMB = unwrap(PMR);
173 report_fatal_error("Legacy PM not supported with LLVM 15");
177 extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() {
178 #if LLVM_VERSION_LT(15, 0)
179 return LLVMPassManagerBuilderCreate();
181 report_fatal_error("Legacy PM not supported with LLVM 15");
185 extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) {
186 #if LLVM_VERSION_LT(15, 0)
187 LLVMPassManagerBuilderDispose(PMB);
189 report_fatal_error("Legacy PM not supported with LLVM 15");
193 extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager(
194 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) {
195 #if LLVM_VERSION_LT(15, 0)
196 LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM);
198 report_fatal_error("Legacy PM not supported with LLVM 15");
202 extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager(
203 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) {
204 #if LLVM_VERSION_LT(15, 0)
205 LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM);
207 report_fatal_error("Legacy PM not supported with LLVM 15");
211 extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager(
212 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) {
213 #if LLVM_VERSION_LT(15, 0)
214 LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner);
216 report_fatal_error("Legacy PM not supported with LLVM 15");
221 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
222 LLVMPassManagerBuilderRef PMBR,
223 LLVMPassManagerRef PMR
225 #if LLVM_VERSION_LT(15, 0)
226 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
228 report_fatal_error("Legacy PM not supported with LLVM 15");
232 extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold(
233 LLVMPassManagerBuilderRef PMB, unsigned Threshold) {
234 #if LLVM_VERSION_LT(15, 0)
235 LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold);
237 report_fatal_error("Legacy PM not supported with LLVM 15");
242 void LLVMRustAddLastExtensionPasses(
243 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
244 #if LLVM_VERSION_LT(15, 0)
245 auto AddExtensionPasses = [Passes, NumPasses](
246 const PassManagerBuilder &Builder, PassManagerBase &PM) {
247 for (size_t I = 0; I < NumPasses; I++) {
248 PM.add(unwrap(Passes[I]));
251 // Add the passes to both of the pre-finalization extension points,
252 // so they are run for optimized and non-optimized builds.
253 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
255 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
258 report_fatal_error("Legacy PM not supported with LLVM 15");
262 #ifdef LLVM_COMPONENT_X86
263 #define SUBTARGET_X86 SUBTARGET(X86)
265 #define SUBTARGET_X86
268 #ifdef LLVM_COMPONENT_ARM
269 #define SUBTARGET_ARM SUBTARGET(ARM)
271 #define SUBTARGET_ARM
274 #ifdef LLVM_COMPONENT_AARCH64
275 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
277 #define SUBTARGET_AARCH64
280 #ifdef LLVM_COMPONENT_AVR
281 #define SUBTARGET_AVR SUBTARGET(AVR)
283 #define SUBTARGET_AVR
286 #ifdef LLVM_COMPONENT_M68k
287 #define SUBTARGET_M68K SUBTARGET(M68k)
289 #define SUBTARGET_M68K
292 #ifdef LLVM_COMPONENT_MIPS
293 #define SUBTARGET_MIPS SUBTARGET(Mips)
295 #define SUBTARGET_MIPS
298 #ifdef LLVM_COMPONENT_POWERPC
299 #define SUBTARGET_PPC SUBTARGET(PPC)
301 #define SUBTARGET_PPC
304 #ifdef LLVM_COMPONENT_SYSTEMZ
305 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
307 #define SUBTARGET_SYSTEMZ
310 #ifdef LLVM_COMPONENT_MSP430
311 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
313 #define SUBTARGET_MSP430
316 #ifdef LLVM_COMPONENT_RISCV
317 #define SUBTARGET_RISCV SUBTARGET(RISCV)
319 #define SUBTARGET_RISCV
322 #ifdef LLVM_COMPONENT_SPARC
323 #define SUBTARGET_SPARC SUBTARGET(Sparc)
325 #define SUBTARGET_SPARC
328 #ifdef LLVM_COMPONENT_HEXAGON
329 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
331 #define SUBTARGET_HEXAGON
334 #define GEN_SUBTARGETS \
348 #define SUBTARGET(x) \
350 extern const SubtargetFeatureKV x##FeatureKV[]; \
351 extern const SubtargetFeatureKV x##SubTypeKV[]; \
357 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
358 const char *Feature) {
359 TargetMachine *Target = unwrap(TM);
360 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
361 return MCInfo->checkFeatures(std::string("+") + Feature);
364 enum class LLVMRustCodeModel {
373 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
375 case LLVMRustCodeModel::Tiny:
376 return CodeModel::Tiny;
377 case LLVMRustCodeModel::Small:
378 return CodeModel::Small;
379 case LLVMRustCodeModel::Kernel:
380 return CodeModel::Kernel;
381 case LLVMRustCodeModel::Medium:
382 return CodeModel::Medium;
383 case LLVMRustCodeModel::Large:
384 return CodeModel::Large;
385 case LLVMRustCodeModel::None:
388 report_fatal_error("Bad CodeModel.");
392 enum class LLVMRustCodeGenOptLevel {
399 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
401 case LLVMRustCodeGenOptLevel::None:
402 return CodeGenOpt::None;
403 case LLVMRustCodeGenOptLevel::Less:
404 return CodeGenOpt::Less;
405 case LLVMRustCodeGenOptLevel::Default:
406 return CodeGenOpt::Default;
407 case LLVMRustCodeGenOptLevel::Aggressive:
408 return CodeGenOpt::Aggressive;
410 report_fatal_error("Bad CodeGenOptLevel.");
414 enum class LLVMRustPassBuilderOptLevel {
423 #if LLVM_VERSION_LT(14,0)
424 using OptimizationLevel = PassBuilder::OptimizationLevel;
427 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
429 case LLVMRustPassBuilderOptLevel::O0:
430 return OptimizationLevel::O0;
431 case LLVMRustPassBuilderOptLevel::O1:
432 return OptimizationLevel::O1;
433 case LLVMRustPassBuilderOptLevel::O2:
434 return OptimizationLevel::O2;
435 case LLVMRustPassBuilderOptLevel::O3:
436 return OptimizationLevel::O3;
437 case LLVMRustPassBuilderOptLevel::Os:
438 return OptimizationLevel::Os;
439 case LLVMRustPassBuilderOptLevel::Oz:
440 return OptimizationLevel::Oz;
442 report_fatal_error("Bad PassBuilderOptLevel.");
446 enum class LLVMRustRelocModel {
455 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
457 case LLVMRustRelocModel::Static:
458 return Reloc::Static;
459 case LLVMRustRelocModel::PIC:
461 case LLVMRustRelocModel::DynamicNoPic:
462 return Reloc::DynamicNoPIC;
463 case LLVMRustRelocModel::ROPI:
465 case LLVMRustRelocModel::RWPI:
467 case LLVMRustRelocModel::ROPIRWPI:
468 return Reloc::ROPI_RWPI;
470 report_fatal_error("Bad RelocModel.");
474 /// getLongestEntryLength - Return the length of the longest entry in the table.
475 template<typename KV>
476 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
478 for (auto &I : Table)
479 MaxLen = std::max(MaxLen, std::strlen(I.Key));
483 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
484 const TargetMachine *Target = unwrap(TM);
485 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
486 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
487 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
488 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
489 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
491 printf("Available CPUs for this target:\n");
492 if (HostArch == TargetArch) {
493 const StringRef HostCPU = sys::getHostCPUName();
494 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
495 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
497 for (auto &CPU : CPUTable)
498 printf(" %-*s\n", MaxCPULen, CPU.Key);
502 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
503 const TargetMachine *Target = unwrap(TM);
504 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
505 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
506 return FeatTable.size();
509 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
510 const char** Feature, const char** Desc) {
511 const TargetMachine *Target = unwrap(TM);
512 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
513 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
514 const SubtargetFeatureKV Feat = FeatTable[Index];
521 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
522 printf("Target CPU help is not supported by this LLVM version.\n\n");
525 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
529 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
532 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
533 StringRef Name = sys::getHostCPUName();
538 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
539 const char *TripleStr, const char *CPU, const char *Feature,
540 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
541 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
542 bool FunctionSections,
544 bool UniqueSectionNames,
545 bool TrapUnreachable,
548 bool EmitStackSizeSection,
549 bool RelaxELFRelocations,
551 const char *SplitDwarfFile) {
553 auto OptLevel = fromRust(RustOptLevel);
554 auto RM = fromRust(RustReloc);
555 auto CM = fromRust(RustCM);
558 Triple Trip(Triple::normalize(TripleStr));
559 const llvm::Target *TheTarget =
560 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
561 if (TheTarget == nullptr) {
562 LLVMRustSetLastError(Error.c_str());
566 TargetOptions Options;
568 Options.FloatABIType = FloatABI::Default;
570 Options.FloatABIType = FloatABI::Soft;
572 Options.DataSections = DataSections;
573 Options.FunctionSections = FunctionSections;
574 Options.UniqueSectionNames = UniqueSectionNames;
575 Options.MCOptions.AsmVerbose = AsmComments;
576 Options.MCOptions.PreserveAsmComments = AsmComments;
577 Options.MCOptions.ABIName = ABIStr;
578 if (SplitDwarfFile) {
579 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
581 Options.RelaxELFRelocations = RelaxELFRelocations;
582 Options.UseInitArray = UseInitArray;
584 if (TrapUnreachable) {
585 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
586 // This limits the extent of possible undefined behavior in some cases, as
587 // it prevents control flow from "falling through" into whatever code
588 // happens to be laid out next in memory.
589 Options.TrapUnreachable = true;
593 Options.ThreadModel = ThreadModel::Single;
596 Options.EmitStackSizeSection = EmitStackSizeSection;
598 TargetMachine *TM = TheTarget->createTargetMachine(
599 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
603 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
607 extern "C" void LLVMRustConfigurePassManagerBuilder(
608 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
609 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
610 const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath,
612 #if LLVM_VERSION_LT(15, 0)
613 unwrap(PMBR)->MergeFunctions = MergeFunctions;
614 unwrap(PMBR)->SLPVectorize = SLPVectorize;
615 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
616 unwrap(PMBR)->LoopVectorize = LoopVectorize;
617 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
618 unwrap(PMBR)->SizeLevel = SizeLevel;
619 unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0;
622 assert(!PGOUsePath && !PGOSampleUsePath);
623 unwrap(PMBR)->EnablePGOInstrGen = true;
624 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
625 } else if (PGOUsePath) {
626 assert(!PGOSampleUsePath);
627 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
628 } else if (PGOSampleUsePath) {
629 unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath;
632 report_fatal_error("Legacy PM not supported with LLVM 15");
636 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
637 // field of a PassManagerBuilder, we expose our own method of doing so.
638 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
640 bool DisableSimplifyLibCalls) {
641 Triple TargetTriple(unwrap(M)->getTargetTriple());
642 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
643 if (DisableSimplifyLibCalls)
644 TLI->disableAllFunctions();
645 unwrap(PMBR)->LibraryInfo = TLI;
648 // Unfortunately, the LLVM C API doesn't provide a way to create the
649 // TargetLibraryInfo pass, so we use this method to do so.
650 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
651 bool DisableSimplifyLibCalls) {
652 Triple TargetTriple(unwrap(M)->getTargetTriple());
653 TargetLibraryInfoImpl TLII(TargetTriple);
654 if (DisableSimplifyLibCalls)
655 TLII.disableAllFunctions();
656 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
659 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
660 // all the functions in a module, so we do that manually here. You'll find
661 // similar code in clang's BackendUtil.cpp file.
662 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
664 llvm::legacy::FunctionPassManager *P =
665 unwrap<llvm::legacy::FunctionPassManager>(PMR);
666 P->doInitialization();
668 // Upgrade all calls to old intrinsics first.
669 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
670 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
672 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
674 if (!I->isDeclaration())
680 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
681 // Initializing the command-line options more than once is not allowed. So,
682 // check if they've already been initialized. (This could happen if we're
683 // being called from rustpkg, for example). If the arguments change, then
684 // that's just kinda unfortunate.
685 static bool Initialized = false;
689 cl::ParseCommandLineOptions(Argc, Argv);
692 enum class LLVMRustFileType {
697 static CodeGenFileType fromRust(LLVMRustFileType Type) {
699 case LLVMRustFileType::AssemblyFile:
700 return CGFT_AssemblyFile;
701 case LLVMRustFileType::ObjectFile:
702 return CGFT_ObjectFile;
704 report_fatal_error("Bad FileType.");
708 extern "C" LLVMRustResult
709 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
710 LLVMModuleRef M, const char *Path, const char *DwoPath,
711 LLVMRustFileType RustFileType) {
712 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
713 auto FileType = fromRust(RustFileType);
715 std::string ErrorInfo;
717 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
719 ErrorInfo = EC.message();
720 if (ErrorInfo != "") {
721 LLVMRustSetLastError(ErrorInfo.c_str());
722 return LLVMRustResult::Failure;
725 buffer_ostream BOS(OS);
727 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
730 ErrorInfo = EC.message();
731 if (ErrorInfo != "") {
732 LLVMRustSetLastError(ErrorInfo.c_str());
733 return LLVMRustResult::Failure;
735 buffer_ostream DBOS(DOS);
736 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
739 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
743 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
744 // stream (OS), so the only real safe place to delete this is here? Don't we
745 // wish this was written in Rust?
746 LLVMDisposePassManager(PMR);
747 return LLVMRustResult::Success;
750 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
751 const char*, // pass name
752 const char*); // IR name
753 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
755 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
756 if (any_isa<const Module *>(WrappedIr))
757 return any_cast<const Module *>(WrappedIr)->getName().str();
758 if (any_isa<const Function *>(WrappedIr))
759 return any_cast<const Function *>(WrappedIr)->getName().str();
760 if (any_isa<const Loop *>(WrappedIr))
761 return any_cast<const Loop *>(WrappedIr)->getName().str();
762 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
763 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
768 void LLVMSelfProfileInitializeCallbacks(
769 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
770 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
771 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
772 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
773 StringRef Pass, llvm::Any Ir) {
774 std::string PassName = Pass.str();
775 std::string IrName = LLVMRustwrappedIrGetName(Ir);
776 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
779 PIC.registerAfterPassCallback(
780 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
781 const PreservedAnalyses &Preserved) {
782 AfterPassCallback(LlvmSelfProfiler);
785 PIC.registerAfterPassInvalidatedCallback(
786 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
787 AfterPassCallback(LlvmSelfProfiler);
790 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
791 StringRef Pass, llvm::Any Ir) {
792 std::string PassName = Pass.str();
793 std::string IrName = LLVMRustwrappedIrGetName(Ir);
794 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
797 PIC.registerAfterAnalysisCallback(
798 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
799 AfterPassCallback(LlvmSelfProfiler);
803 enum class LLVMRustOptStage {
811 struct LLVMRustSanitizerOptions {
812 bool SanitizeAddress;
813 bool SanitizeAddressRecover;
815 bool SanitizeMemoryRecover;
816 int SanitizeMemoryTrackOrigins;
818 bool SanitizeHWAddress;
819 bool SanitizeHWAddressRecover;
822 extern "C" LLVMRustResult
823 LLVMRustOptimizeWithNewPassManager(
824 LLVMModuleRef ModuleRef,
825 LLVMTargetMachineRef TMRef,
826 LLVMRustPassBuilderOptLevel OptLevelRust,
827 LLVMRustOptStage OptStage,
828 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
829 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
830 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
831 LLVMRustSanitizerOptions *SanitizerOptions,
832 const char *PGOGenPath, const char *PGOUsePath,
833 bool InstrumentCoverage, const char *InstrProfileOutput,
835 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
836 void* LlvmSelfProfiler,
837 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
838 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
839 const char *ExtraPasses, size_t ExtraPassesLen,
840 const char *LLVMPlugins, size_t LLVMPluginsLen) {
841 Module *TheModule = unwrap(ModuleRef);
842 TargetMachine *TM = unwrap(TMRef);
843 OptimizationLevel OptLevel = fromRust(OptLevelRust);
846 PipelineTuningOptions PTO;
847 PTO.LoopUnrolling = UnrollLoops;
848 PTO.LoopInterleaving = UnrollLoops;
849 PTO.LoopVectorization = LoopVectorize;
850 PTO.SLPVectorization = SLPVectorize;
851 PTO.MergeFunctions = MergeFunctions;
853 // FIXME: We may want to expose this as an option.
854 bool DebugPassManager = false;
856 PassInstrumentationCallbacks PIC;
857 StandardInstrumentations SI(DebugPassManager);
858 SI.registerCallbacks(PIC);
860 if (LlvmSelfProfiler){
861 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
864 Optional<PGOOptions> PGOOpt;
866 assert(!PGOUsePath && !PGOSampleUsePath);
867 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
868 PGOOptions::NoCSAction, DebugInfoForProfiling);
869 } else if (PGOUsePath) {
870 assert(!PGOSampleUsePath);
871 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
872 PGOOptions::NoCSAction, DebugInfoForProfiling);
873 } else if (PGOSampleUsePath) {
874 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
875 PGOOptions::NoCSAction, DebugInfoForProfiling);
876 } else if (DebugInfoForProfiling) {
877 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
878 PGOOptions::NoCSAction, DebugInfoForProfiling);
881 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
882 LoopAnalysisManager LAM;
883 FunctionAnalysisManager FAM;
884 CGSCCAnalysisManager CGAM;
885 ModuleAnalysisManager MAM;
887 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
889 Triple TargetTriple(TheModule->getTargetTriple());
890 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
891 if (DisableSimplifyLibCalls)
892 TLII->disableAllFunctions();
893 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
895 PB.registerModuleAnalyses(MAM);
896 PB.registerCGSCCAnalyses(CGAM);
897 PB.registerFunctionAnalyses(FAM);
898 PB.registerLoopAnalyses(LAM);
899 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
901 // We manually collect pipeline callbacks so we can apply them at O0, where the
902 // PassBuilder does not create a pipeline.
903 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
904 PipelineStartEPCallbacks;
905 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
906 OptimizerLastEPCallbacks;
909 PipelineStartEPCallbacks.push_back(
910 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
911 MPM.addPass(VerifierPass());
916 if (InstrumentGCOV) {
917 PipelineStartEPCallbacks.push_back(
918 [](ModulePassManager &MPM, OptimizationLevel Level) {
919 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
924 if (InstrumentCoverage) {
925 PipelineStartEPCallbacks.push_back(
926 [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) {
927 InstrProfOptions Options;
928 if (InstrProfileOutput) {
929 Options.InstrProfileOutput = InstrProfileOutput;
931 MPM.addPass(InstrProfiling(Options, false));
936 if (SanitizerOptions) {
937 if (SanitizerOptions->SanitizeMemory) {
938 #if LLVM_VERSION_GE(14, 0)
939 MemorySanitizerOptions Options(
940 SanitizerOptions->SanitizeMemoryTrackOrigins,
941 SanitizerOptions->SanitizeMemoryRecover,
942 /*CompileKernel=*/false,
943 /*EagerChecks=*/true);
945 MemorySanitizerOptions Options(
946 SanitizerOptions->SanitizeMemoryTrackOrigins,
947 SanitizerOptions->SanitizeMemoryRecover,
948 /*CompileKernel=*/false);
950 OptimizerLastEPCallbacks.push_back(
951 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
952 #if LLVM_VERSION_GE(14, 0) && LLVM_VERSION_LT(16, 0)
953 MPM.addPass(ModuleMemorySanitizerPass(Options));
955 MPM.addPass(MemorySanitizerPass(Options));
957 #if LLVM_VERSION_LT(16, 0)
958 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
964 if (SanitizerOptions->SanitizeThread) {
965 OptimizerLastEPCallbacks.push_back(
966 [](ModulePassManager &MPM, OptimizationLevel Level) {
967 #if LLVM_VERSION_GE(14, 0)
968 MPM.addPass(ModuleThreadSanitizerPass());
970 MPM.addPass(ThreadSanitizerPass());
972 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
977 if (SanitizerOptions->SanitizeAddress) {
978 OptimizerLastEPCallbacks.push_back(
979 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
980 #if LLVM_VERSION_LT(15, 0)
981 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
983 #if LLVM_VERSION_GE(14, 0)
984 AddressSanitizerOptions opts = AddressSanitizerOptions{
985 /*CompileKernel=*/false,
986 SanitizerOptions->SanitizeAddressRecover,
987 /*UseAfterScope=*/true,
988 AsanDetectStackUseAfterReturnMode::Runtime,
990 #if LLVM_VERSION_LT(16, 0)
991 MPM.addPass(ModuleAddressSanitizerPass(opts));
993 MPM.addPass(AddressSanitizerPass(opts));
996 MPM.addPass(ModuleAddressSanitizerPass(
997 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
998 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
999 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
1000 /*UseAfterScope=*/true)));
1005 if (SanitizerOptions->SanitizeHWAddress) {
1006 OptimizerLastEPCallbacks.push_back(
1007 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
1008 #if LLVM_VERSION_GE(14, 0)
1009 HWAddressSanitizerOptions opts(
1010 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
1011 /*DisableOptimization=*/false);
1012 MPM.addPass(HWAddressSanitizerPass(opts));
1014 MPM.addPass(HWAddressSanitizerPass(
1015 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
1022 if (LLVMPluginsLen) {
1023 auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
1024 SmallVector<StringRef> Plugins;
1025 PluginsStr.split(Plugins, ',', -1, false);
1026 for (auto PluginPath: Plugins) {
1027 auto Plugin = PassPlugin::Load(PluginPath.str());
1029 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
1032 Plugin->registerPassBuilderCallbacks(PB);
1036 ModulePassManager MPM;
1037 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
1038 if (!NoPrepopulatePasses) {
1039 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
1040 // At the same time, the LTO pipelines do support O0 and using them is required.
1041 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
1042 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
1043 for (const auto &C : PipelineStartEPCallbacks)
1044 PB.registerPipelineStartEPCallback(C);
1045 for (const auto &C : OptimizerLastEPCallbacks)
1046 PB.registerOptimizerLastEPCallback(C);
1048 // Pass false as we manually schedule ThinLTOBufferPasses below.
1049 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
1051 for (const auto &C : PipelineStartEPCallbacks)
1052 PB.registerPipelineStartEPCallback(C);
1053 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
1054 for (const auto &C : OptimizerLastEPCallbacks)
1055 PB.registerOptimizerLastEPCallback(C);
1059 case LLVMRustOptStage::PreLinkNoLTO:
1060 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
1062 case LLVMRustOptStage::PreLinkThinLTO:
1063 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
1064 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
1065 // passes may still run afterwards. This means we need to run the buffer passes again.
1066 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
1067 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
1068 if (OptimizerLastEPCallbacks.empty())
1069 NeedThinLTOBufferPasses = false;
1070 for (const auto &C : OptimizerLastEPCallbacks)
1073 case LLVMRustOptStage::PreLinkFatLTO:
1074 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
1075 NeedThinLTOBufferPasses = false;
1077 case LLVMRustOptStage::ThinLTO:
1078 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
1079 // It only seems to be needed for C++ specific optimizations.
1080 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
1082 case LLVMRustOptStage::FatLTO:
1083 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
1088 // We're not building any of the default pipelines but we still want to
1089 // add the verifier, instrumentation, etc passes if they were requested
1090 for (const auto &C : PipelineStartEPCallbacks)
1092 for (const auto &C : OptimizerLastEPCallbacks)
1096 if (ExtraPassesLen) {
1097 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
1098 std::string ErrMsg = toString(std::move(Err));
1099 LLVMRustSetLastError(ErrMsg.c_str());
1100 return LLVMRustResult::Failure;
1104 if (NeedThinLTOBufferPasses) {
1105 MPM.addPass(CanonicalizeAliasesPass());
1106 MPM.addPass(NameAnonGlobalPass());
1109 // Upgrade all calls to old intrinsics first.
1110 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
1111 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
1113 MPM.run(*TheModule, MAM);
1114 return LLVMRustResult::Success;
1117 // Callback to demangle function name
1119 // * name to be demangled
1122 // * output buffer len
1123 // Returns len of demangled string, or 0 if demangle failed.
1124 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1129 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1130 DemangleFn Demangle;
1131 std::vector<char> Buf;
1134 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1136 // Return empty string if demangle failed
1137 // or if name does not need to be demangled
1138 StringRef CallDemangle(StringRef name) {
1143 if (Buf.size() < name.size() * 2) {
1144 // Semangled name usually shorter than mangled,
1145 // but allocate twice as much memory just in case
1146 Buf.resize(name.size() * 2);
1149 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1155 auto Demangled = StringRef(Buf.data(), R);
1156 if (Demangled == name) {
1157 // Do not print anything if demangled name is equal to mangled.
1164 void emitFunctionAnnot(const Function *F,
1165 formatted_raw_ostream &OS) override {
1166 StringRef Demangled = CallDemangle(F->getName());
1167 if (Demangled.empty()) {
1171 OS << "; " << Demangled << "\n";
1174 void emitInstructionAnnot(const Instruction *I,
1175 formatted_raw_ostream &OS) override {
1178 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1180 Value = CI->getCalledOperand();
1181 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1183 Value = II->getCalledOperand();
1185 // Could demangle more operations, e. g.
1186 // `store %place, @function`.
1190 if (!Value->hasName()) {
1194 StringRef Demangled = CallDemangle(Value->getName());
1195 if (Demangled.empty()) {
1199 OS << "; " << Name << " " << Demangled << "\n";
1205 extern "C" LLVMRustResult
1206 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1207 std::string ErrorInfo;
1209 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
1211 ErrorInfo = EC.message();
1212 if (ErrorInfo != "") {
1213 LLVMRustSetLastError(ErrorInfo.c_str());
1214 return LLVMRustResult::Failure;
1217 RustAssemblyAnnotationWriter AAW(Demangle);
1218 formatted_raw_ostream FOS(OS);
1219 unwrap(M)->print(FOS, &AAW);
1221 return LLVMRustResult::Success;
1224 extern "C" void LLVMRustPrintPasses() {
1225 LLVMInitializePasses();
1226 struct MyListener : PassRegistrationListener {
1227 void passEnumerate(const PassInfo *Info) {
1228 StringRef PassArg = Info->getPassArgument();
1229 StringRef PassName = Info->getPassName();
1230 if (!PassArg.empty()) {
1231 // These unsigned->signed casts could theoretically overflow, but
1232 // realistically never will (and even if, the result is implementation
1233 // defined rather plain UB).
1234 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1235 (int)PassName.size(), PassName.data());
1240 PassRegistry *PR = PassRegistry::getPassRegistry();
1241 PR->enumerateWith(&Listener);
1244 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1245 bool AddLifetimes) {
1246 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1249 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1251 llvm::legacy::PassManager passes;
1253 auto PreserveFunctions = [=](const GlobalValue &GV) {
1254 for (size_t I = 0; I < Len; I++) {
1255 if (GV.getName() == Symbols[I]) {
1262 passes.add(llvm::createInternalizePass(PreserveFunctions));
1264 passes.run(*unwrap(M));
1268 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1269 LLVMTargetMachineRef TMR) {
1270 TargetMachine *Target = unwrap(TMR);
1271 unwrap(Module)->setDataLayout(Target->createDataLayout());
1274 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1275 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1278 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1279 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1282 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1283 LLVMRustCodeModel Model) {
1284 auto CM = fromRust(Model);
1287 unwrap(M)->setCodeModel(*CM);
1290 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1291 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1292 // LLVM, and otherwise it's just blanket rejected from other compilers.
1294 // Most of this implementation is straight copied from LLVM. At the time of
1295 // this writing it wasn't *quite* suitable to reuse more code from upstream
1296 // for our purposes, but we should strive to upstream this support once it's
1297 // ready to go! I figure we may want a bit of testing locally first before
1298 // sending this upstream to LLVM. I hear though they're quite eager to receive
1299 // feedback like this!
1301 // If you're reading this code and wondering "what in the world" or you're
1302 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1303 // then fear not! (ok maybe fear a little). All code here is mostly based
1304 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1306 // You'll find that the general layout here roughly corresponds to the `run`
1307 // method in that file as well as `ProcessThinLTOModule`. Functions are
1308 // specifically commented below as well, but if you're updating this code
1309 // or otherwise trying to understand it, the LLVM source will be useful in
1310 // interpreting the mysteries within.
1312 // Otherwise I'll apologize in advance, it probably requires a relatively
1313 // significant investment on your part to "truly understand" what's going on
1314 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1315 // and various online resources about ThinLTO to make heads or tails of all
1318 // This is a shared data structure which *must* be threadsafe to share
1319 // read-only amongst threads. This also corresponds basically to the arguments
1320 // of the `ProcessThinLTOModule` function in the LLVM source.
1321 struct LLVMRustThinLTOData {
1322 // The combined index that is the global analysis over all modules we're
1323 // performing ThinLTO for. This is mostly managed by LLVM.
1324 ModuleSummaryIndex Index;
1326 // All modules we may look at, stored as in-memory serialized versions. This
1327 // is later used when inlining to ensure we can extract any module to inline
1329 StringMap<MemoryBufferRef> ModuleMap;
1331 // A set that we manage of everything we *don't* want internalized. Note that
1332 // this includes all transitive references right now as well, but it may not
1334 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1336 // Not 100% sure what these are, but they impact what's internalized and
1337 // what's inlined across modules, I believe.
1338 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1339 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1340 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1341 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1343 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1346 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1347 struct LLVMRustThinLTOModule {
1348 const char *identifier;
1353 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1355 static const GlobalValueSummary *
1356 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1357 auto StrongDefForLinker = llvm::find_if(
1358 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1359 auto Linkage = Summary->linkage();
1360 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1361 !GlobalValue::isWeakForLinker(Linkage);
1363 if (StrongDefForLinker != GVSummaryList.end())
1364 return StrongDefForLinker->get();
1366 auto FirstDefForLinker = llvm::find_if(
1367 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1368 auto Linkage = Summary->linkage();
1369 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1371 if (FirstDefForLinker == GVSummaryList.end())
1373 return FirstDefForLinker->get();
1376 // The main entry point for creating the global ThinLTO analysis. The structure
1377 // here is basically the same as before threads are spawned in the `run`
1378 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1379 extern "C" LLVMRustThinLTOData*
1380 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1382 const char **preserved_symbols,
1384 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1386 // Load each module's summary and merge it into one combined index
1387 for (int i = 0; i < num_modules; i++) {
1388 auto module = &modules[i];
1389 StringRef buffer(module->data, module->len);
1390 MemoryBufferRef mem_buffer(buffer, module->identifier);
1392 Ret->ModuleMap[module->identifier] = mem_buffer;
1394 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1395 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1400 // Collect for each module the list of function it defines (GUID -> Summary)
1401 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1403 // Convert the preserved symbols set from string to GUID, this is then needed
1404 // for internalization.
1405 for (int i = 0; i < num_symbols; i++) {
1406 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1407 Ret->GUIDPreservedSymbols.insert(GUID);
1410 // Collect the import/export lists for all modules from the call-graph in the
1413 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1414 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1415 return PrevailingType::Unknown;
1417 // We don't have a complete picture in our use of ThinLTO, just our immediate
1418 // crate, so we need `ImportEnabled = false` to limit internalization.
1419 // Otherwise, we sometimes lose `static` values -- see #60184.
1420 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1421 deadIsPrevailing, /* ImportEnabled = */ false);
1422 ComputeCrossModuleImport(
1424 Ret->ModuleToDefinedGVSummaries,
1429 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1430 // impacts the caching.
1432 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1433 // being lifted from `lib/LTO/LTO.cpp` as well
1434 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1435 for (auto &I : Ret->Index) {
1436 if (I.second.SummaryList.size() > 1)
1437 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1439 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1440 const auto &Prevailing = PrevailingCopy.find(GUID);
1441 if (Prevailing == PrevailingCopy.end())
1443 return Prevailing->second == S;
1445 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1446 GlobalValue::GUID GUID,
1447 GlobalValue::LinkageTypes NewLinkage) {
1448 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1451 // Uses FromPrevailing visibility scheme which works for many binary
1452 // formats. We probably could and should use ELF visibility scheme for many of
1453 // our targets, however.
1455 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1456 Ret->GUIDPreservedSymbols);
1458 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1459 // callback below. This callback below will dictate the linkage for all
1460 // summaries in the index, and we basically just only want to ensure that dead
1461 // symbols are internalized. Otherwise everything that's already external
1462 // linkage will stay as external, and internal will stay as internal.
1463 std::set<GlobalValue::GUID> ExportedGUIDs;
1464 for (auto &List : Ret->Index) {
1465 for (auto &GVS: List.second.SummaryList) {
1466 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1468 auto GUID = GVS->getOriginalName();
1469 if (GVS->flags().Live)
1470 ExportedGUIDs.insert(GUID);
1473 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1474 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1475 return (ExportList != Ret->ExportLists.end() &&
1476 ExportList->second.count(VI)) ||
1477 ExportedGUIDs.count(VI.getGUID());
1479 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1481 return Ret.release();
1485 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1489 // Below are the various passes that happen *per module* when doing ThinLTO.
1491 // In other words, these are the functions that are all run concurrently
1492 // with one another, one per module. The passes here correspond to the analysis
1493 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1494 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1495 // so rustc can save off the intermediate bytecode between each step.
1498 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1499 // When linking an ELF shared object, dso_local should be dropped. We
1500 // conservatively do this for -fpic.
1501 bool ClearDSOLocalOnDeclarations =
1502 TM.getTargetTriple().isOSBinFormatELF() &&
1503 TM.getRelocationModel() != Reloc::Static &&
1504 Mod.getPIELevel() == PIELevel::Default;
1505 return ClearDSOLocalOnDeclarations;
1509 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1510 LLVMTargetMachineRef TM) {
1511 Module &Mod = *unwrap(M);
1512 TargetMachine &Target = *unwrap(TM);
1514 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1515 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1518 LLVMRustSetLastError("renameModuleForThinLTO failed");
1525 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1526 Module &Mod = *unwrap(M);
1527 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1528 #if LLVM_VERSION_GE(14, 0)
1529 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1531 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1537 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1538 Module &Mod = *unwrap(M);
1539 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1540 thinLTOInternalizeModule(Mod, DefinedGlobals);
1545 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1546 LLVMTargetMachineRef TM) {
1547 Module &Mod = *unwrap(M);
1548 TargetMachine &Target = *unwrap(TM);
1550 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1551 auto Loader = [&](StringRef Identifier) {
1552 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1553 auto &Context = Mod.getContext();
1554 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1559 // The rest of this closure is a workaround for
1560 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1561 // we accidentally import wasm custom sections into different modules,
1562 // duplicating them by in the final output artifact.
1564 // The issue is worked around here by manually removing the
1565 // `wasm.custom_sections` named metadata node from any imported module. This
1566 // we know isn't used by any optimization pass so there's no need for it to
1569 // Note that the metadata is currently lazily loaded, so we materialize it
1570 // here before looking up if there's metadata inside. The `FunctionImporter`
1571 // will immediately materialize metadata anyway after an import, so this
1572 // shouldn't be a perf hit.
1573 if (Error Err = (*MOrErr)->materializeMetadata()) {
1574 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1578 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1579 if (WasmCustomSections)
1580 WasmCustomSections->eraseFromParent();
1584 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1585 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1586 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1588 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1594 // This struct and various functions are sort of a hack right now, but the
1595 // problem is that we've got in-memory LLVM modules after we generate and
1596 // optimize all codegen-units for one compilation in rustc. To be compatible
1597 // with the LTO support above we need to serialize the modules plus their
1598 // ThinLTO summary into memory.
1600 // This structure is basically an owned version of a serialize module, with
1601 // a ThinLTO summary attached.
1602 struct LLVMRustThinLTOBuffer {
1606 extern "C" LLVMRustThinLTOBuffer*
1607 LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) {
1608 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1610 raw_string_ostream OS(Ret->data);
1613 #if LLVM_VERSION_LT(15, 0)
1614 legacy::PassManager PM;
1615 PM.add(createWriteThinLTOBitcodePass(OS));
1619 LoopAnalysisManager LAM;
1620 FunctionAnalysisManager FAM;
1621 CGSCCAnalysisManager CGAM;
1622 ModuleAnalysisManager MAM;
1623 PB.registerModuleAnalyses(MAM);
1624 PB.registerCGSCCAnalyses(CGAM);
1625 PB.registerFunctionAnalyses(FAM);
1626 PB.registerLoopAnalyses(LAM);
1627 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1628 ModulePassManager MPM;
1629 MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr));
1630 MPM.run(*unwrap(M), MAM);
1633 legacy::PassManager PM;
1634 PM.add(createBitcodeWriterPass(OS));
1639 return Ret.release();
1643 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1647 extern "C" const void*
1648 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1649 return Buffer->data.data();
1653 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1654 return Buffer->data.length();
1657 // This is what we used to parse upstream bitcode for actual ThinLTO
1658 // processing. We'll call this once per module optimized through ThinLTO, and
1659 // it'll be called concurrently on many threads.
1660 extern "C" LLVMModuleRef
1661 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1664 const char *identifier) {
1665 StringRef Data(data, len);
1666 MemoryBufferRef Buffer(Data, identifier);
1667 unwrap(Context)->enableDebugTypeODRUniquing();
1668 Expected<std::unique_ptr<Module>> SrcOrError =
1669 parseBitcodeFile(Buffer, *unwrap(Context));
1671 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1674 return wrap(std::move(*SrcOrError).release());
1677 // Find the bitcode section in the object file data and return it as a slice.
1678 // Fail if the bitcode section is present but empty.
1680 // On success, the return value is the pointer to the start of the slice and
1681 // `out_len` is filled with the (non-zero) length. On failure, the return value
1682 // is `nullptr` and `out_len` is set to zero.
1683 extern "C" const char*
1684 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1689 StringRef Data(data, len);
1690 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1692 Expected<MemoryBufferRef> BitcodeOrError =
1693 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1694 if (!BitcodeOrError) {
1695 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1699 *out_len = BitcodeOrError->getBufferSize();
1700 return BitcodeOrError->getBufferStart();
1703 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1704 // the comment in `back/lto.rs` for why this exists.
1706 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1708 DICompileUnit **B) {
1709 Module *M = unwrap(Mod);
1710 DICompileUnit **Cur = A;
1711 DICompileUnit **Next = B;
1712 for (DICompileUnit *CU : M->debug_compile_units()) {
1721 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1722 // the comment in `back/lto.rs` for why this exists.
1724 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1725 Module *M = unwrap(Mod);
1727 // If the original source module didn't have a `DICompileUnit` then try to
1728 // merge all the existing compile units. If there aren't actually any though
1729 // then there's not much for us to do so return.
1730 if (Unit == nullptr) {
1731 for (DICompileUnit *CU : M->debug_compile_units()) {
1735 if (Unit == nullptr)
1739 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1740 // process it recursively. Note that we used to specifically iterate over
1741 // instructions to ensure we feed everything into it, but `processModule`
1742 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1743 DebugInfoFinder Finder;
1744 Finder.processModule(*M);
1746 // After we've found all our debuginfo, rewrite all subprograms to point to
1747 // the same `DICompileUnit`.
1748 for (auto &F : Finder.subprograms()) {
1749 F->replaceUnit(Unit);
1752 // Erase any other references to other `DICompileUnit` instances, the verifier
1753 // will later ensure that we don't actually have any other stale references to
1755 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1756 MD->clearOperands();
1757 MD->addOperand(Unit);
1760 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1761 // storing the result in 'KeyOut'.
1762 // Currently, this cache key is a SHA-1 hash of anything that could affect
1763 // the result of optimizing this module (e.g. module imports, exports, liveness
1764 // of access globals, etc).
1765 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1766 // used during the normal linker-plugin incremental thin-LTO process.
1768 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1769 SmallString<40> Key;
1770 llvm::lto::Config conf;
1771 const auto &ImportList = Data->ImportLists.lookup(ModId);
1772 const auto &ExportList = Data->ExportLists.lookup(ModId);
1773 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1774 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1775 std::set<GlobalValue::GUID> CfiFunctionDefs;
1776 std::set<GlobalValue::GUID> CfiFunctionDecls;
1778 // Based on the 'InProcessThinBackend' constructor in LLVM
1779 for (auto &Name : Data->Index.cfiFunctionDefs())
1780 CfiFunctionDefs.insert(
1781 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1782 for (auto &Name : Data->Index.cfiFunctionDecls())
1783 CfiFunctionDecls.insert(
1784 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1786 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1787 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1790 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());