6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/PassPlugin.h"
21 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #if LLVM_VERSION_LT(14, 0)
26 #include "llvm/Support/TargetRegistry.h"
28 #include "llvm/MC/TargetRegistry.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
32 #include "llvm/Transforms/IPO/AlwaysInliner.h"
33 #include "llvm/Transforms/IPO/FunctionImport.h"
34 #include "llvm/Transforms/Utils/AddDiscriminators.h"
35 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
36 #include "llvm/LTO/LTO.h"
37 #include "llvm/Bitcode/BitcodeWriterPass.h"
38 #include "llvm-c/Transforms/PassManagerBuilder.h"
40 #include "llvm/Transforms/Instrumentation.h"
41 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
42 #include "llvm/Support/TimeProfiler.h"
43 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
44 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
45 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
46 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
47 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
48 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
49 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
50 #include "llvm/Transforms/Utils.h"
54 typedef struct LLVMOpaquePass *LLVMPassRef;
55 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
57 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
58 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
60 extern "C" void LLVMInitializePasses() {
61 PassRegistry &Registry = *PassRegistry::getPassRegistry();
62 initializeCore(Registry);
63 initializeCodeGen(Registry);
64 initializeScalarOpts(Registry);
65 initializeVectorization(Registry);
66 initializeIPO(Registry);
67 initializeAnalysis(Registry);
68 initializeTransformUtils(Registry);
69 initializeInstCombine(Registry);
70 initializeInstrumentation(Registry);
71 initializeTarget(Registry);
74 extern "C" void LLVMTimeTraceProfilerInitialize() {
75 timeTraceProfilerInitialize(
76 /* TimeTraceGranularity */ 0,
77 /* ProcName */ "rustc");
80 extern "C" void LLVMTimeTraceProfilerFinishThread() {
81 timeTraceProfilerFinishThread();
84 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
85 StringRef FN(FileName);
87 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
89 timeTraceProfilerWrite(OS);
90 timeTraceProfilerCleanup();
93 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
94 #if LLVM_VERSION_LT(15, 0)
95 StringRef SR(PassName);
96 PassRegistry *PR = PassRegistry::getPassRegistry();
98 const PassInfo *PI = PR->getPassInfo(SR);
100 return wrap(PI->createPass());
104 report_fatal_error("Legacy PM not supported with LLVM 15");
108 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
109 #if LLVM_VERSION_LT(15, 0)
110 const bool CompileKernel = false;
111 const bool UseAfterScope = true;
113 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
115 report_fatal_error("Legacy PM not supported with LLVM 15");
119 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
120 #if LLVM_VERSION_LT(15, 0)
121 const bool CompileKernel = false;
123 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
125 report_fatal_error("Legacy PM not supported with LLVM 15");
129 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
130 #if LLVM_VERSION_LT(15, 0)
131 const bool CompileKernel = false;
133 return wrap(createMemorySanitizerLegacyPassPass(
134 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
136 report_fatal_error("Legacy PM not supported with LLVM 15");
140 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
141 #if LLVM_VERSION_LT(15, 0)
142 return wrap(createThreadSanitizerLegacyPassPass());
144 report_fatal_error("Legacy PM not supported with LLVM 15");
148 extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) {
149 #if LLVM_VERSION_LT(15, 0)
150 const bool CompileKernel = false;
152 return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover));
154 report_fatal_error("Legacy PM not supported with LLVM 15");
158 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
159 #if LLVM_VERSION_LT(15, 0)
161 Pass *Pass = unwrap(RustPass);
162 PassManagerBase *PMB = unwrap(PMR);
165 report_fatal_error("Legacy PM not supported with LLVM 15");
169 extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() {
170 #if LLVM_VERSION_LT(15, 0)
171 return LLVMPassManagerBuilderCreate();
173 report_fatal_error("Legacy PM not supported with LLVM 15");
177 extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) {
178 #if LLVM_VERSION_LT(15, 0)
179 LLVMPassManagerBuilderDispose(PMB);
181 report_fatal_error("Legacy PM not supported with LLVM 15");
185 extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager(
186 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) {
187 #if LLVM_VERSION_LT(15, 0)
188 LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM);
190 report_fatal_error("Legacy PM not supported with LLVM 15");
194 extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager(
195 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) {
196 #if LLVM_VERSION_LT(15, 0)
197 LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM);
199 report_fatal_error("Legacy PM not supported with LLVM 15");
203 extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager(
204 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) {
205 #if LLVM_VERSION_LT(15, 0)
206 LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner);
208 report_fatal_error("Legacy PM not supported with LLVM 15");
213 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
214 LLVMPassManagerBuilderRef PMBR,
215 LLVMPassManagerRef PMR
217 #if LLVM_VERSION_LT(15, 0)
218 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
220 report_fatal_error("Legacy PM not supported with LLVM 15");
224 extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold(
225 LLVMPassManagerBuilderRef PMB, unsigned Threshold) {
226 #if LLVM_VERSION_LT(15, 0)
227 LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold);
229 report_fatal_error("Legacy PM not supported with LLVM 15");
234 void LLVMRustAddLastExtensionPasses(
235 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
236 #if LLVM_VERSION_LT(15, 0)
237 auto AddExtensionPasses = [Passes, NumPasses](
238 const PassManagerBuilder &Builder, PassManagerBase &PM) {
239 for (size_t I = 0; I < NumPasses; I++) {
240 PM.add(unwrap(Passes[I]));
243 // Add the passes to both of the pre-finalization extension points,
244 // so they are run for optimized and non-optimized builds.
245 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
247 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
250 report_fatal_error("Legacy PM not supported with LLVM 15");
254 #ifdef LLVM_COMPONENT_X86
255 #define SUBTARGET_X86 SUBTARGET(X86)
257 #define SUBTARGET_X86
260 #ifdef LLVM_COMPONENT_ARM
261 #define SUBTARGET_ARM SUBTARGET(ARM)
263 #define SUBTARGET_ARM
266 #ifdef LLVM_COMPONENT_AARCH64
267 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
269 #define SUBTARGET_AARCH64
272 #ifdef LLVM_COMPONENT_AVR
273 #define SUBTARGET_AVR SUBTARGET(AVR)
275 #define SUBTARGET_AVR
278 #ifdef LLVM_COMPONENT_M68k
279 #define SUBTARGET_M68K SUBTARGET(M68k)
281 #define SUBTARGET_M68K
284 #ifdef LLVM_COMPONENT_MIPS
285 #define SUBTARGET_MIPS SUBTARGET(Mips)
287 #define SUBTARGET_MIPS
290 #ifdef LLVM_COMPONENT_POWERPC
291 #define SUBTARGET_PPC SUBTARGET(PPC)
293 #define SUBTARGET_PPC
296 #ifdef LLVM_COMPONENT_SYSTEMZ
297 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
299 #define SUBTARGET_SYSTEMZ
302 #ifdef LLVM_COMPONENT_MSP430
303 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
305 #define SUBTARGET_MSP430
308 #ifdef LLVM_COMPONENT_RISCV
309 #define SUBTARGET_RISCV SUBTARGET(RISCV)
311 #define SUBTARGET_RISCV
314 #ifdef LLVM_COMPONENT_SPARC
315 #define SUBTARGET_SPARC SUBTARGET(Sparc)
317 #define SUBTARGET_SPARC
320 #ifdef LLVM_COMPONENT_HEXAGON
321 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
323 #define SUBTARGET_HEXAGON
326 #define GEN_SUBTARGETS \
340 #define SUBTARGET(x) \
342 extern const SubtargetFeatureKV x##FeatureKV[]; \
343 extern const SubtargetFeatureKV x##SubTypeKV[]; \
349 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
350 const char *Feature) {
351 TargetMachine *Target = unwrap(TM);
352 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
353 return MCInfo->checkFeatures(std::string("+") + Feature);
356 enum class LLVMRustCodeModel {
365 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
367 case LLVMRustCodeModel::Tiny:
368 return CodeModel::Tiny;
369 case LLVMRustCodeModel::Small:
370 return CodeModel::Small;
371 case LLVMRustCodeModel::Kernel:
372 return CodeModel::Kernel;
373 case LLVMRustCodeModel::Medium:
374 return CodeModel::Medium;
375 case LLVMRustCodeModel::Large:
376 return CodeModel::Large;
377 case LLVMRustCodeModel::None:
380 report_fatal_error("Bad CodeModel.");
384 enum class LLVMRustCodeGenOptLevel {
391 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
393 case LLVMRustCodeGenOptLevel::None:
394 return CodeGenOpt::None;
395 case LLVMRustCodeGenOptLevel::Less:
396 return CodeGenOpt::Less;
397 case LLVMRustCodeGenOptLevel::Default:
398 return CodeGenOpt::Default;
399 case LLVMRustCodeGenOptLevel::Aggressive:
400 return CodeGenOpt::Aggressive;
402 report_fatal_error("Bad CodeGenOptLevel.");
406 enum class LLVMRustPassBuilderOptLevel {
415 #if LLVM_VERSION_LT(14,0)
416 using OptimizationLevel = PassBuilder::OptimizationLevel;
419 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
421 case LLVMRustPassBuilderOptLevel::O0:
422 return OptimizationLevel::O0;
423 case LLVMRustPassBuilderOptLevel::O1:
424 return OptimizationLevel::O1;
425 case LLVMRustPassBuilderOptLevel::O2:
426 return OptimizationLevel::O2;
427 case LLVMRustPassBuilderOptLevel::O3:
428 return OptimizationLevel::O3;
429 case LLVMRustPassBuilderOptLevel::Os:
430 return OptimizationLevel::Os;
431 case LLVMRustPassBuilderOptLevel::Oz:
432 return OptimizationLevel::Oz;
434 report_fatal_error("Bad PassBuilderOptLevel.");
438 enum class LLVMRustRelocModel {
447 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
449 case LLVMRustRelocModel::Static:
450 return Reloc::Static;
451 case LLVMRustRelocModel::PIC:
453 case LLVMRustRelocModel::DynamicNoPic:
454 return Reloc::DynamicNoPIC;
455 case LLVMRustRelocModel::ROPI:
457 case LLVMRustRelocModel::RWPI:
459 case LLVMRustRelocModel::ROPIRWPI:
460 return Reloc::ROPI_RWPI;
462 report_fatal_error("Bad RelocModel.");
466 /// getLongestEntryLength - Return the length of the longest entry in the table.
467 template<typename KV>
468 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
470 for (auto &I : Table)
471 MaxLen = std::max(MaxLen, std::strlen(I.Key));
475 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
476 const TargetMachine *Target = unwrap(TM);
477 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
478 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
479 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
480 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
481 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
483 printf("Available CPUs for this target:\n");
484 if (HostArch == TargetArch) {
485 const StringRef HostCPU = sys::getHostCPUName();
486 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
487 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
489 for (auto &CPU : CPUTable)
490 printf(" %-*s\n", MaxCPULen, CPU.Key);
494 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
495 const TargetMachine *Target = unwrap(TM);
496 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
497 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
498 return FeatTable.size();
501 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
502 const char** Feature, const char** Desc) {
503 const TargetMachine *Target = unwrap(TM);
504 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
505 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
506 const SubtargetFeatureKV Feat = FeatTable[Index];
513 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
514 printf("Target CPU help is not supported by this LLVM version.\n\n");
517 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
521 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
524 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
525 StringRef Name = sys::getHostCPUName();
530 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
531 const char *TripleStr, const char *CPU, const char *Feature,
532 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
533 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
534 bool FunctionSections,
536 bool UniqueSectionNames,
537 bool TrapUnreachable,
540 bool EmitStackSizeSection,
541 bool RelaxELFRelocations,
543 const char *SplitDwarfFile) {
545 auto OptLevel = fromRust(RustOptLevel);
546 auto RM = fromRust(RustReloc);
547 auto CM = fromRust(RustCM);
550 Triple Trip(Triple::normalize(TripleStr));
551 const llvm::Target *TheTarget =
552 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
553 if (TheTarget == nullptr) {
554 LLVMRustSetLastError(Error.c_str());
558 TargetOptions Options;
560 Options.FloatABIType = FloatABI::Default;
562 Options.FloatABIType = FloatABI::Soft;
564 Options.DataSections = DataSections;
565 Options.FunctionSections = FunctionSections;
566 Options.UniqueSectionNames = UniqueSectionNames;
567 Options.MCOptions.AsmVerbose = AsmComments;
568 Options.MCOptions.PreserveAsmComments = AsmComments;
569 Options.MCOptions.ABIName = ABIStr;
570 if (SplitDwarfFile) {
571 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
573 Options.RelaxELFRelocations = RelaxELFRelocations;
574 Options.UseInitArray = UseInitArray;
576 if (TrapUnreachable) {
577 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
578 // This limits the extent of possible undefined behavior in some cases, as
579 // it prevents control flow from "falling through" into whatever code
580 // happens to be laid out next in memory.
581 Options.TrapUnreachable = true;
585 Options.ThreadModel = ThreadModel::Single;
588 Options.EmitStackSizeSection = EmitStackSizeSection;
590 TargetMachine *TM = TheTarget->createTargetMachine(
591 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
595 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
599 extern "C" void LLVMRustConfigurePassManagerBuilder(
600 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
601 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
602 const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath,
604 #if LLVM_VERSION_LT(15, 0)
605 unwrap(PMBR)->MergeFunctions = MergeFunctions;
606 unwrap(PMBR)->SLPVectorize = SLPVectorize;
607 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
608 unwrap(PMBR)->LoopVectorize = LoopVectorize;
609 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
610 unwrap(PMBR)->SizeLevel = SizeLevel;
611 unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0;
614 assert(!PGOUsePath && !PGOSampleUsePath);
615 unwrap(PMBR)->EnablePGOInstrGen = true;
616 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
617 } else if (PGOUsePath) {
618 assert(!PGOSampleUsePath);
619 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
620 } else if (PGOSampleUsePath) {
621 unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath;
624 report_fatal_error("Legacy PM not supported with LLVM 15");
628 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
629 // field of a PassManagerBuilder, we expose our own method of doing so.
630 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
632 bool DisableSimplifyLibCalls) {
633 Triple TargetTriple(unwrap(M)->getTargetTriple());
634 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
635 if (DisableSimplifyLibCalls)
636 TLI->disableAllFunctions();
637 unwrap(PMBR)->LibraryInfo = TLI;
640 // Unfortunately, the LLVM C API doesn't provide a way to create the
641 // TargetLibraryInfo pass, so we use this method to do so.
642 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
643 bool DisableSimplifyLibCalls) {
644 Triple TargetTriple(unwrap(M)->getTargetTriple());
645 TargetLibraryInfoImpl TLII(TargetTriple);
646 if (DisableSimplifyLibCalls)
647 TLII.disableAllFunctions();
648 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
651 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
652 // all the functions in a module, so we do that manually here. You'll find
653 // similar code in clang's BackendUtil.cpp file.
654 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
656 llvm::legacy::FunctionPassManager *P =
657 unwrap<llvm::legacy::FunctionPassManager>(PMR);
658 P->doInitialization();
660 // Upgrade all calls to old intrinsics first.
661 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
662 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
664 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
666 if (!I->isDeclaration())
672 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
673 // Initializing the command-line options more than once is not allowed. So,
674 // check if they've already been initialized. (This could happen if we're
675 // being called from rustpkg, for example). If the arguments change, then
676 // that's just kinda unfortunate.
677 static bool Initialized = false;
681 cl::ParseCommandLineOptions(Argc, Argv);
684 enum class LLVMRustFileType {
689 static CodeGenFileType fromRust(LLVMRustFileType Type) {
691 case LLVMRustFileType::AssemblyFile:
692 return CGFT_AssemblyFile;
693 case LLVMRustFileType::ObjectFile:
694 return CGFT_ObjectFile;
696 report_fatal_error("Bad FileType.");
700 extern "C" LLVMRustResult
701 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
702 LLVMModuleRef M, const char *Path, const char *DwoPath,
703 LLVMRustFileType RustFileType) {
704 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
705 auto FileType = fromRust(RustFileType);
707 std::string ErrorInfo;
709 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
711 ErrorInfo = EC.message();
712 if (ErrorInfo != "") {
713 LLVMRustSetLastError(ErrorInfo.c_str());
714 return LLVMRustResult::Failure;
717 buffer_ostream BOS(OS);
719 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
722 ErrorInfo = EC.message();
723 if (ErrorInfo != "") {
724 LLVMRustSetLastError(ErrorInfo.c_str());
725 return LLVMRustResult::Failure;
727 buffer_ostream DBOS(DOS);
728 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
731 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
735 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
736 // stream (OS), so the only real safe place to delete this is here? Don't we
737 // wish this was written in Rust?
738 LLVMDisposePassManager(PMR);
739 return LLVMRustResult::Success;
742 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
743 const char*, // pass name
744 const char*); // IR name
745 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
747 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
748 if (any_isa<const Module *>(WrappedIr))
749 return any_cast<const Module *>(WrappedIr)->getName().str();
750 if (any_isa<const Function *>(WrappedIr))
751 return any_cast<const Function *>(WrappedIr)->getName().str();
752 if (any_isa<const Loop *>(WrappedIr))
753 return any_cast<const Loop *>(WrappedIr)->getName().str();
754 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
755 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
760 void LLVMSelfProfileInitializeCallbacks(
761 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
762 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
763 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
764 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
765 StringRef Pass, llvm::Any Ir) {
766 std::string PassName = Pass.str();
767 std::string IrName = LLVMRustwrappedIrGetName(Ir);
768 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
771 PIC.registerAfterPassCallback(
772 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
773 const PreservedAnalyses &Preserved) {
774 AfterPassCallback(LlvmSelfProfiler);
777 PIC.registerAfterPassInvalidatedCallback(
778 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
779 AfterPassCallback(LlvmSelfProfiler);
782 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
783 StringRef Pass, llvm::Any Ir) {
784 std::string PassName = Pass.str();
785 std::string IrName = LLVMRustwrappedIrGetName(Ir);
786 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
789 PIC.registerAfterAnalysisCallback(
790 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
791 AfterPassCallback(LlvmSelfProfiler);
795 enum class LLVMRustOptStage {
803 struct LLVMRustSanitizerOptions {
804 bool SanitizeAddress;
805 bool SanitizeAddressRecover;
807 bool SanitizeMemoryRecover;
808 int SanitizeMemoryTrackOrigins;
810 bool SanitizeHWAddress;
811 bool SanitizeHWAddressRecover;
814 extern "C" LLVMRustResult
815 LLVMRustOptimizeWithNewPassManager(
816 LLVMModuleRef ModuleRef,
817 LLVMTargetMachineRef TMRef,
818 LLVMRustPassBuilderOptLevel OptLevelRust,
819 LLVMRustOptStage OptStage,
820 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
821 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
822 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
823 LLVMRustSanitizerOptions *SanitizerOptions,
824 const char *PGOGenPath, const char *PGOUsePath,
825 bool InstrumentCoverage, bool InstrumentGCOV,
826 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
827 void* LlvmSelfProfiler,
828 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
829 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
830 const char *ExtraPasses, size_t ExtraPassesLen,
831 const char *LLVMPlugins, size_t LLVMPluginsLen) {
832 Module *TheModule = unwrap(ModuleRef);
833 TargetMachine *TM = unwrap(TMRef);
834 OptimizationLevel OptLevel = fromRust(OptLevelRust);
837 PipelineTuningOptions PTO;
838 PTO.LoopUnrolling = UnrollLoops;
839 PTO.LoopInterleaving = UnrollLoops;
840 PTO.LoopVectorization = LoopVectorize;
841 PTO.SLPVectorization = SLPVectorize;
842 PTO.MergeFunctions = MergeFunctions;
844 // FIXME: We may want to expose this as an option.
845 bool DebugPassManager = false;
847 PassInstrumentationCallbacks PIC;
848 StandardInstrumentations SI(DebugPassManager);
849 SI.registerCallbacks(PIC);
851 if (LlvmSelfProfiler){
852 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
855 Optional<PGOOptions> PGOOpt;
857 assert(!PGOUsePath && !PGOSampleUsePath);
858 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
859 PGOOptions::NoCSAction, DebugInfoForProfiling);
860 } else if (PGOUsePath) {
861 assert(!PGOSampleUsePath);
862 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
863 PGOOptions::NoCSAction, DebugInfoForProfiling);
864 } else if (PGOSampleUsePath) {
865 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
866 PGOOptions::NoCSAction, DebugInfoForProfiling);
867 } else if (DebugInfoForProfiling) {
868 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
869 PGOOptions::NoCSAction, DebugInfoForProfiling);
872 #if LLVM_VERSION_GE(13, 0)
873 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
874 LoopAnalysisManager LAM;
875 FunctionAnalysisManager FAM;
876 CGSCCAnalysisManager CGAM;
877 ModuleAnalysisManager MAM;
879 PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC);
880 LoopAnalysisManager LAM(DebugPassManager);
881 FunctionAnalysisManager FAM(DebugPassManager);
882 CGSCCAnalysisManager CGAM(DebugPassManager);
883 ModuleAnalysisManager MAM(DebugPassManager);
886 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
888 Triple TargetTriple(TheModule->getTargetTriple());
889 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
890 if (DisableSimplifyLibCalls)
891 TLII->disableAllFunctions();
892 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
894 PB.registerModuleAnalyses(MAM);
895 PB.registerCGSCCAnalyses(CGAM);
896 PB.registerFunctionAnalyses(FAM);
897 PB.registerLoopAnalyses(LAM);
898 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
900 // We manually collect pipeline callbacks so we can apply them at O0, where the
901 // PassBuilder does not create a pipeline.
902 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
903 PipelineStartEPCallbacks;
904 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
905 OptimizerLastEPCallbacks;
908 PipelineStartEPCallbacks.push_back(
909 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
910 MPM.addPass(VerifierPass());
915 if (InstrumentGCOV) {
916 PipelineStartEPCallbacks.push_back(
917 [](ModulePassManager &MPM, OptimizationLevel Level) {
918 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
923 if (InstrumentCoverage) {
924 PipelineStartEPCallbacks.push_back(
925 [](ModulePassManager &MPM, OptimizationLevel Level) {
926 InstrProfOptions Options;
927 MPM.addPass(InstrProfiling(Options, false));
932 if (SanitizerOptions) {
933 if (SanitizerOptions->SanitizeMemory) {
934 MemorySanitizerOptions Options(
935 SanitizerOptions->SanitizeMemoryTrackOrigins,
936 SanitizerOptions->SanitizeMemoryRecover,
937 /*CompileKernel=*/false);
938 OptimizerLastEPCallbacks.push_back(
939 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
940 #if LLVM_VERSION_GE(14, 0)
941 MPM.addPass(ModuleMemorySanitizerPass(Options));
943 MPM.addPass(MemorySanitizerPass(Options));
945 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
950 if (SanitizerOptions->SanitizeThread) {
951 OptimizerLastEPCallbacks.push_back(
952 [](ModulePassManager &MPM, OptimizationLevel Level) {
953 #if LLVM_VERSION_GE(14, 0)
954 MPM.addPass(ModuleThreadSanitizerPass());
956 MPM.addPass(ThreadSanitizerPass());
958 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
963 if (SanitizerOptions->SanitizeAddress) {
964 OptimizerLastEPCallbacks.push_back(
965 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
966 #if LLVM_VERSION_LT(15, 0)
967 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
969 #if LLVM_VERSION_GE(14, 0)
970 AddressSanitizerOptions opts = AddressSanitizerOptions{
971 /*CompileKernel=*/false,
972 SanitizerOptions->SanitizeAddressRecover,
973 /*UseAfterScope=*/true,
974 AsanDetectStackUseAfterReturnMode::Runtime,
976 MPM.addPass(ModuleAddressSanitizerPass(opts));
978 MPM.addPass(ModuleAddressSanitizerPass(
979 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
980 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
981 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
982 /*UseAfterScope=*/true)));
987 if (SanitizerOptions->SanitizeHWAddress) {
988 OptimizerLastEPCallbacks.push_back(
989 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
990 #if LLVM_VERSION_GE(14, 0)
991 HWAddressSanitizerOptions opts(
992 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
993 /*DisableOptimization=*/false);
994 MPM.addPass(HWAddressSanitizerPass(opts));
996 MPM.addPass(HWAddressSanitizerPass(
997 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
1004 if (LLVMPluginsLen) {
1005 auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
1006 SmallVector<StringRef> Plugins;
1007 PluginsStr.split(Plugins, ',', -1, false);
1008 for (auto PluginPath: Plugins) {
1009 auto Plugin = PassPlugin::Load(PluginPath.str());
1011 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
1014 Plugin->registerPassBuilderCallbacks(PB);
1018 #if LLVM_VERSION_GE(13, 0)
1019 ModulePassManager MPM;
1021 ModulePassManager MPM(DebugPassManager);
1023 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
1024 if (!NoPrepopulatePasses) {
1025 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
1026 // At the same time, the LTO pipelines do support O0 and using them is required.
1027 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
1028 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
1029 for (const auto &C : PipelineStartEPCallbacks)
1030 PB.registerPipelineStartEPCallback(C);
1031 for (const auto &C : OptimizerLastEPCallbacks)
1032 PB.registerOptimizerLastEPCallback(C);
1034 // Pass false as we manually schedule ThinLTOBufferPasses below.
1035 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
1037 for (const auto &C : PipelineStartEPCallbacks)
1038 PB.registerPipelineStartEPCallback(C);
1039 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
1040 for (const auto &C : OptimizerLastEPCallbacks)
1041 PB.registerOptimizerLastEPCallback(C);
1045 case LLVMRustOptStage::PreLinkNoLTO:
1046 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
1048 case LLVMRustOptStage::PreLinkThinLTO:
1049 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
1050 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
1051 // passes may still run afterwards. This means we need to run the buffer passes again.
1052 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
1053 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
1054 if (OptimizerLastEPCallbacks.empty())
1055 NeedThinLTOBufferPasses = false;
1056 for (const auto &C : OptimizerLastEPCallbacks)
1059 case LLVMRustOptStage::PreLinkFatLTO:
1060 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
1061 NeedThinLTOBufferPasses = false;
1063 case LLVMRustOptStage::ThinLTO:
1064 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
1065 // It only seems to be needed for C++ specific optimizations.
1066 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
1068 case LLVMRustOptStage::FatLTO:
1069 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
1074 // We're not building any of the default pipelines but we still want to
1075 // add the verifier, instrumentation, etc passes if they were requested
1076 for (const auto &C : PipelineStartEPCallbacks)
1078 for (const auto &C : OptimizerLastEPCallbacks)
1082 if (ExtraPassesLen) {
1083 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
1084 std::string ErrMsg = toString(std::move(Err));
1085 LLVMRustSetLastError(ErrMsg.c_str());
1086 return LLVMRustResult::Failure;
1090 if (NeedThinLTOBufferPasses) {
1091 MPM.addPass(CanonicalizeAliasesPass());
1092 MPM.addPass(NameAnonGlobalPass());
1095 // Upgrade all calls to old intrinsics first.
1096 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
1097 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
1099 MPM.run(*TheModule, MAM);
1100 return LLVMRustResult::Success;
1103 // Callback to demangle function name
1105 // * name to be demangled
1108 // * output buffer len
1109 // Returns len of demangled string, or 0 if demangle failed.
1110 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1115 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1116 DemangleFn Demangle;
1117 std::vector<char> Buf;
1120 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1122 // Return empty string if demangle failed
1123 // or if name does not need to be demangled
1124 StringRef CallDemangle(StringRef name) {
1129 if (Buf.size() < name.size() * 2) {
1130 // Semangled name usually shorter than mangled,
1131 // but allocate twice as much memory just in case
1132 Buf.resize(name.size() * 2);
1135 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1141 auto Demangled = StringRef(Buf.data(), R);
1142 if (Demangled == name) {
1143 // Do not print anything if demangled name is equal to mangled.
1150 void emitFunctionAnnot(const Function *F,
1151 formatted_raw_ostream &OS) override {
1152 StringRef Demangled = CallDemangle(F->getName());
1153 if (Demangled.empty()) {
1157 OS << "; " << Demangled << "\n";
1160 void emitInstructionAnnot(const Instruction *I,
1161 formatted_raw_ostream &OS) override {
1164 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1166 Value = CI->getCalledOperand();
1167 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1169 Value = II->getCalledOperand();
1171 // Could demangle more operations, e. g.
1172 // `store %place, @function`.
1176 if (!Value->hasName()) {
1180 StringRef Demangled = CallDemangle(Value->getName());
1181 if (Demangled.empty()) {
1185 OS << "; " << Name << " " << Demangled << "\n";
1191 extern "C" LLVMRustResult
1192 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1193 std::string ErrorInfo;
1195 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
1197 ErrorInfo = EC.message();
1198 if (ErrorInfo != "") {
1199 LLVMRustSetLastError(ErrorInfo.c_str());
1200 return LLVMRustResult::Failure;
1203 RustAssemblyAnnotationWriter AAW(Demangle);
1204 formatted_raw_ostream FOS(OS);
1205 unwrap(M)->print(FOS, &AAW);
1207 return LLVMRustResult::Success;
1210 extern "C" void LLVMRustPrintPasses() {
1211 LLVMInitializePasses();
1212 struct MyListener : PassRegistrationListener {
1213 void passEnumerate(const PassInfo *Info) {
1214 StringRef PassArg = Info->getPassArgument();
1215 StringRef PassName = Info->getPassName();
1216 if (!PassArg.empty()) {
1217 // These unsigned->signed casts could theoretically overflow, but
1218 // realistically never will (and even if, the result is implementation
1219 // defined rather plain UB).
1220 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1221 (int)PassName.size(), PassName.data());
1226 PassRegistry *PR = PassRegistry::getPassRegistry();
1227 PR->enumerateWith(&Listener);
1230 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1231 bool AddLifetimes) {
1232 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1235 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1237 llvm::legacy::PassManager passes;
1239 auto PreserveFunctions = [=](const GlobalValue &GV) {
1240 for (size_t I = 0; I < Len; I++) {
1241 if (GV.getName() == Symbols[I]) {
1248 passes.add(llvm::createInternalizePass(PreserveFunctions));
1250 passes.run(*unwrap(M));
1254 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1255 LLVMTargetMachineRef TMR) {
1256 TargetMachine *Target = unwrap(TMR);
1257 unwrap(Module)->setDataLayout(Target->createDataLayout());
1260 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1261 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1264 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1265 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1268 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1269 LLVMRustCodeModel Model) {
1270 auto CM = fromRust(Model);
1273 unwrap(M)->setCodeModel(*CM);
1276 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1277 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1278 // LLVM, and otherwise it's just blanket rejected from other compilers.
1280 // Most of this implementation is straight copied from LLVM. At the time of
1281 // this writing it wasn't *quite* suitable to reuse more code from upstream
1282 // for our purposes, but we should strive to upstream this support once it's
1283 // ready to go! I figure we may want a bit of testing locally first before
1284 // sending this upstream to LLVM. I hear though they're quite eager to receive
1285 // feedback like this!
1287 // If you're reading this code and wondering "what in the world" or you're
1288 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1289 // then fear not! (ok maybe fear a little). All code here is mostly based
1290 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1292 // You'll find that the general layout here roughly corresponds to the `run`
1293 // method in that file as well as `ProcessThinLTOModule`. Functions are
1294 // specifically commented below as well, but if you're updating this code
1295 // or otherwise trying to understand it, the LLVM source will be useful in
1296 // interpreting the mysteries within.
1298 // Otherwise I'll apologize in advance, it probably requires a relatively
1299 // significant investment on your part to "truly understand" what's going on
1300 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1301 // and various online resources about ThinLTO to make heads or tails of all
1304 // This is a shared data structure which *must* be threadsafe to share
1305 // read-only amongst threads. This also corresponds basically to the arguments
1306 // of the `ProcessThinLTOModule` function in the LLVM source.
1307 struct LLVMRustThinLTOData {
1308 // The combined index that is the global analysis over all modules we're
1309 // performing ThinLTO for. This is mostly managed by LLVM.
1310 ModuleSummaryIndex Index;
1312 // All modules we may look at, stored as in-memory serialized versions. This
1313 // is later used when inlining to ensure we can extract any module to inline
1315 StringMap<MemoryBufferRef> ModuleMap;
1317 // A set that we manage of everything we *don't* want internalized. Note that
1318 // this includes all transitive references right now as well, but it may not
1320 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1322 // Not 100% sure what these are, but they impact what's internalized and
1323 // what's inlined across modules, I believe.
1324 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1325 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1326 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1327 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1329 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1332 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1333 struct LLVMRustThinLTOModule {
1334 const char *identifier;
1339 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1341 static const GlobalValueSummary *
1342 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1343 auto StrongDefForLinker = llvm::find_if(
1344 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1345 auto Linkage = Summary->linkage();
1346 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1347 !GlobalValue::isWeakForLinker(Linkage);
1349 if (StrongDefForLinker != GVSummaryList.end())
1350 return StrongDefForLinker->get();
1352 auto FirstDefForLinker = llvm::find_if(
1353 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1354 auto Linkage = Summary->linkage();
1355 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1357 if (FirstDefForLinker == GVSummaryList.end())
1359 return FirstDefForLinker->get();
1362 // The main entry point for creating the global ThinLTO analysis. The structure
1363 // here is basically the same as before threads are spawned in the `run`
1364 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1365 extern "C" LLVMRustThinLTOData*
1366 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1368 const char **preserved_symbols,
1370 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1372 // Load each module's summary and merge it into one combined index
1373 for (int i = 0; i < num_modules; i++) {
1374 auto module = &modules[i];
1375 StringRef buffer(module->data, module->len);
1376 MemoryBufferRef mem_buffer(buffer, module->identifier);
1378 Ret->ModuleMap[module->identifier] = mem_buffer;
1380 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1381 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1386 // Collect for each module the list of function it defines (GUID -> Summary)
1387 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1389 // Convert the preserved symbols set from string to GUID, this is then needed
1390 // for internalization.
1391 for (int i = 0; i < num_symbols; i++) {
1392 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1393 Ret->GUIDPreservedSymbols.insert(GUID);
1396 // Collect the import/export lists for all modules from the call-graph in the
1399 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1400 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1401 return PrevailingType::Unknown;
1403 // We don't have a complete picture in our use of ThinLTO, just our immediate
1404 // crate, so we need `ImportEnabled = false` to limit internalization.
1405 // Otherwise, we sometimes lose `static` values -- see #60184.
1406 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1407 deadIsPrevailing, /* ImportEnabled = */ false);
1408 ComputeCrossModuleImport(
1410 Ret->ModuleToDefinedGVSummaries,
1415 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1416 // impacts the caching.
1418 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1419 // being lifted from `lib/LTO/LTO.cpp` as well
1420 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1421 for (auto &I : Ret->Index) {
1422 if (I.second.SummaryList.size() > 1)
1423 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1425 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1426 const auto &Prevailing = PrevailingCopy.find(GUID);
1427 if (Prevailing == PrevailingCopy.end())
1429 return Prevailing->second == S;
1431 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1432 GlobalValue::GUID GUID,
1433 GlobalValue::LinkageTypes NewLinkage) {
1434 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1437 #if LLVM_VERSION_GE(13,0)
1438 // Uses FromPrevailing visibility scheme which works for many binary
1439 // formats. We probably could and should use ELF visibility scheme for many of
1440 // our targets, however.
1442 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1443 Ret->GUIDPreservedSymbols);
1445 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
1446 Ret->GUIDPreservedSymbols);
1448 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1449 // callback below. This callback below will dictate the linkage for all
1450 // summaries in the index, and we basically just only want to ensure that dead
1451 // symbols are internalized. Otherwise everything that's already external
1452 // linkage will stay as external, and internal will stay as internal.
1453 std::set<GlobalValue::GUID> ExportedGUIDs;
1454 for (auto &List : Ret->Index) {
1455 for (auto &GVS: List.second.SummaryList) {
1456 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1458 auto GUID = GVS->getOriginalName();
1459 if (GVS->flags().Live)
1460 ExportedGUIDs.insert(GUID);
1463 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1464 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1465 return (ExportList != Ret->ExportLists.end() &&
1466 ExportList->second.count(VI)) ||
1467 ExportedGUIDs.count(VI.getGUID());
1469 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1471 return Ret.release();
1475 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1479 // Below are the various passes that happen *per module* when doing ThinLTO.
1481 // In other words, these are the functions that are all run concurrently
1482 // with one another, one per module. The passes here correspond to the analysis
1483 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1484 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1485 // so rustc can save off the intermediate bytecode between each step.
1488 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1489 // When linking an ELF shared object, dso_local should be dropped. We
1490 // conservatively do this for -fpic.
1491 bool ClearDSOLocalOnDeclarations =
1492 TM.getTargetTriple().isOSBinFormatELF() &&
1493 TM.getRelocationModel() != Reloc::Static &&
1494 Mod.getPIELevel() == PIELevel::Default;
1495 return ClearDSOLocalOnDeclarations;
1499 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1500 LLVMTargetMachineRef TM) {
1501 Module &Mod = *unwrap(M);
1502 TargetMachine &Target = *unwrap(TM);
1504 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1505 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1508 LLVMRustSetLastError("renameModuleForThinLTO failed");
1515 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1516 Module &Mod = *unwrap(M);
1517 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1518 #if LLVM_VERSION_GE(14, 0)
1519 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1521 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1527 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1528 Module &Mod = *unwrap(M);
1529 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1530 thinLTOInternalizeModule(Mod, DefinedGlobals);
1535 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1536 LLVMTargetMachineRef TM) {
1537 Module &Mod = *unwrap(M);
1538 TargetMachine &Target = *unwrap(TM);
1540 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1541 auto Loader = [&](StringRef Identifier) {
1542 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1543 auto &Context = Mod.getContext();
1544 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1549 // The rest of this closure is a workaround for
1550 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1551 // we accidentally import wasm custom sections into different modules,
1552 // duplicating them by in the final output artifact.
1554 // The issue is worked around here by manually removing the
1555 // `wasm.custom_sections` named metadata node from any imported module. This
1556 // we know isn't used by any optimization pass so there's no need for it to
1559 // Note that the metadata is currently lazily loaded, so we materialize it
1560 // here before looking up if there's metadata inside. The `FunctionImporter`
1561 // will immediately materialize metadata anyway after an import, so this
1562 // shouldn't be a perf hit.
1563 if (Error Err = (*MOrErr)->materializeMetadata()) {
1564 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1568 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1569 if (WasmCustomSections)
1570 WasmCustomSections->eraseFromParent();
1574 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1575 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1576 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1578 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1584 // This struct and various functions are sort of a hack right now, but the
1585 // problem is that we've got in-memory LLVM modules after we generate and
1586 // optimize all codegen-units for one compilation in rustc. To be compatible
1587 // with the LTO support above we need to serialize the modules plus their
1588 // ThinLTO summary into memory.
1590 // This structure is basically an owned version of a serialize module, with
1591 // a ThinLTO summary attached.
1592 struct LLVMRustThinLTOBuffer {
1596 extern "C" LLVMRustThinLTOBuffer*
1597 LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) {
1598 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1600 raw_string_ostream OS(Ret->data);
1602 legacy::PassManager PM;
1604 PM.add(createWriteThinLTOBitcodePass(OS));
1606 PM.add(createBitcodeWriterPass(OS));
1611 return Ret.release();
1615 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1619 extern "C" const void*
1620 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1621 return Buffer->data.data();
1625 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1626 return Buffer->data.length();
1629 // This is what we used to parse upstream bitcode for actual ThinLTO
1630 // processing. We'll call this once per module optimized through ThinLTO, and
1631 // it'll be called concurrently on many threads.
1632 extern "C" LLVMModuleRef
1633 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1636 const char *identifier) {
1637 StringRef Data(data, len);
1638 MemoryBufferRef Buffer(Data, identifier);
1639 unwrap(Context)->enableDebugTypeODRUniquing();
1640 Expected<std::unique_ptr<Module>> SrcOrError =
1641 parseBitcodeFile(Buffer, *unwrap(Context));
1643 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1646 return wrap(std::move(*SrcOrError).release());
1649 // Find the bitcode section in the object file data and return it as a slice.
1650 // Fail if the bitcode section is present but empty.
1652 // On success, the return value is the pointer to the start of the slice and
1653 // `out_len` is filled with the (non-zero) length. On failure, the return value
1654 // is `nullptr` and `out_len` is set to zero.
1655 extern "C" const char*
1656 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1661 StringRef Data(data, len);
1662 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1664 Expected<MemoryBufferRef> BitcodeOrError =
1665 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1666 if (!BitcodeOrError) {
1667 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1671 *out_len = BitcodeOrError->getBufferSize();
1672 return BitcodeOrError->getBufferStart();
1675 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1676 // the comment in `back/lto.rs` for why this exists.
1678 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1680 DICompileUnit **B) {
1681 Module *M = unwrap(Mod);
1682 DICompileUnit **Cur = A;
1683 DICompileUnit **Next = B;
1684 for (DICompileUnit *CU : M->debug_compile_units()) {
1693 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1694 // the comment in `back/lto.rs` for why this exists.
1696 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1697 Module *M = unwrap(Mod);
1699 // If the original source module didn't have a `DICompileUnit` then try to
1700 // merge all the existing compile units. If there aren't actually any though
1701 // then there's not much for us to do so return.
1702 if (Unit == nullptr) {
1703 for (DICompileUnit *CU : M->debug_compile_units()) {
1707 if (Unit == nullptr)
1711 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1712 // process it recursively. Note that we used to specifically iterate over
1713 // instructions to ensure we feed everything into it, but `processModule`
1714 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1715 DebugInfoFinder Finder;
1716 Finder.processModule(*M);
1718 // After we've found all our debuginfo, rewrite all subprograms to point to
1719 // the same `DICompileUnit`.
1720 for (auto &F : Finder.subprograms()) {
1721 F->replaceUnit(Unit);
1724 // Erase any other references to other `DICompileUnit` instances, the verifier
1725 // will later ensure that we don't actually have any other stale references to
1727 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1728 MD->clearOperands();
1729 MD->addOperand(Unit);
1732 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1733 // storing the result in 'KeyOut'.
1734 // Currently, this cache key is a SHA-1 hash of anything that could affect
1735 // the result of optimizing this module (e.g. module imports, exports, liveness
1736 // of access globals, etc).
1737 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1738 // used during the normal linker-plugin incremental thin-LTO process.
1740 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1741 SmallString<40> Key;
1742 llvm::lto::Config conf;
1743 const auto &ImportList = Data->ImportLists.lookup(ModId);
1744 const auto &ExportList = Data->ExportLists.lookup(ModId);
1745 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1746 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1747 std::set<GlobalValue::GUID> CfiFunctionDefs;
1748 std::set<GlobalValue::GUID> CfiFunctionDecls;
1750 // Based on the 'InProcessThinBackend' constructor in LLVM
1751 for (auto &Name : Data->Index.cfiFunctionDefs())
1752 CfiFunctionDefs.insert(
1753 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1754 for (auto &Name : Data->Index.cfiFunctionDecls())
1755 CfiFunctionDecls.insert(
1756 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1758 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1759 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1762 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());